1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 drbd_int.h
4
5 This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
6
7 Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
8 Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
9 Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
10
11
12 */
13
14 #ifndef _DRBD_INT_H
15 #define _DRBD_INT_H
16
17 #include <crypto/hash.h>
18 #include <linux/compiler.h>
19 #include <linux/types.h>
20 #include <linux/list.h>
21 #include <linux/sched/signal.h>
22 #include <linux/bitops.h>
23 #include <linux/slab.h>
24 #include <linux/ratelimit.h>
25 #include <linux/tcp.h>
26 #include <linux/mutex.h>
27 #include <linux/major.h>
28 #include <linux/blkdev.h>
29 #include <linux/backing-dev.h>
30 #include <linux/idr.h>
31 #include <linux/dynamic_debug.h>
32 #include <net/tcp.h>
33 #include <linux/lru_cache.h>
34 #include <linux/prefetch.h>
35 #include <linux/drbd_genl_api.h>
36 #include <linux/drbd.h>
37 #include <linux/drbd_config.h>
38 #include "drbd_strings.h"
39 #include "drbd_state.h"
40 #include "drbd_protocol.h"
41 #include "drbd_polymorph_printk.h"
42
43 /* shared module parameters, defined in drbd_main.c */
44 #ifdef CONFIG_DRBD_FAULT_INJECTION
45 extern int drbd_enable_faults;
46 extern int drbd_fault_rate;
47 #endif
48
49 extern unsigned int drbd_minor_count;
50 extern char drbd_usermode_helper[];
51 extern int drbd_proc_details;
52
53
54 /* This is used to stop/restart our threads.
55 * Cannot use SIGTERM nor SIGKILL, since these
56 * are sent out by init on runlevel changes
57 * I choose SIGHUP for now.
58 */
59 #define DRBD_SIGKILL SIGHUP
60
61 #define ID_IN_SYNC (4711ULL)
62 #define ID_OUT_OF_SYNC (4712ULL)
63 #define ID_SYNCER (-1ULL)
64
65 #define UUID_NEW_BM_OFFSET ((u64)0x0001000000000000ULL)
66
67 struct drbd_device;
68 struct drbd_connection;
69 struct drbd_peer_device;
70
71 /* Defines to control fault insertion */
72 enum {
73 DRBD_FAULT_MD_WR = 0, /* meta data write */
74 DRBD_FAULT_MD_RD = 1, /* read */
75 DRBD_FAULT_RS_WR = 2, /* resync */
76 DRBD_FAULT_RS_RD = 3,
77 DRBD_FAULT_DT_WR = 4, /* data */
78 DRBD_FAULT_DT_RD = 5,
79 DRBD_FAULT_DT_RA = 6, /* data read ahead */
80 DRBD_FAULT_BM_ALLOC = 7, /* bitmap allocation */
81 DRBD_FAULT_AL_EE = 8, /* alloc ee */
82 DRBD_FAULT_RECEIVE = 9, /* Changes some bytes upon receiving a [rs]data block */
83
84 DRBD_FAULT_MAX,
85 };
86
87 extern unsigned int
88 _drbd_insert_fault(struct drbd_device *device, unsigned int type);
89
90 static inline int
drbd_insert_fault(struct drbd_device * device,unsigned int type)91 drbd_insert_fault(struct drbd_device *device, unsigned int type) {
92 #ifdef CONFIG_DRBD_FAULT_INJECTION
93 return drbd_fault_rate &&
94 (drbd_enable_faults & (1<<type)) &&
95 _drbd_insert_fault(device, type);
96 #else
97 return 0;
98 #endif
99 }
100
101 /* integer division, round _UP_ to the next integer */
102 #define div_ceil(A, B) ((A)/(B) + ((A)%(B) ? 1 : 0))
103 /* usual integer division */
104 #define div_floor(A, B) ((A)/(B))
105
106 extern struct ratelimit_state drbd_ratelimit_state;
107 extern struct idr drbd_devices; /* RCU, updates: genl_lock() */
108 extern struct list_head drbd_resources; /* RCU, updates: genl_lock() */
109
110 extern const char *cmdname(enum drbd_packet cmd);
111
112 /* for sending/receiving the bitmap,
113 * possibly in some encoding scheme */
114 struct bm_xfer_ctx {
115 /* "const"
116 * stores total bits and long words
117 * of the bitmap, so we don't need to
118 * call the accessor functions over and again. */
119 unsigned long bm_bits;
120 unsigned long bm_words;
121 /* during xfer, current position within the bitmap */
122 unsigned long bit_offset;
123 unsigned long word_offset;
124
125 /* statistics; index: (h->command == P_BITMAP) */
126 unsigned packets[2];
127 unsigned bytes[2];
128 };
129
130 extern void INFO_bm_xfer_stats(struct drbd_peer_device *peer_device,
131 const char *direction, struct bm_xfer_ctx *c);
132
bm_xfer_ctx_bit_to_word_offset(struct bm_xfer_ctx * c)133 static inline void bm_xfer_ctx_bit_to_word_offset(struct bm_xfer_ctx *c)
134 {
135 /* word_offset counts "native long words" (32 or 64 bit),
136 * aligned at 64 bit.
137 * Encoded packet may end at an unaligned bit offset.
138 * In case a fallback clear text packet is transmitted in
139 * between, we adjust this offset back to the last 64bit
140 * aligned "native long word", which makes coding and decoding
141 * the plain text bitmap much more convenient. */
142 #if BITS_PER_LONG == 64
143 c->word_offset = c->bit_offset >> 6;
144 #elif BITS_PER_LONG == 32
145 c->word_offset = c->bit_offset >> 5;
146 c->word_offset &= ~(1UL);
147 #else
148 # error "unsupported BITS_PER_LONG"
149 #endif
150 }
151
152 extern unsigned int drbd_header_size(struct drbd_connection *connection);
153
154 /**********************************************************************/
155 enum drbd_thread_state {
156 NONE,
157 RUNNING,
158 EXITING,
159 RESTARTING
160 };
161
162 struct drbd_thread {
163 spinlock_t t_lock;
164 struct task_struct *task;
165 struct completion stop;
166 enum drbd_thread_state t_state;
167 int (*function) (struct drbd_thread *);
168 struct drbd_resource *resource;
169 struct drbd_connection *connection;
170 int reset_cpu_mask;
171 const char *name;
172 };
173
get_t_state(struct drbd_thread * thi)174 static inline enum drbd_thread_state get_t_state(struct drbd_thread *thi)
175 {
176 /* THINK testing the t_state seems to be uncritical in all cases
177 * (but thread_{start,stop}), so we can read it *without* the lock.
178 * --lge */
179
180 smp_rmb();
181 return thi->t_state;
182 }
183
184 struct drbd_work {
185 struct list_head list;
186 int (*cb)(struct drbd_work *, int cancel);
187 };
188
189 struct drbd_device_work {
190 struct drbd_work w;
191 struct drbd_device *device;
192 };
193
194 #include "drbd_interval.h"
195
196 extern int drbd_wait_misc(struct drbd_device *, struct drbd_interval *);
197
198 extern void lock_all_resources(void);
199 extern void unlock_all_resources(void);
200
201 struct drbd_request {
202 struct drbd_work w;
203 struct drbd_device *device;
204
205 /* if local IO is not allowed, will be NULL.
206 * if local IO _is_ allowed, holds the locally submitted bio clone,
207 * or, after local IO completion, the ERR_PTR(error).
208 * see drbd_request_endio(). */
209 struct bio *private_bio;
210
211 struct drbd_interval i;
212
213 /* epoch: used to check on "completion" whether this req was in
214 * the current epoch, and we therefore have to close it,
215 * causing a p_barrier packet to be send, starting a new epoch.
216 *
217 * This corresponds to "barrier" in struct p_barrier[_ack],
218 * and to "barrier_nr" in struct drbd_epoch (and various
219 * comments/function parameters/local variable names).
220 */
221 unsigned int epoch;
222
223 struct list_head tl_requests; /* ring list in the transfer log */
224 struct bio *master_bio; /* master bio pointer */
225
226 /* see struct drbd_device */
227 struct list_head req_pending_master_completion;
228 struct list_head req_pending_local;
229
230 /* for generic IO accounting */
231 unsigned long start_jif;
232
233 /* for DRBD internal statistics */
234
235 /* Minimal set of time stamps to determine if we wait for activity log
236 * transactions, local disk or peer. 32 bit "jiffies" are good enough,
237 * we don't expect a DRBD request to be stalled for several month.
238 */
239
240 /* before actual request processing */
241 unsigned long in_actlog_jif;
242
243 /* local disk */
244 unsigned long pre_submit_jif;
245
246 /* per connection */
247 unsigned long pre_send_jif;
248 unsigned long acked_jif;
249 unsigned long net_done_jif;
250
251 /* Possibly even more detail to track each phase:
252 * master_completion_jif
253 * how long did it take to complete the master bio
254 * (application visible latency)
255 * allocated_jif
256 * how long the master bio was blocked until we finally allocated
257 * a tracking struct
258 * in_actlog_jif
259 * how long did we wait for activity log transactions
260 *
261 * net_queued_jif
262 * when did we finally queue it for sending
263 * pre_send_jif
264 * when did we start sending it
265 * post_send_jif
266 * how long did we block in the network stack trying to send it
267 * acked_jif
268 * when did we receive (or fake, in protocol A) a remote ACK
269 * net_done_jif
270 * when did we receive final acknowledgement (P_BARRIER_ACK),
271 * or decide, e.g. on connection loss, that we do no longer expect
272 * anything from this peer for this request.
273 *
274 * pre_submit_jif
275 * post_sub_jif
276 * when did we start submiting to the lower level device,
277 * and how long did we block in that submit function
278 * local_completion_jif
279 * how long did it take the lower level device to complete this request
280 */
281
282
283 /* once it hits 0, we may complete the master_bio */
284 atomic_t completion_ref;
285 /* once it hits 0, we may destroy this drbd_request object */
286 struct kref kref;
287
288 unsigned rq_state; /* see comments above _req_mod() */
289 };
290
291 struct drbd_epoch {
292 struct drbd_connection *connection;
293 struct list_head list;
294 unsigned int barrier_nr;
295 atomic_t epoch_size; /* increased on every request added. */
296 atomic_t active; /* increased on every req. added, and dec on every finished. */
297 unsigned long flags;
298 };
299
300 /* drbd_epoch flag bits */
301 enum {
302 DE_HAVE_BARRIER_NUMBER,
303 };
304
305 enum epoch_event {
306 EV_PUT,
307 EV_GOT_BARRIER_NR,
308 EV_BECAME_LAST,
309 EV_CLEANUP = 32, /* used as flag */
310 };
311
312 struct digest_info {
313 int digest_size;
314 void *digest;
315 };
316
317 struct drbd_peer_request {
318 struct drbd_work w;
319 struct drbd_peer_device *peer_device;
320 struct drbd_epoch *epoch; /* for writes */
321 struct page *pages;
322 blk_opf_t opf;
323 atomic_t pending_bios;
324 struct drbd_interval i;
325 /* see comments on ee flag bits below */
326 unsigned long flags;
327 unsigned long submit_jif;
328 union {
329 u64 block_id;
330 struct digest_info *digest;
331 };
332 };
333
334 /* Equivalent to bio_op and req_op. */
335 #define peer_req_op(peer_req) \
336 ((peer_req)->opf & REQ_OP_MASK)
337
338 /* ee flag bits.
339 * While corresponding bios are in flight, the only modification will be
340 * set_bit WAS_ERROR, which has to be atomic.
341 * If no bios are in flight yet, or all have been completed,
342 * non-atomic modification to ee->flags is ok.
343 */
344 enum {
345 __EE_CALL_AL_COMPLETE_IO,
346 __EE_MAY_SET_IN_SYNC,
347
348 /* is this a TRIM aka REQ_OP_DISCARD? */
349 __EE_TRIM,
350 /* explicit zero-out requested, or
351 * our lower level cannot handle trim,
352 * and we want to fall back to zeroout instead */
353 __EE_ZEROOUT,
354
355 /* In case a barrier failed,
356 * we need to resubmit without the barrier flag. */
357 __EE_RESUBMITTED,
358
359 /* we may have several bios per peer request.
360 * if any of those fail, we set this flag atomically
361 * from the endio callback */
362 __EE_WAS_ERROR,
363
364 /* This ee has a pointer to a digest instead of a block id */
365 __EE_HAS_DIGEST,
366
367 /* Conflicting local requests need to be restarted after this request */
368 __EE_RESTART_REQUESTS,
369
370 /* The peer wants a write ACK for this (wire proto C) */
371 __EE_SEND_WRITE_ACK,
372
373 /* Is set when net_conf had two_primaries set while creating this peer_req */
374 __EE_IN_INTERVAL_TREE,
375
376 /* for debugfs: */
377 /* has this been submitted, or does it still wait for something else? */
378 __EE_SUBMITTED,
379
380 /* this is/was a write request */
381 __EE_WRITE,
382
383 /* this is/was a write same request */
384 __EE_WRITE_SAME,
385
386 /* this originates from application on peer
387 * (not some resync or verify or other DRBD internal request) */
388 __EE_APPLICATION,
389
390 /* If it contains only 0 bytes, send back P_RS_DEALLOCATED */
391 __EE_RS_THIN_REQ,
392 };
393 #define EE_CALL_AL_COMPLETE_IO (1<<__EE_CALL_AL_COMPLETE_IO)
394 #define EE_MAY_SET_IN_SYNC (1<<__EE_MAY_SET_IN_SYNC)
395 #define EE_TRIM (1<<__EE_TRIM)
396 #define EE_ZEROOUT (1<<__EE_ZEROOUT)
397 #define EE_RESUBMITTED (1<<__EE_RESUBMITTED)
398 #define EE_WAS_ERROR (1<<__EE_WAS_ERROR)
399 #define EE_HAS_DIGEST (1<<__EE_HAS_DIGEST)
400 #define EE_RESTART_REQUESTS (1<<__EE_RESTART_REQUESTS)
401 #define EE_SEND_WRITE_ACK (1<<__EE_SEND_WRITE_ACK)
402 #define EE_IN_INTERVAL_TREE (1<<__EE_IN_INTERVAL_TREE)
403 #define EE_SUBMITTED (1<<__EE_SUBMITTED)
404 #define EE_WRITE (1<<__EE_WRITE)
405 #define EE_WRITE_SAME (1<<__EE_WRITE_SAME)
406 #define EE_APPLICATION (1<<__EE_APPLICATION)
407 #define EE_RS_THIN_REQ (1<<__EE_RS_THIN_REQ)
408
409 /* flag bits per device */
410 enum {
411 UNPLUG_REMOTE, /* sending a "UnplugRemote" could help */
412 MD_DIRTY, /* current uuids and flags not yet on disk */
413 USE_DEGR_WFC_T, /* degr-wfc-timeout instead of wfc-timeout. */
414 CL_ST_CHG_SUCCESS,
415 CL_ST_CHG_FAIL,
416 CRASHED_PRIMARY, /* This node was a crashed primary.
417 * Gets cleared when the state.conn
418 * goes into C_CONNECTED state. */
419 CONSIDER_RESYNC,
420
421 MD_NO_FUA, /* Users wants us to not use FUA/FLUSH on meta data dev */
422
423 BITMAP_IO, /* suspend application io;
424 once no more io in flight, start bitmap io */
425 BITMAP_IO_QUEUED, /* Started bitmap IO */
426 WAS_IO_ERROR, /* Local disk failed, returned IO error */
427 WAS_READ_ERROR, /* Local disk READ failed (set additionally to the above) */
428 FORCE_DETACH, /* Force-detach from local disk, aborting any pending local IO */
429 RESYNC_AFTER_NEG, /* Resync after online grow after the attach&negotiate finished. */
430 RESIZE_PENDING, /* Size change detected locally, waiting for the response from
431 * the peer, if it changed there as well. */
432 NEW_CUR_UUID, /* Create new current UUID when thawing IO */
433 AL_SUSPENDED, /* Activity logging is currently suspended. */
434 AHEAD_TO_SYNC_SOURCE, /* Ahead -> SyncSource queued */
435 B_RS_H_DONE, /* Before resync handler done (already executed) */
436 DISCARD_MY_DATA, /* discard_my_data flag per volume */
437 READ_BALANCE_RR,
438
439 FLUSH_PENDING, /* if set, device->flush_jif is when we submitted that flush
440 * from drbd_flush_after_epoch() */
441
442 /* cleared only after backing device related structures have been destroyed. */
443 GOING_DISKLESS, /* Disk is being detached, because of io-error, or admin request. */
444
445 /* to be used in drbd_device_post_work() */
446 GO_DISKLESS, /* tell worker to schedule cleanup before detach */
447 DESTROY_DISK, /* tell worker to close backing devices and destroy related structures. */
448 MD_SYNC, /* tell worker to call drbd_md_sync() */
449 RS_START, /* tell worker to start resync/OV */
450 RS_PROGRESS, /* tell worker that resync made significant progress */
451 RS_DONE, /* tell worker that resync is done */
452 };
453
454 struct drbd_bitmap; /* opaque for drbd_device */
455
456 /* definition of bits in bm_flags to be used in drbd_bm_lock
457 * and drbd_bitmap_io and friends. */
458 enum bm_flag {
459 /* currently locked for bulk operation */
460 BM_LOCKED_MASK = 0xf,
461
462 /* in detail, that is: */
463 BM_DONT_CLEAR = 0x1,
464 BM_DONT_SET = 0x2,
465 BM_DONT_TEST = 0x4,
466
467 /* so we can mark it locked for bulk operation,
468 * and still allow all non-bulk operations */
469 BM_IS_LOCKED = 0x8,
470
471 /* (test bit, count bit) allowed (common case) */
472 BM_LOCKED_TEST_ALLOWED = BM_DONT_CLEAR | BM_DONT_SET | BM_IS_LOCKED,
473
474 /* testing bits, as well as setting new bits allowed, but clearing bits
475 * would be unexpected. Used during bitmap receive. Setting new bits
476 * requires sending of "out-of-sync" information, though. */
477 BM_LOCKED_SET_ALLOWED = BM_DONT_CLEAR | BM_IS_LOCKED,
478
479 /* for drbd_bm_write_copy_pages, everything is allowed,
480 * only concurrent bulk operations are locked out. */
481 BM_LOCKED_CHANGE_ALLOWED = BM_IS_LOCKED,
482 };
483
484 struct drbd_work_queue {
485 struct list_head q;
486 spinlock_t q_lock; /* to protect the list. */
487 wait_queue_head_t q_wait;
488 };
489
490 struct drbd_socket {
491 struct mutex mutex;
492 struct socket *socket;
493 /* this way we get our
494 * send/receive buffers off the stack */
495 void *sbuf;
496 void *rbuf;
497 };
498
499 struct drbd_md {
500 u64 md_offset; /* sector offset to 'super' block */
501
502 u64 la_size_sect; /* last agreed size, unit sectors */
503 spinlock_t uuid_lock;
504 u64 uuid[UI_SIZE];
505 u64 device_uuid;
506 u32 flags;
507 u32 md_size_sect;
508
509 s32 al_offset; /* signed relative sector offset to activity log */
510 s32 bm_offset; /* signed relative sector offset to bitmap */
511
512 /* cached value of bdev->disk_conf->meta_dev_idx (see below) */
513 s32 meta_dev_idx;
514
515 /* see al_tr_number_to_on_disk_sector() */
516 u32 al_stripes;
517 u32 al_stripe_size_4k;
518 u32 al_size_4k; /* cached product of the above */
519 };
520
521 struct drbd_backing_dev {
522 struct block_device *backing_bdev;
523 struct file *backing_bdev_file;
524 struct block_device *md_bdev;
525 struct file *f_md_bdev;
526 struct drbd_md md;
527 struct disk_conf *disk_conf; /* RCU, for updates: resource->conf_update */
528 sector_t known_size; /* last known size of that backing device */
529 };
530
531 struct drbd_md_io {
532 struct page *page;
533 unsigned long start_jif; /* last call to drbd_md_get_buffer */
534 unsigned long submit_jif; /* last _drbd_md_sync_page_io() submit */
535 const char *current_use;
536 atomic_t in_use;
537 unsigned int done;
538 int error;
539 };
540
541 struct bm_io_work {
542 struct drbd_work w;
543 struct drbd_peer_device *peer_device;
544 char *why;
545 enum bm_flag flags;
546 int (*io_fn)(struct drbd_device *device, struct drbd_peer_device *peer_device);
547 void (*done)(struct drbd_device *device, int rv);
548 };
549
550 struct fifo_buffer {
551 unsigned int head_index;
552 unsigned int size;
553 int total; /* sum of all values */
554 int values[] __counted_by(size);
555 };
556 extern struct fifo_buffer *fifo_alloc(unsigned int fifo_size);
557
558 /* flag bits per connection */
559 enum {
560 NET_CONGESTED, /* The data socket is congested */
561 RESOLVE_CONFLICTS, /* Set on one node, cleared on the peer! */
562 SEND_PING,
563 GOT_PING_ACK, /* set when we receive a ping_ack packet, ping_wait gets woken */
564 CONN_WD_ST_CHG_REQ, /* A cluster wide state change on the connection is active */
565 CONN_WD_ST_CHG_OKAY,
566 CONN_WD_ST_CHG_FAIL,
567 CONN_DRY_RUN, /* Expect disconnect after resync handshake. */
568 CREATE_BARRIER, /* next P_DATA is preceded by a P_BARRIER */
569 STATE_SENT, /* Do not change state/UUIDs while this is set */
570 CALLBACK_PENDING, /* Whether we have a call_usermodehelper(, UMH_WAIT_PROC)
571 * pending, from drbd worker context.
572 */
573 DISCONNECT_SENT,
574
575 DEVICE_WORK_PENDING, /* tell worker that some device has pending work */
576 };
577
578 enum which_state { NOW, OLD = NOW, NEW };
579
580 struct drbd_resource {
581 char *name;
582 #ifdef CONFIG_DEBUG_FS
583 struct dentry *debugfs_res;
584 struct dentry *debugfs_res_volumes;
585 struct dentry *debugfs_res_connections;
586 struct dentry *debugfs_res_in_flight_summary;
587 #endif
588 struct kref kref;
589 struct idr devices; /* volume number to device mapping */
590 struct list_head connections;
591 struct list_head resources;
592 struct res_opts res_opts;
593 struct mutex conf_update; /* mutex for ready-copy-update of net_conf and disk_conf */
594 struct mutex adm_mutex; /* mutex to serialize administrative requests */
595 spinlock_t req_lock;
596
597 unsigned susp:1; /* IO suspended by user */
598 unsigned susp_nod:1; /* IO suspended because no data */
599 unsigned susp_fen:1; /* IO suspended because fence peer handler runs */
600
601 enum write_ordering_e write_ordering;
602
603 cpumask_var_t cpu_mask;
604 };
605
606 struct drbd_thread_timing_details
607 {
608 unsigned long start_jif;
609 void *cb_addr;
610 const char *caller_fn;
611 unsigned int line;
612 unsigned int cb_nr;
613 };
614
615 struct drbd_connection {
616 struct list_head connections;
617 struct drbd_resource *resource;
618 #ifdef CONFIG_DEBUG_FS
619 struct dentry *debugfs_conn;
620 struct dentry *debugfs_conn_callback_history;
621 struct dentry *debugfs_conn_oldest_requests;
622 #endif
623 struct kref kref;
624 struct idr peer_devices; /* volume number to peer device mapping */
625 enum drbd_conns cstate; /* Only C_STANDALONE to C_WF_REPORT_PARAMS */
626 struct mutex cstate_mutex; /* Protects graceful disconnects */
627 unsigned int connect_cnt; /* Inc each time a connection is established */
628
629 unsigned long flags;
630 struct net_conf *net_conf; /* content protected by rcu */
631 wait_queue_head_t ping_wait; /* Woken upon reception of a ping, and a state change */
632
633 struct sockaddr_storage my_addr;
634 int my_addr_len;
635 struct sockaddr_storage peer_addr;
636 int peer_addr_len;
637
638 struct drbd_socket data; /* data/barrier/cstate/parameter packets */
639 struct drbd_socket meta; /* ping/ack (metadata) packets */
640 int agreed_pro_version; /* actually used protocol version */
641 u32 agreed_features;
642 unsigned long last_received; /* in jiffies, either socket */
643 unsigned int ko_count;
644
645 struct list_head transfer_log; /* all requests not yet fully processed */
646
647 struct crypto_shash *cram_hmac_tfm;
648 struct crypto_shash *integrity_tfm; /* checksums we compute, updates protected by connection->data->mutex */
649 struct crypto_shash *peer_integrity_tfm; /* checksums we verify, only accessed from receiver thread */
650 struct crypto_shash *csums_tfm;
651 struct crypto_shash *verify_tfm;
652 void *int_dig_in;
653 void *int_dig_vv;
654
655 /* receiver side */
656 struct drbd_epoch *current_epoch;
657 spinlock_t epoch_lock;
658 unsigned int epochs;
659 atomic_t current_tle_nr; /* transfer log epoch number */
660 unsigned current_tle_writes; /* writes seen within this tl epoch */
661
662 unsigned long last_reconnect_jif;
663 /* empty member on older kernels without blk_start_plug() */
664 struct blk_plug receiver_plug;
665 struct drbd_thread receiver;
666 struct drbd_thread worker;
667 struct drbd_thread ack_receiver;
668 struct workqueue_struct *ack_sender;
669
670 /* cached pointers,
671 * so we can look up the oldest pending requests more quickly.
672 * protected by resource->req_lock */
673 struct drbd_request *req_next; /* DRBD 9: todo.req_next */
674 struct drbd_request *req_ack_pending;
675 struct drbd_request *req_not_net_done;
676
677 /* sender side */
678 struct drbd_work_queue sender_work;
679
680 #define DRBD_THREAD_DETAILS_HIST 16
681 unsigned int w_cb_nr; /* keeps counting up */
682 unsigned int r_cb_nr; /* keeps counting up */
683 struct drbd_thread_timing_details w_timing_details[DRBD_THREAD_DETAILS_HIST];
684 struct drbd_thread_timing_details r_timing_details[DRBD_THREAD_DETAILS_HIST];
685
686 struct {
687 unsigned long last_sent_barrier_jif;
688
689 /* whether this sender thread
690 * has processed a single write yet. */
691 bool seen_any_write_yet;
692
693 /* Which barrier number to send with the next P_BARRIER */
694 int current_epoch_nr;
695
696 /* how many write requests have been sent
697 * with req->epoch == current_epoch_nr.
698 * If none, no P_BARRIER will be sent. */
699 unsigned current_epoch_writes;
700 } send;
701 };
702
has_net_conf(struct drbd_connection * connection)703 static inline bool has_net_conf(struct drbd_connection *connection)
704 {
705 bool has_net_conf;
706
707 rcu_read_lock();
708 has_net_conf = rcu_dereference(connection->net_conf);
709 rcu_read_unlock();
710
711 return has_net_conf;
712 }
713
714 void __update_timing_details(
715 struct drbd_thread_timing_details *tdp,
716 unsigned int *cb_nr,
717 void *cb,
718 const char *fn, const unsigned int line);
719
720 #define update_worker_timing_details(c, cb) \
721 __update_timing_details(c->w_timing_details, &c->w_cb_nr, cb, __func__ , __LINE__ )
722 #define update_receiver_timing_details(c, cb) \
723 __update_timing_details(c->r_timing_details, &c->r_cb_nr, cb, __func__ , __LINE__ )
724
725 struct submit_worker {
726 struct workqueue_struct *wq;
727 struct work_struct worker;
728
729 /* protected by ..->resource->req_lock */
730 struct list_head writes;
731 };
732
733 struct drbd_peer_device {
734 struct list_head peer_devices;
735 struct drbd_device *device;
736 struct drbd_connection *connection;
737 struct work_struct send_acks_work;
738 #ifdef CONFIG_DEBUG_FS
739 struct dentry *debugfs_peer_dev;
740 #endif
741 };
742
743 struct drbd_device {
744 struct drbd_resource *resource;
745 struct list_head peer_devices;
746 struct list_head pending_bitmap_io;
747
748 unsigned long flush_jif;
749 #ifdef CONFIG_DEBUG_FS
750 struct dentry *debugfs_minor;
751 struct dentry *debugfs_vol;
752 struct dentry *debugfs_vol_oldest_requests;
753 struct dentry *debugfs_vol_act_log_extents;
754 struct dentry *debugfs_vol_resync_extents;
755 struct dentry *debugfs_vol_data_gen_id;
756 struct dentry *debugfs_vol_ed_gen_id;
757 #endif
758
759 unsigned int vnr; /* volume number within the connection */
760 unsigned int minor; /* device minor number */
761
762 struct kref kref;
763
764 /* things that are stored as / read from meta data on disk */
765 unsigned long flags;
766
767 /* configured by drbdsetup */
768 struct drbd_backing_dev *ldev;
769
770 sector_t p_size; /* partner's disk size */
771 struct request_queue *rq_queue;
772 struct gendisk *vdisk;
773
774 unsigned long last_reattach_jif;
775 struct drbd_work resync_work;
776 struct drbd_work unplug_work;
777 struct timer_list resync_timer;
778 struct timer_list md_sync_timer;
779 struct timer_list start_resync_timer;
780 struct timer_list request_timer;
781
782 /* Used after attach while negotiating new disk state. */
783 union drbd_state new_state_tmp;
784
785 union drbd_dev_state state;
786 wait_queue_head_t misc_wait;
787 wait_queue_head_t state_wait; /* upon each state change. */
788 unsigned int send_cnt;
789 unsigned int recv_cnt;
790 unsigned int read_cnt;
791 unsigned int writ_cnt;
792 unsigned int al_writ_cnt;
793 unsigned int bm_writ_cnt;
794 atomic_t ap_bio_cnt; /* Requests we need to complete */
795 atomic_t ap_actlog_cnt; /* Requests waiting for activity log */
796 atomic_t ap_pending_cnt; /* AP data packets on the wire, ack expected */
797 atomic_t rs_pending_cnt; /* RS request/data packets on the wire */
798 atomic_t unacked_cnt; /* Need to send replies for */
799 atomic_t local_cnt; /* Waiting for local completion */
800 atomic_t suspend_cnt;
801
802 /* Interval tree of pending local requests */
803 struct rb_root read_requests;
804 struct rb_root write_requests;
805
806 /* for statistics and timeouts */
807 /* [0] read, [1] write */
808 struct list_head pending_master_completion[2];
809 struct list_head pending_completion[2];
810
811 /* use checksums for *this* resync */
812 bool use_csums;
813 /* blocks to resync in this run [unit BM_BLOCK_SIZE] */
814 unsigned long rs_total;
815 /* number of resync blocks that failed in this run */
816 unsigned long rs_failed;
817 /* Syncer's start time [unit jiffies] */
818 unsigned long rs_start;
819 /* cumulated time in PausedSyncX state [unit jiffies] */
820 unsigned long rs_paused;
821 /* skipped because csum was equal [unit BM_BLOCK_SIZE] */
822 unsigned long rs_same_csum;
823 #define DRBD_SYNC_MARKS 8
824 #define DRBD_SYNC_MARK_STEP (3*HZ)
825 /* block not up-to-date at mark [unit BM_BLOCK_SIZE] */
826 unsigned long rs_mark_left[DRBD_SYNC_MARKS];
827 /* marks's time [unit jiffies] */
828 unsigned long rs_mark_time[DRBD_SYNC_MARKS];
829 /* current index into rs_mark_{left,time} */
830 int rs_last_mark;
831 unsigned long rs_last_bcast; /* [unit jiffies] */
832
833 /* where does the admin want us to start? (sector) */
834 sector_t ov_start_sector;
835 sector_t ov_stop_sector;
836 /* where are we now? (sector) */
837 sector_t ov_position;
838 /* Start sector of out of sync range (to merge printk reporting). */
839 sector_t ov_last_oos_start;
840 /* size of out-of-sync range in sectors. */
841 sector_t ov_last_oos_size;
842 unsigned long ov_left; /* in bits */
843
844 struct drbd_bitmap *bitmap;
845 unsigned long bm_resync_fo; /* bit offset for drbd_bm_find_next */
846
847 /* Used to track operations of resync... */
848 struct lru_cache *resync;
849 /* Number of locked elements in resync LRU */
850 unsigned int resync_locked;
851 /* resync extent number waiting for application requests */
852 unsigned int resync_wenr;
853
854 int open_cnt;
855 u64 *p_uuid;
856
857 struct list_head active_ee; /* IO in progress (P_DATA gets written to disk) */
858 struct list_head sync_ee; /* IO in progress (P_RS_DATA_REPLY gets written to disk) */
859 struct list_head done_ee; /* need to send P_WRITE_ACK */
860 struct list_head read_ee; /* [RS]P_DATA_REQUEST being read */
861 struct list_head net_ee; /* zero-copy network send in progress */
862
863 struct list_head resync_reads;
864 atomic_t pp_in_use; /* allocated from page pool */
865 atomic_t pp_in_use_by_net; /* sendpage()d, still referenced by tcp */
866 wait_queue_head_t ee_wait;
867 struct drbd_md_io md_io;
868 spinlock_t al_lock;
869 wait_queue_head_t al_wait;
870 struct lru_cache *act_log; /* activity log */
871 unsigned int al_tr_number;
872 int al_tr_cycle;
873 wait_queue_head_t seq_wait;
874 atomic_t packet_seq;
875 unsigned int peer_seq;
876 spinlock_t peer_seq_lock;
877 unsigned long comm_bm_set; /* communicated number of set bits. */
878 struct bm_io_work bm_io_work;
879 u64 ed_uuid; /* UUID of the exposed data */
880 struct mutex own_state_mutex;
881 struct mutex *state_mutex; /* either own_state_mutex or first_peer_device(device)->connection->cstate_mutex */
882 char congestion_reason; /* Why we where congested... */
883 atomic_t rs_sect_in; /* for incoming resync data rate, SyncTarget */
884 atomic_t rs_sect_ev; /* for submitted resync data rate, both */
885 int rs_last_sect_ev; /* counter to compare with */
886 int rs_last_events; /* counter of read or write "events" (unit sectors)
887 * on the lower level device when we last looked. */
888 int c_sync_rate; /* current resync rate after syncer throttle magic */
889 struct fifo_buffer *rs_plan_s; /* correction values of resync planer (RCU, connection->conn_update) */
890 int rs_in_flight; /* resync sectors in flight (to proxy, in proxy and from proxy) */
891 atomic_t ap_in_flight; /* App sectors in flight (waiting for ack) */
892 unsigned int peer_max_bio_size;
893 unsigned int local_max_bio_size;
894
895 /* any requests that would block in drbd_make_request()
896 * are deferred to this single-threaded work queue */
897 struct submit_worker submit;
898 };
899
900 struct drbd_bm_aio_ctx {
901 struct drbd_device *device;
902 struct list_head list; /* on device->pending_bitmap_io */;
903 unsigned long start_jif;
904 atomic_t in_flight;
905 unsigned int done;
906 unsigned flags;
907 #define BM_AIO_COPY_PAGES 1
908 #define BM_AIO_WRITE_HINTED 2
909 #define BM_AIO_WRITE_ALL_PAGES 4
910 #define BM_AIO_READ 8
911 int error;
912 struct kref kref;
913 };
914
915 struct drbd_config_context {
916 /* assigned from drbd_genlmsghdr */
917 unsigned int minor;
918 /* assigned from request attributes, if present */
919 unsigned int volume;
920 #define VOLUME_UNSPECIFIED (-1U)
921 /* pointer into the request skb,
922 * limited lifetime! */
923 char *resource_name;
924 struct nlattr *my_addr;
925 struct nlattr *peer_addr;
926
927 /* reply buffer */
928 struct sk_buff *reply_skb;
929 /* pointer into reply buffer */
930 struct drbd_genlmsghdr *reply_dh;
931 /* resolved from attributes, if possible */
932 struct drbd_device *device;
933 struct drbd_resource *resource;
934 struct drbd_connection *connection;
935 };
936
minor_to_device(unsigned int minor)937 static inline struct drbd_device *minor_to_device(unsigned int minor)
938 {
939 return (struct drbd_device *)idr_find(&drbd_devices, minor);
940 }
941
first_peer_device(struct drbd_device * device)942 static inline struct drbd_peer_device *first_peer_device(struct drbd_device *device)
943 {
944 return list_first_entry_or_null(&device->peer_devices, struct drbd_peer_device, peer_devices);
945 }
946
947 static inline struct drbd_peer_device *
conn_peer_device(struct drbd_connection * connection,int volume_number)948 conn_peer_device(struct drbd_connection *connection, int volume_number)
949 {
950 return idr_find(&connection->peer_devices, volume_number);
951 }
952
953 #define for_each_resource(resource, _resources) \
954 list_for_each_entry(resource, _resources, resources)
955
956 #define for_each_resource_rcu(resource, _resources) \
957 list_for_each_entry_rcu(resource, _resources, resources)
958
959 #define for_each_resource_safe(resource, tmp, _resources) \
960 list_for_each_entry_safe(resource, tmp, _resources, resources)
961
962 #define for_each_connection(connection, resource) \
963 list_for_each_entry(connection, &resource->connections, connections)
964
965 #define for_each_connection_rcu(connection, resource) \
966 list_for_each_entry_rcu(connection, &resource->connections, connections)
967
968 #define for_each_connection_safe(connection, tmp, resource) \
969 list_for_each_entry_safe(connection, tmp, &resource->connections, connections)
970
971 #define for_each_peer_device(peer_device, device) \
972 list_for_each_entry(peer_device, &device->peer_devices, peer_devices)
973
974 #define for_each_peer_device_rcu(peer_device, device) \
975 list_for_each_entry_rcu(peer_device, &device->peer_devices, peer_devices)
976
977 #define for_each_peer_device_safe(peer_device, tmp, device) \
978 list_for_each_entry_safe(peer_device, tmp, &device->peer_devices, peer_devices)
979
device_to_minor(struct drbd_device * device)980 static inline unsigned int device_to_minor(struct drbd_device *device)
981 {
982 return device->minor;
983 }
984
985 /*
986 * function declarations
987 *************************/
988
989 /* drbd_main.c */
990
991 enum dds_flags {
992 DDSF_FORCED = 1,
993 DDSF_NO_RESYNC = 2, /* Do not run a resync for the new space */
994 };
995
996 extern void drbd_init_set_defaults(struct drbd_device *device);
997 extern int drbd_thread_start(struct drbd_thread *thi);
998 extern void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait);
999 #ifdef CONFIG_SMP
1000 extern void drbd_thread_current_set_cpu(struct drbd_thread *thi);
1001 #else
1002 #define drbd_thread_current_set_cpu(A) ({})
1003 #endif
1004 extern void tl_release(struct drbd_connection *, unsigned int barrier_nr,
1005 unsigned int set_size);
1006 extern void tl_clear(struct drbd_connection *);
1007 extern void drbd_free_sock(struct drbd_connection *connection);
1008 extern int drbd_send(struct drbd_connection *connection, struct socket *sock,
1009 void *buf, size_t size, unsigned msg_flags);
1010 extern int drbd_send_all(struct drbd_connection *, struct socket *, void *, size_t,
1011 unsigned);
1012
1013 extern int __drbd_send_protocol(struct drbd_connection *connection, enum drbd_packet cmd);
1014 extern int drbd_send_protocol(struct drbd_connection *connection);
1015 extern int drbd_send_uuids(struct drbd_peer_device *);
1016 extern int drbd_send_uuids_skip_initial_sync(struct drbd_peer_device *);
1017 extern void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *);
1018 extern int drbd_send_sizes(struct drbd_peer_device *, int trigger_reply, enum dds_flags flags);
1019 extern int drbd_send_state(struct drbd_peer_device *, union drbd_state s);
1020 extern int drbd_send_current_state(struct drbd_peer_device *);
1021 extern int drbd_send_sync_param(struct drbd_peer_device *);
1022 extern void drbd_send_b_ack(struct drbd_connection *connection, u32 barrier_nr,
1023 u32 set_size);
1024 extern int drbd_send_ack(struct drbd_peer_device *, enum drbd_packet,
1025 struct drbd_peer_request *);
1026 extern void drbd_send_ack_rp(struct drbd_peer_device *, enum drbd_packet,
1027 struct p_block_req *rp);
1028 extern void drbd_send_ack_dp(struct drbd_peer_device *, enum drbd_packet,
1029 struct p_data *dp, int data_size);
1030 extern int drbd_send_ack_ex(struct drbd_peer_device *, enum drbd_packet,
1031 sector_t sector, int blksize, u64 block_id);
1032 extern int drbd_send_out_of_sync(struct drbd_peer_device *, struct drbd_request *);
1033 extern int drbd_send_block(struct drbd_peer_device *, enum drbd_packet,
1034 struct drbd_peer_request *);
1035 extern int drbd_send_dblock(struct drbd_peer_device *, struct drbd_request *req);
1036 extern int drbd_send_drequest(struct drbd_peer_device *, int cmd,
1037 sector_t sector, int size, u64 block_id);
1038 extern int drbd_send_drequest_csum(struct drbd_peer_device *, sector_t sector,
1039 int size, void *digest, int digest_size,
1040 enum drbd_packet cmd);
1041 extern int drbd_send_ov_request(struct drbd_peer_device *, sector_t sector, int size);
1042
1043 extern int drbd_send_bitmap(struct drbd_device *device, struct drbd_peer_device *peer_device);
1044 extern void drbd_send_sr_reply(struct drbd_peer_device *, enum drbd_state_rv retcode);
1045 extern void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode);
1046 extern int drbd_send_rs_deallocated(struct drbd_peer_device *, struct drbd_peer_request *);
1047 extern void drbd_backing_dev_free(struct drbd_device *device, struct drbd_backing_dev *ldev);
1048 extern void drbd_device_cleanup(struct drbd_device *device);
1049 extern void drbd_print_uuids(struct drbd_device *device, const char *text);
1050 extern void drbd_queue_unplug(struct drbd_device *device);
1051
1052 extern void conn_md_sync(struct drbd_connection *connection);
1053 extern void drbd_md_write(struct drbd_device *device, void *buffer);
1054 extern void drbd_md_sync(struct drbd_device *device);
1055 extern int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev);
1056 extern void drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local);
1057 extern void _drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local);
1058 extern void drbd_uuid_new_current(struct drbd_device *device) __must_hold(local);
1059 extern void drbd_uuid_set_bm(struct drbd_device *device, u64 val) __must_hold(local);
1060 extern void drbd_uuid_move_history(struct drbd_device *device) __must_hold(local);
1061 extern void __drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local);
1062 extern void drbd_md_set_flag(struct drbd_device *device, int flags) __must_hold(local);
1063 extern void drbd_md_clear_flag(struct drbd_device *device, int flags)__must_hold(local);
1064 extern int drbd_md_test_flag(struct drbd_backing_dev *, int);
1065 extern void drbd_md_mark_dirty(struct drbd_device *device);
1066 extern void drbd_queue_bitmap_io(struct drbd_device *device,
1067 int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
1068 void (*done)(struct drbd_device *, int),
1069 char *why, enum bm_flag flags,
1070 struct drbd_peer_device *peer_device);
1071 extern int drbd_bitmap_io(struct drbd_device *device,
1072 int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
1073 char *why, enum bm_flag flags,
1074 struct drbd_peer_device *peer_device);
1075 extern int drbd_bitmap_io_from_worker(struct drbd_device *device,
1076 int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
1077 char *why, enum bm_flag flags,
1078 struct drbd_peer_device *peer_device);
1079 extern int drbd_bmio_set_n_write(struct drbd_device *device,
1080 struct drbd_peer_device *peer_device) __must_hold(local);
1081 extern int drbd_bmio_clear_n_write(struct drbd_device *device,
1082 struct drbd_peer_device *peer_device) __must_hold(local);
1083
1084 /* Meta data layout
1085 *
1086 * We currently have two possible layouts.
1087 * Offsets in (512 byte) sectors.
1088 * external:
1089 * |----------- md_size_sect ------------------|
1090 * [ 4k superblock ][ activity log ][ Bitmap ]
1091 * | al_offset == 8 |
1092 * | bm_offset = al_offset + X |
1093 * ==> bitmap sectors = md_size_sect - bm_offset
1094 *
1095 * Variants:
1096 * old, indexed fixed size meta data:
1097 *
1098 * internal:
1099 * |----------- md_size_sect ------------------|
1100 * [data.....][ Bitmap ][ activity log ][ 4k superblock ][padding*]
1101 * | al_offset < 0 |
1102 * | bm_offset = al_offset - Y |
1103 * ==> bitmap sectors = Y = al_offset - bm_offset
1104 *
1105 * [padding*] are zero or up to 7 unused 512 Byte sectors to the
1106 * end of the device, so that the [4k superblock] will be 4k aligned.
1107 *
1108 * The activity log consists of 4k transaction blocks,
1109 * which are written in a ring-buffer, or striped ring-buffer like fashion,
1110 * which are writtensize used to be fixed 32kB,
1111 * but is about to become configurable.
1112 */
1113
1114 /* Our old fixed size meta data layout
1115 * allows up to about 3.8TB, so if you want more,
1116 * you need to use the "flexible" meta data format. */
1117 #define MD_128MB_SECT (128LLU << 11) /* 128 MB, unit sectors */
1118 #define MD_4kB_SECT 8
1119 #define MD_32kB_SECT 64
1120
1121 /* One activity log extent represents 4M of storage */
1122 #define AL_EXTENT_SHIFT 22
1123 #define AL_EXTENT_SIZE (1<<AL_EXTENT_SHIFT)
1124
1125 /* We could make these currently hardcoded constants configurable
1126 * variables at create-md time (or even re-configurable at runtime?).
1127 * Which will require some more changes to the DRBD "super block"
1128 * and attach code.
1129 *
1130 * updates per transaction:
1131 * This many changes to the active set can be logged with one transaction.
1132 * This number is arbitrary.
1133 * context per transaction:
1134 * This many context extent numbers are logged with each transaction.
1135 * This number is resulting from the transaction block size (4k), the layout
1136 * of the transaction header, and the number of updates per transaction.
1137 * See drbd_actlog.c:struct al_transaction_on_disk
1138 * */
1139 #define AL_UPDATES_PER_TRANSACTION 64 // arbitrary
1140 #define AL_CONTEXT_PER_TRANSACTION 919 // (4096 - 36 - 6*64)/4
1141
1142 #if BITS_PER_LONG == 32
1143 #define LN2_BPL 5
1144 #define cpu_to_lel(A) cpu_to_le32(A)
1145 #define lel_to_cpu(A) le32_to_cpu(A)
1146 #elif BITS_PER_LONG == 64
1147 #define LN2_BPL 6
1148 #define cpu_to_lel(A) cpu_to_le64(A)
1149 #define lel_to_cpu(A) le64_to_cpu(A)
1150 #else
1151 #error "LN2 of BITS_PER_LONG unknown!"
1152 #endif
1153
1154 /* resync bitmap */
1155 /* 16MB sized 'bitmap extent' to track syncer usage */
1156 struct bm_extent {
1157 int rs_left; /* number of bits set (out of sync) in this extent. */
1158 int rs_failed; /* number of failed resync requests in this extent. */
1159 unsigned long flags;
1160 struct lc_element lce;
1161 };
1162
1163 #define BME_NO_WRITES 0 /* bm_extent.flags: no more requests on this one! */
1164 #define BME_LOCKED 1 /* bm_extent.flags: syncer active on this one. */
1165 #define BME_PRIORITY 2 /* finish resync IO on this extent ASAP! App IO waiting! */
1166
1167 /* drbd_bitmap.c */
1168 /*
1169 * We need to store one bit for a block.
1170 * Example: 1GB disk @ 4096 byte blocks ==> we need 32 KB bitmap.
1171 * Bit 0 ==> local node thinks this block is binary identical on both nodes
1172 * Bit 1 ==> local node thinks this block needs to be synced.
1173 */
1174
1175 #define SLEEP_TIME (HZ/10)
1176
1177 /* We do bitmap IO in units of 4k blocks.
1178 * We also still have a hardcoded 4k per bit relation. */
1179 #define BM_BLOCK_SHIFT 12 /* 4k per bit */
1180 #define BM_BLOCK_SIZE (1<<BM_BLOCK_SHIFT)
1181 /* mostly arbitrarily set the represented size of one bitmap extent,
1182 * aka resync extent, to 16 MiB (which is also 512 Byte worth of bitmap
1183 * at 4k per bit resolution) */
1184 #define BM_EXT_SHIFT 24 /* 16 MiB per resync extent */
1185 #define BM_EXT_SIZE (1<<BM_EXT_SHIFT)
1186
1187 #if (BM_EXT_SHIFT != 24) || (BM_BLOCK_SHIFT != 12)
1188 #error "HAVE YOU FIXED drbdmeta AS WELL??"
1189 #endif
1190
1191 /* thus many _storage_ sectors are described by one bit */
1192 #define BM_SECT_TO_BIT(x) ((x)>>(BM_BLOCK_SHIFT-9))
1193 #define BM_BIT_TO_SECT(x) ((sector_t)(x)<<(BM_BLOCK_SHIFT-9))
1194 #define BM_SECT_PER_BIT BM_BIT_TO_SECT(1)
1195
1196 /* bit to represented kilo byte conversion */
1197 #define Bit2KB(bits) ((bits)<<(BM_BLOCK_SHIFT-10))
1198
1199 /* in which _bitmap_ extent (resp. sector) the bit for a certain
1200 * _storage_ sector is located in */
1201 #define BM_SECT_TO_EXT(x) ((x)>>(BM_EXT_SHIFT-9))
1202 #define BM_BIT_TO_EXT(x) ((x) >> (BM_EXT_SHIFT - BM_BLOCK_SHIFT))
1203
1204 /* first storage sector a bitmap extent corresponds to */
1205 #define BM_EXT_TO_SECT(x) ((sector_t)(x) << (BM_EXT_SHIFT-9))
1206 /* how much _storage_ sectors we have per bitmap extent */
1207 #define BM_SECT_PER_EXT BM_EXT_TO_SECT(1)
1208 /* how many bits are covered by one bitmap extent (resync extent) */
1209 #define BM_BITS_PER_EXT (1UL << (BM_EXT_SHIFT - BM_BLOCK_SHIFT))
1210
1211 #define BM_BLOCKS_PER_BM_EXT_MASK (BM_BITS_PER_EXT - 1)
1212
1213
1214 /* in one sector of the bitmap, we have this many activity_log extents. */
1215 #define AL_EXT_PER_BM_SECT (1 << (BM_EXT_SHIFT - AL_EXTENT_SHIFT))
1216
1217 /* the extent in "PER_EXTENT" below is an activity log extent
1218 * we need that many (long words/bytes) to store the bitmap
1219 * of one AL_EXTENT_SIZE chunk of storage.
1220 * we can store the bitmap for that many AL_EXTENTS within
1221 * one sector of the _on_disk_ bitmap:
1222 * bit 0 bit 37 bit 38 bit (512*8)-1
1223 * ...|........|........|.. // ..|........|
1224 * sect. 0 `296 `304 ^(512*8*8)-1
1225 *
1226 #define BM_WORDS_PER_EXT ( (AL_EXT_SIZE/BM_BLOCK_SIZE) / BITS_PER_LONG )
1227 #define BM_BYTES_PER_EXT ( (AL_EXT_SIZE/BM_BLOCK_SIZE) / 8 ) // 128
1228 #define BM_EXT_PER_SECT ( 512 / BM_BYTES_PER_EXTENT ) // 4
1229 */
1230
1231 #define DRBD_MAX_SECTORS_32 (0xffffffffLU)
1232 /* we have a certain meta data variant that has a fixed on-disk size of 128
1233 * MiB, of which 4k are our "superblock", and 32k are the fixed size activity
1234 * log, leaving this many sectors for the bitmap.
1235 */
1236
1237 #define DRBD_MAX_SECTORS_FIXED_BM \
1238 ((MD_128MB_SECT - MD_32kB_SECT - MD_4kB_SECT) * (1LL<<(BM_EXT_SHIFT-9)))
1239 #define DRBD_MAX_SECTORS DRBD_MAX_SECTORS_FIXED_BM
1240 /* 16 TB in units of sectors */
1241 #if BITS_PER_LONG == 32
1242 /* adjust by one page worth of bitmap,
1243 * so we won't wrap around in drbd_bm_find_next_bit.
1244 * you should use 64bit OS for that much storage, anyways. */
1245 #define DRBD_MAX_SECTORS_FLEX BM_BIT_TO_SECT(0xffff7fff)
1246 #else
1247 /* we allow up to 1 PiB now on 64bit architecture with "flexible" meta data */
1248 #define DRBD_MAX_SECTORS_FLEX (1UL << 51)
1249 /* corresponds to (1UL << 38) bits right now. */
1250 #endif
1251
1252 /* Estimate max bio size as 256 * PAGE_SIZE,
1253 * so for typical PAGE_SIZE of 4k, that is (1<<20) Byte.
1254 * Since we may live in a mixed-platform cluster,
1255 * we limit us to a platform agnostic constant here for now.
1256 * A followup commit may allow even bigger BIO sizes,
1257 * once we thought that through. */
1258 #define DRBD_MAX_BIO_SIZE (1U << 20)
1259 #if DRBD_MAX_BIO_SIZE > (BIO_MAX_VECS << PAGE_SHIFT)
1260 #error Architecture not supported: DRBD_MAX_BIO_SIZE > BIO_MAX_SIZE
1261 #endif
1262 #define DRBD_MAX_BIO_SIZE_SAFE (1U << 12) /* Works always = 4k */
1263
1264 #define DRBD_MAX_SIZE_H80_PACKET (1U << 15) /* Header 80 only allows packets up to 32KiB data */
1265 #define DRBD_MAX_BIO_SIZE_P95 (1U << 17) /* Protocol 95 to 99 allows bios up to 128KiB */
1266
1267 /* For now, don't allow more than half of what we can "activate" in one
1268 * activity log transaction to be discarded in one go. We may need to rework
1269 * drbd_al_begin_io() to allow for even larger discard ranges */
1270 #define DRBD_MAX_BATCH_BIO_SIZE (AL_UPDATES_PER_TRANSACTION/2*AL_EXTENT_SIZE)
1271 #define DRBD_MAX_BBIO_SECTORS (DRBD_MAX_BATCH_BIO_SIZE >> 9)
1272
1273 extern int drbd_bm_init(struct drbd_device *device);
1274 extern int drbd_bm_resize(struct drbd_device *device, sector_t sectors, int set_new_bits);
1275 extern void drbd_bm_cleanup(struct drbd_device *device);
1276 extern void drbd_bm_set_all(struct drbd_device *device);
1277 extern void drbd_bm_clear_all(struct drbd_device *device);
1278 /* set/clear/test only a few bits at a time */
1279 extern int drbd_bm_set_bits(
1280 struct drbd_device *device, unsigned long s, unsigned long e);
1281 extern int drbd_bm_clear_bits(
1282 struct drbd_device *device, unsigned long s, unsigned long e);
1283 extern int drbd_bm_count_bits(
1284 struct drbd_device *device, const unsigned long s, const unsigned long e);
1285 /* bm_set_bits variant for use while holding drbd_bm_lock,
1286 * may process the whole bitmap in one go */
1287 extern void _drbd_bm_set_bits(struct drbd_device *device,
1288 const unsigned long s, const unsigned long e);
1289 extern int drbd_bm_test_bit(struct drbd_device *device, unsigned long bitnr);
1290 extern int drbd_bm_e_weight(struct drbd_device *device, unsigned long enr);
1291 extern int drbd_bm_read(struct drbd_device *device,
1292 struct drbd_peer_device *peer_device) __must_hold(local);
1293 extern void drbd_bm_mark_for_writeout(struct drbd_device *device, int page_nr);
1294 extern int drbd_bm_write(struct drbd_device *device,
1295 struct drbd_peer_device *peer_device) __must_hold(local);
1296 extern void drbd_bm_reset_al_hints(struct drbd_device *device) __must_hold(local);
1297 extern int drbd_bm_write_hinted(struct drbd_device *device) __must_hold(local);
1298 extern int drbd_bm_write_lazy(struct drbd_device *device, unsigned upper_idx) __must_hold(local);
1299 extern int drbd_bm_write_all(struct drbd_device *device,
1300 struct drbd_peer_device *peer_device) __must_hold(local);
1301 extern int drbd_bm_write_copy_pages(struct drbd_device *device,
1302 struct drbd_peer_device *peer_device) __must_hold(local);
1303 extern size_t drbd_bm_words(struct drbd_device *device);
1304 extern unsigned long drbd_bm_bits(struct drbd_device *device);
1305 extern sector_t drbd_bm_capacity(struct drbd_device *device);
1306
1307 #define DRBD_END_OF_BITMAP (~(unsigned long)0)
1308 extern unsigned long drbd_bm_find_next(struct drbd_device *device, unsigned long bm_fo);
1309 /* bm_find_next variants for use while you hold drbd_bm_lock() */
1310 extern unsigned long _drbd_bm_find_next(struct drbd_device *device, unsigned long bm_fo);
1311 extern unsigned long _drbd_bm_find_next_zero(struct drbd_device *device, unsigned long bm_fo);
1312 extern unsigned long _drbd_bm_total_weight(struct drbd_device *device);
1313 extern unsigned long drbd_bm_total_weight(struct drbd_device *device);
1314 /* for receive_bitmap */
1315 extern void drbd_bm_merge_lel(struct drbd_device *device, size_t offset,
1316 size_t number, unsigned long *buffer);
1317 /* for _drbd_send_bitmap */
1318 extern void drbd_bm_get_lel(struct drbd_device *device, size_t offset,
1319 size_t number, unsigned long *buffer);
1320
1321 extern void drbd_bm_lock(struct drbd_device *device, char *why, enum bm_flag flags);
1322 extern void drbd_bm_unlock(struct drbd_device *device);
1323 /* drbd_main.c */
1324
1325 extern struct kmem_cache *drbd_request_cache;
1326 extern struct kmem_cache *drbd_ee_cache; /* peer requests */
1327 extern struct kmem_cache *drbd_bm_ext_cache; /* bitmap extents */
1328 extern struct kmem_cache *drbd_al_ext_cache; /* activity log extents */
1329 extern mempool_t drbd_request_mempool;
1330 extern mempool_t drbd_ee_mempool;
1331
1332 /* drbd's page pool, used to buffer data received from the peer,
1333 * or data requested by the peer.
1334 *
1335 * This does not have an emergency reserve.
1336 *
1337 * When allocating from this pool, it first takes pages from the pool.
1338 * Only if the pool is depleted will try to allocate from the system.
1339 *
1340 * The assumption is that pages taken from this pool will be processed,
1341 * and given back, "quickly", and then can be recycled, so we can avoid
1342 * frequent calls to alloc_page(), and still will be able to make progress even
1343 * under memory pressure.
1344 */
1345 extern struct page *drbd_pp_pool;
1346 extern spinlock_t drbd_pp_lock;
1347 extern int drbd_pp_vacant;
1348 extern wait_queue_head_t drbd_pp_wait;
1349
1350 /* We also need a standard (emergency-reserve backed) page pool
1351 * for meta data IO (activity log, bitmap).
1352 * We can keep it global, as long as it is used as "N pages at a time".
1353 * 128 should be plenty, currently we probably can get away with as few as 1.
1354 */
1355 #define DRBD_MIN_POOL_PAGES 128
1356 extern mempool_t drbd_md_io_page_pool;
1357
1358 /* We also need to make sure we get a bio
1359 * when we need it for housekeeping purposes */
1360 extern struct bio_set drbd_md_io_bio_set;
1361
1362 /* And a bio_set for cloning */
1363 extern struct bio_set drbd_io_bio_set;
1364
1365 extern struct mutex resources_mutex;
1366
1367 extern int conn_lowest_minor(struct drbd_connection *connection);
1368 extern enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsigned int minor);
1369 extern void drbd_destroy_device(struct kref *kref);
1370 extern void drbd_delete_device(struct drbd_device *device);
1371
1372 extern struct drbd_resource *drbd_create_resource(const char *name);
1373 extern void drbd_free_resource(struct drbd_resource *resource);
1374
1375 extern int set_resource_options(struct drbd_resource *resource, struct res_opts *res_opts);
1376 extern struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts);
1377 extern void drbd_destroy_connection(struct kref *kref);
1378 extern struct drbd_connection *conn_get_by_addrs(void *my_addr, int my_addr_len,
1379 void *peer_addr, int peer_addr_len);
1380 extern struct drbd_resource *drbd_find_resource(const char *name);
1381 extern void drbd_destroy_resource(struct kref *kref);
1382 extern void conn_free_crypto(struct drbd_connection *connection);
1383
1384 /* drbd_req */
1385 extern void do_submit(struct work_struct *ws);
1386 extern void __drbd_make_request(struct drbd_device *, struct bio *);
1387 void drbd_submit_bio(struct bio *bio);
1388
1389 /* drbd_nl.c */
1390
1391 extern struct mutex notification_mutex;
1392
1393 extern void drbd_suspend_io(struct drbd_device *device);
1394 extern void drbd_resume_io(struct drbd_device *device);
1395 extern char *ppsize(char *buf, unsigned long long size);
1396 extern sector_t drbd_new_dev_size(struct drbd_device *, struct drbd_backing_dev *, sector_t, int);
1397 enum determine_dev_size {
1398 DS_ERROR_SHRINK = -3,
1399 DS_ERROR_SPACE_MD = -2,
1400 DS_ERROR = -1,
1401 DS_UNCHANGED = 0,
1402 DS_SHRUNK = 1,
1403 DS_GREW = 2,
1404 DS_GREW_FROM_ZERO = 3,
1405 };
1406 extern enum determine_dev_size
1407 drbd_determine_dev_size(struct drbd_device *, enum dds_flags, struct resize_parms *) __must_hold(local);
1408 extern void resync_after_online_grow(struct drbd_device *);
1409 extern void drbd_reconsider_queue_parameters(struct drbd_device *device,
1410 struct drbd_backing_dev *bdev, struct o_qlim *o);
1411 extern enum drbd_state_rv drbd_set_role(struct drbd_device *device,
1412 enum drbd_role new_role,
1413 int force);
1414 extern bool conn_try_outdate_peer(struct drbd_connection *connection);
1415 extern void conn_try_outdate_peer_async(struct drbd_connection *connection);
1416 extern enum drbd_peer_state conn_khelper(struct drbd_connection *connection, char *cmd);
1417 extern int drbd_khelper(struct drbd_device *device, char *cmd);
1418
1419 /* drbd_worker.c */
1420 /* bi_end_io handlers */
1421 extern void drbd_md_endio(struct bio *bio);
1422 extern void drbd_peer_request_endio(struct bio *bio);
1423 extern void drbd_request_endio(struct bio *bio);
1424 extern int drbd_worker(struct drbd_thread *thi);
1425 enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int o_minor);
1426 void drbd_resync_after_changed(struct drbd_device *device);
1427 extern void drbd_start_resync(struct drbd_device *device, enum drbd_conns side);
1428 extern void resume_next_sg(struct drbd_device *device);
1429 extern void suspend_other_sg(struct drbd_device *device);
1430 extern int drbd_resync_finished(struct drbd_peer_device *peer_device);
1431 /* maybe rather drbd_main.c ? */
1432 extern void *drbd_md_get_buffer(struct drbd_device *device, const char *intent);
1433 extern void drbd_md_put_buffer(struct drbd_device *device);
1434 extern int drbd_md_sync_page_io(struct drbd_device *device,
1435 struct drbd_backing_dev *bdev, sector_t sector, enum req_op op);
1436 extern void drbd_ov_out_of_sync_found(struct drbd_peer_device *peer_device,
1437 sector_t sector, int size);
1438 extern void wait_until_done_or_force_detached(struct drbd_device *device,
1439 struct drbd_backing_dev *bdev, unsigned int *done);
1440 extern void drbd_rs_controller_reset(struct drbd_peer_device *peer_device);
1441
ov_out_of_sync_print(struct drbd_peer_device * peer_device)1442 static inline void ov_out_of_sync_print(struct drbd_peer_device *peer_device)
1443 {
1444 struct drbd_device *device = peer_device->device;
1445
1446 if (device->ov_last_oos_size) {
1447 drbd_err(peer_device, "Out of sync: start=%llu, size=%lu (sectors)\n",
1448 (unsigned long long)device->ov_last_oos_start,
1449 (unsigned long)device->ov_last_oos_size);
1450 }
1451 device->ov_last_oos_size = 0;
1452 }
1453
1454
1455 extern void drbd_csum_bio(struct crypto_shash *, struct bio *, void *);
1456 extern void drbd_csum_ee(struct crypto_shash *, struct drbd_peer_request *,
1457 void *);
1458 /* worker callbacks */
1459 extern int w_e_end_data_req(struct drbd_work *, int);
1460 extern int w_e_end_rsdata_req(struct drbd_work *, int);
1461 extern int w_e_end_csum_rs_req(struct drbd_work *, int);
1462 extern int w_e_end_ov_reply(struct drbd_work *, int);
1463 extern int w_e_end_ov_req(struct drbd_work *, int);
1464 extern int w_ov_finished(struct drbd_work *, int);
1465 extern int w_resync_timer(struct drbd_work *, int);
1466 extern int w_send_write_hint(struct drbd_work *, int);
1467 extern int w_send_dblock(struct drbd_work *, int);
1468 extern int w_send_read_req(struct drbd_work *, int);
1469 extern int w_restart_disk_io(struct drbd_work *, int);
1470 extern int w_send_out_of_sync(struct drbd_work *, int);
1471
1472 extern void resync_timer_fn(struct timer_list *t);
1473 extern void start_resync_timer_fn(struct timer_list *t);
1474
1475 extern void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req);
1476
1477 /* drbd_receiver.c */
1478 extern int drbd_issue_discard_or_zero_out(struct drbd_device *device,
1479 sector_t start, unsigned int nr_sectors, int flags);
1480 extern int drbd_receiver(struct drbd_thread *thi);
1481 extern int drbd_ack_receiver(struct drbd_thread *thi);
1482 extern void drbd_send_acks_wf(struct work_struct *ws);
1483 extern bool drbd_rs_c_min_rate_throttle(struct drbd_device *device);
1484 extern bool drbd_rs_should_slow_down(struct drbd_peer_device *peer_device, sector_t sector,
1485 bool throttle_if_app_is_waiting);
1486 extern int drbd_submit_peer_request(struct drbd_peer_request *peer_req);
1487 extern int drbd_free_peer_reqs(struct drbd_device *, struct list_head *);
1488 extern struct drbd_peer_request *drbd_alloc_peer_req(struct drbd_peer_device *, u64,
1489 sector_t, unsigned int,
1490 unsigned int,
1491 gfp_t) __must_hold(local);
1492 extern void __drbd_free_peer_req(struct drbd_device *, struct drbd_peer_request *,
1493 int);
1494 #define drbd_free_peer_req(m,e) __drbd_free_peer_req(m, e, 0)
1495 #define drbd_free_net_peer_req(m,e) __drbd_free_peer_req(m, e, 1)
1496 extern struct page *drbd_alloc_pages(struct drbd_peer_device *, unsigned int, bool);
1497 extern void _drbd_clear_done_ee(struct drbd_device *device, struct list_head *to_be_freed);
1498 extern int drbd_connected(struct drbd_peer_device *);
1499
1500 /* sets the number of 512 byte sectors of our virtual device */
1501 void drbd_set_my_capacity(struct drbd_device *device, sector_t size);
1502
1503 /*
1504 * used to submit our private bio
1505 */
drbd_submit_bio_noacct(struct drbd_device * device,int fault_type,struct bio * bio)1506 static inline void drbd_submit_bio_noacct(struct drbd_device *device,
1507 int fault_type, struct bio *bio)
1508 {
1509 __release(local);
1510 if (!bio->bi_bdev) {
1511 drbd_err(device, "drbd_submit_bio_noacct: bio->bi_bdev == NULL\n");
1512 bio->bi_status = BLK_STS_IOERR;
1513 bio_endio(bio);
1514 return;
1515 }
1516
1517 if (drbd_insert_fault(device, fault_type))
1518 bio_io_error(bio);
1519 else
1520 submit_bio_noacct(bio);
1521 }
1522
1523 void drbd_bump_write_ordering(struct drbd_resource *resource, struct drbd_backing_dev *bdev,
1524 enum write_ordering_e wo);
1525
1526 /* drbd_proc.c */
1527 extern struct proc_dir_entry *drbd_proc;
1528 int drbd_seq_show(struct seq_file *seq, void *v);
1529
1530 /* drbd_actlog.c */
1531 extern bool drbd_al_begin_io_prepare(struct drbd_device *device, struct drbd_interval *i);
1532 extern int drbd_al_begin_io_nonblock(struct drbd_device *device, struct drbd_interval *i);
1533 extern void drbd_al_begin_io_commit(struct drbd_device *device);
1534 extern bool drbd_al_begin_io_fastpath(struct drbd_device *device, struct drbd_interval *i);
1535 extern void drbd_al_begin_io(struct drbd_device *device, struct drbd_interval *i);
1536 extern void drbd_al_complete_io(struct drbd_device *device, struct drbd_interval *i);
1537 extern void drbd_rs_complete_io(struct drbd_device *device, sector_t sector);
1538 extern int drbd_rs_begin_io(struct drbd_device *device, sector_t sector);
1539 extern int drbd_try_rs_begin_io(struct drbd_peer_device *peer_device, sector_t sector);
1540 extern void drbd_rs_cancel_all(struct drbd_device *device);
1541 extern int drbd_rs_del_all(struct drbd_device *device);
1542 extern void drbd_rs_failed_io(struct drbd_peer_device *peer_device,
1543 sector_t sector, int size);
1544 extern void drbd_advance_rs_marks(struct drbd_peer_device *peer_device, unsigned long still_to_go);
1545
1546 enum update_sync_bits_mode { RECORD_RS_FAILED, SET_OUT_OF_SYNC, SET_IN_SYNC };
1547 extern int __drbd_change_sync(struct drbd_peer_device *peer_device, sector_t sector, int size,
1548 enum update_sync_bits_mode mode);
1549 #define drbd_set_in_sync(peer_device, sector, size) \
1550 __drbd_change_sync(peer_device, sector, size, SET_IN_SYNC)
1551 #define drbd_set_out_of_sync(peer_device, sector, size) \
1552 __drbd_change_sync(peer_device, sector, size, SET_OUT_OF_SYNC)
1553 #define drbd_rs_failed_io(peer_device, sector, size) \
1554 __drbd_change_sync(peer_device, sector, size, RECORD_RS_FAILED)
1555 extern void drbd_al_shrink(struct drbd_device *device);
1556 extern int drbd_al_initialize(struct drbd_device *, void *);
1557
1558 /* drbd_nl.c */
1559 /* state info broadcast */
1560 struct sib_info {
1561 enum drbd_state_info_bcast_reason sib_reason;
1562 union {
1563 struct {
1564 char *helper_name;
1565 unsigned helper_exit_code;
1566 };
1567 struct {
1568 union drbd_state os;
1569 union drbd_state ns;
1570 };
1571 };
1572 };
1573 void drbd_bcast_event(struct drbd_device *device, const struct sib_info *sib);
1574
1575 extern int notify_resource_state(struct sk_buff *,
1576 unsigned int,
1577 struct drbd_resource *,
1578 struct resource_info *,
1579 enum drbd_notification_type);
1580 extern int notify_device_state(struct sk_buff *,
1581 unsigned int,
1582 struct drbd_device *,
1583 struct device_info *,
1584 enum drbd_notification_type);
1585 extern int notify_connection_state(struct sk_buff *,
1586 unsigned int,
1587 struct drbd_connection *,
1588 struct connection_info *,
1589 enum drbd_notification_type);
1590 extern int notify_peer_device_state(struct sk_buff *,
1591 unsigned int,
1592 struct drbd_peer_device *,
1593 struct peer_device_info *,
1594 enum drbd_notification_type);
1595 extern void notify_helper(enum drbd_notification_type, struct drbd_device *,
1596 struct drbd_connection *, const char *, int);
1597
1598 /*
1599 * inline helper functions
1600 *************************/
1601
1602 /* see also page_chain_add and friends in drbd_receiver.c */
page_chain_next(struct page * page)1603 static inline struct page *page_chain_next(struct page *page)
1604 {
1605 return (struct page *)page_private(page);
1606 }
1607 #define page_chain_for_each(page) \
1608 for (; page && ({ prefetch(page_chain_next(page)); 1; }); \
1609 page = page_chain_next(page))
1610 #define page_chain_for_each_safe(page, n) \
1611 for (; page && ({ n = page_chain_next(page); 1; }); page = n)
1612
1613
drbd_peer_req_has_active_page(struct drbd_peer_request * peer_req)1614 static inline int drbd_peer_req_has_active_page(struct drbd_peer_request *peer_req)
1615 {
1616 struct page *page = peer_req->pages;
1617 page_chain_for_each(page) {
1618 if (page_count(page) > 1)
1619 return 1;
1620 }
1621 return 0;
1622 }
1623
drbd_read_state(struct drbd_device * device)1624 static inline union drbd_state drbd_read_state(struct drbd_device *device)
1625 {
1626 struct drbd_resource *resource = device->resource;
1627 union drbd_state rv;
1628
1629 rv.i = device->state.i;
1630 rv.susp = resource->susp;
1631 rv.susp_nod = resource->susp_nod;
1632 rv.susp_fen = resource->susp_fen;
1633
1634 return rv;
1635 }
1636
1637 enum drbd_force_detach_flags {
1638 DRBD_READ_ERROR,
1639 DRBD_WRITE_ERROR,
1640 DRBD_META_IO_ERROR,
1641 DRBD_FORCE_DETACH,
1642 };
1643
1644 #define __drbd_chk_io_error(m,f) __drbd_chk_io_error_(m,f, __func__)
__drbd_chk_io_error_(struct drbd_device * device,enum drbd_force_detach_flags df,const char * where)1645 static inline void __drbd_chk_io_error_(struct drbd_device *device,
1646 enum drbd_force_detach_flags df,
1647 const char *where)
1648 {
1649 enum drbd_io_error_p ep;
1650
1651 rcu_read_lock();
1652 ep = rcu_dereference(device->ldev->disk_conf)->on_io_error;
1653 rcu_read_unlock();
1654 switch (ep) {
1655 case EP_PASS_ON: /* FIXME would this be better named "Ignore"? */
1656 if (df == DRBD_READ_ERROR || df == DRBD_WRITE_ERROR) {
1657 if (drbd_ratelimit())
1658 drbd_err(device, "Local IO failed in %s.\n", where);
1659 if (device->state.disk > D_INCONSISTENT)
1660 _drbd_set_state(_NS(device, disk, D_INCONSISTENT), CS_HARD, NULL);
1661 break;
1662 }
1663 fallthrough; /* for DRBD_META_IO_ERROR or DRBD_FORCE_DETACH */
1664 case EP_DETACH:
1665 case EP_CALL_HELPER:
1666 /* Remember whether we saw a READ or WRITE error.
1667 *
1668 * Recovery of the affected area for WRITE failure is covered
1669 * by the activity log.
1670 * READ errors may fall outside that area though. Certain READ
1671 * errors can be "healed" by writing good data to the affected
1672 * blocks, which triggers block re-allocation in lower layers.
1673 *
1674 * If we can not write the bitmap after a READ error,
1675 * we may need to trigger a full sync (see w_go_diskless()).
1676 *
1677 * Force-detach is not really an IO error, but rather a
1678 * desperate measure to try to deal with a completely
1679 * unresponsive lower level IO stack.
1680 * Still it should be treated as a WRITE error.
1681 *
1682 * Meta IO error is always WRITE error:
1683 * we read meta data only once during attach,
1684 * which will fail in case of errors.
1685 */
1686 set_bit(WAS_IO_ERROR, &device->flags);
1687 if (df == DRBD_READ_ERROR)
1688 set_bit(WAS_READ_ERROR, &device->flags);
1689 if (df == DRBD_FORCE_DETACH)
1690 set_bit(FORCE_DETACH, &device->flags);
1691 if (device->state.disk > D_FAILED) {
1692 _drbd_set_state(_NS(device, disk, D_FAILED), CS_HARD, NULL);
1693 drbd_err(device,
1694 "Local IO failed in %s. Detaching...\n", where);
1695 }
1696 break;
1697 }
1698 }
1699
1700 /**
1701 * drbd_chk_io_error: Handle the on_io_error setting, should be called from all io completion handlers
1702 * @device: DRBD device.
1703 * @error: Error code passed to the IO completion callback
1704 * @forcedetach: Force detach. I.e. the error happened while accessing the meta data
1705 *
1706 * See also drbd_main.c:after_state_ch() if (os.disk > D_FAILED && ns.disk == D_FAILED)
1707 */
1708 #define drbd_chk_io_error(m,e,f) drbd_chk_io_error_(m,e,f, __func__)
drbd_chk_io_error_(struct drbd_device * device,int error,enum drbd_force_detach_flags forcedetach,const char * where)1709 static inline void drbd_chk_io_error_(struct drbd_device *device,
1710 int error, enum drbd_force_detach_flags forcedetach, const char *where)
1711 {
1712 if (error) {
1713 unsigned long flags;
1714 spin_lock_irqsave(&device->resource->req_lock, flags);
1715 __drbd_chk_io_error_(device, forcedetach, where);
1716 spin_unlock_irqrestore(&device->resource->req_lock, flags);
1717 }
1718 }
1719
1720
1721 /**
1722 * drbd_md_first_sector() - Returns the first sector number of the meta data area
1723 * @bdev: Meta data block device.
1724 *
1725 * BTW, for internal meta data, this happens to be the maximum capacity
1726 * we could agree upon with our peer node.
1727 */
drbd_md_first_sector(struct drbd_backing_dev * bdev)1728 static inline sector_t drbd_md_first_sector(struct drbd_backing_dev *bdev)
1729 {
1730 switch (bdev->md.meta_dev_idx) {
1731 case DRBD_MD_INDEX_INTERNAL:
1732 case DRBD_MD_INDEX_FLEX_INT:
1733 return bdev->md.md_offset + bdev->md.bm_offset;
1734 case DRBD_MD_INDEX_FLEX_EXT:
1735 default:
1736 return bdev->md.md_offset;
1737 }
1738 }
1739
1740 /**
1741 * drbd_md_last_sector() - Return the last sector number of the meta data area
1742 * @bdev: Meta data block device.
1743 */
drbd_md_last_sector(struct drbd_backing_dev * bdev)1744 static inline sector_t drbd_md_last_sector(struct drbd_backing_dev *bdev)
1745 {
1746 switch (bdev->md.meta_dev_idx) {
1747 case DRBD_MD_INDEX_INTERNAL:
1748 case DRBD_MD_INDEX_FLEX_INT:
1749 return bdev->md.md_offset + MD_4kB_SECT -1;
1750 case DRBD_MD_INDEX_FLEX_EXT:
1751 default:
1752 return bdev->md.md_offset + bdev->md.md_size_sect -1;
1753 }
1754 }
1755
1756 /* Returns the number of 512 byte sectors of the device */
drbd_get_capacity(struct block_device * bdev)1757 static inline sector_t drbd_get_capacity(struct block_device *bdev)
1758 {
1759 return bdev ? bdev_nr_sectors(bdev) : 0;
1760 }
1761
1762 /**
1763 * drbd_get_max_capacity() - Returns the capacity we announce to out peer
1764 * @bdev: Meta data block device.
1765 *
1766 * returns the capacity we announce to out peer. we clip ourselves at the
1767 * various MAX_SECTORS, because if we don't, current implementation will
1768 * oops sooner or later
1769 */
drbd_get_max_capacity(struct drbd_backing_dev * bdev)1770 static inline sector_t drbd_get_max_capacity(struct drbd_backing_dev *bdev)
1771 {
1772 sector_t s;
1773
1774 switch (bdev->md.meta_dev_idx) {
1775 case DRBD_MD_INDEX_INTERNAL:
1776 case DRBD_MD_INDEX_FLEX_INT:
1777 s = drbd_get_capacity(bdev->backing_bdev)
1778 ? min_t(sector_t, DRBD_MAX_SECTORS_FLEX,
1779 drbd_md_first_sector(bdev))
1780 : 0;
1781 break;
1782 case DRBD_MD_INDEX_FLEX_EXT:
1783 s = min_t(sector_t, DRBD_MAX_SECTORS_FLEX,
1784 drbd_get_capacity(bdev->backing_bdev));
1785 /* clip at maximum size the meta device can support */
1786 s = min_t(sector_t, s,
1787 BM_EXT_TO_SECT(bdev->md.md_size_sect
1788 - bdev->md.bm_offset));
1789 break;
1790 default:
1791 s = min_t(sector_t, DRBD_MAX_SECTORS,
1792 drbd_get_capacity(bdev->backing_bdev));
1793 }
1794 return s;
1795 }
1796
1797 /**
1798 * drbd_md_ss() - Return the sector number of our meta data super block
1799 * @bdev: Meta data block device.
1800 */
drbd_md_ss(struct drbd_backing_dev * bdev)1801 static inline sector_t drbd_md_ss(struct drbd_backing_dev *bdev)
1802 {
1803 const int meta_dev_idx = bdev->md.meta_dev_idx;
1804
1805 if (meta_dev_idx == DRBD_MD_INDEX_FLEX_EXT)
1806 return 0;
1807
1808 /* Since drbd08, internal meta data is always "flexible".
1809 * position: last 4k aligned block of 4k size */
1810 if (meta_dev_idx == DRBD_MD_INDEX_INTERNAL ||
1811 meta_dev_idx == DRBD_MD_INDEX_FLEX_INT)
1812 return (drbd_get_capacity(bdev->backing_bdev) & ~7ULL) - 8;
1813
1814 /* external, some index; this is the old fixed size layout */
1815 return MD_128MB_SECT * bdev->md.meta_dev_idx;
1816 }
1817
1818 static inline void
drbd_queue_work(struct drbd_work_queue * q,struct drbd_work * w)1819 drbd_queue_work(struct drbd_work_queue *q, struct drbd_work *w)
1820 {
1821 unsigned long flags;
1822 spin_lock_irqsave(&q->q_lock, flags);
1823 list_add_tail(&w->list, &q->q);
1824 spin_unlock_irqrestore(&q->q_lock, flags);
1825 wake_up(&q->q_wait);
1826 }
1827
1828 static inline void
drbd_queue_work_if_unqueued(struct drbd_work_queue * q,struct drbd_work * w)1829 drbd_queue_work_if_unqueued(struct drbd_work_queue *q, struct drbd_work *w)
1830 {
1831 unsigned long flags;
1832 spin_lock_irqsave(&q->q_lock, flags);
1833 if (list_empty_careful(&w->list))
1834 list_add_tail(&w->list, &q->q);
1835 spin_unlock_irqrestore(&q->q_lock, flags);
1836 wake_up(&q->q_wait);
1837 }
1838
1839 static inline void
drbd_device_post_work(struct drbd_device * device,int work_bit)1840 drbd_device_post_work(struct drbd_device *device, int work_bit)
1841 {
1842 if (!test_and_set_bit(work_bit, &device->flags)) {
1843 struct drbd_connection *connection =
1844 first_peer_device(device)->connection;
1845 struct drbd_work_queue *q = &connection->sender_work;
1846 if (!test_and_set_bit(DEVICE_WORK_PENDING, &connection->flags))
1847 wake_up(&q->q_wait);
1848 }
1849 }
1850
1851 extern void drbd_flush_workqueue(struct drbd_work_queue *work_queue);
1852
1853 /* To get the ack_receiver out of the blocking network stack,
1854 * so it can change its sk_rcvtimeo from idle- to ping-timeout,
1855 * and send a ping, we need to send a signal.
1856 * Which signal we send is irrelevant. */
wake_ack_receiver(struct drbd_connection * connection)1857 static inline void wake_ack_receiver(struct drbd_connection *connection)
1858 {
1859 struct task_struct *task = connection->ack_receiver.task;
1860 if (task && get_t_state(&connection->ack_receiver) == RUNNING)
1861 send_sig(SIGXCPU, task, 1);
1862 }
1863
request_ping(struct drbd_connection * connection)1864 static inline void request_ping(struct drbd_connection *connection)
1865 {
1866 set_bit(SEND_PING, &connection->flags);
1867 wake_ack_receiver(connection);
1868 }
1869
1870 extern void *conn_prepare_command(struct drbd_connection *, struct drbd_socket *);
1871 extern void *drbd_prepare_command(struct drbd_peer_device *, struct drbd_socket *);
1872 extern int conn_send_command(struct drbd_connection *, struct drbd_socket *,
1873 enum drbd_packet, unsigned int, void *,
1874 unsigned int);
1875 extern int drbd_send_command(struct drbd_peer_device *, struct drbd_socket *,
1876 enum drbd_packet, unsigned int, void *,
1877 unsigned int);
1878
1879 extern int drbd_send_ping(struct drbd_connection *connection);
1880 extern int drbd_send_ping_ack(struct drbd_connection *connection);
1881 extern int drbd_send_state_req(struct drbd_peer_device *, union drbd_state, union drbd_state);
1882 extern int conn_send_state_req(struct drbd_connection *, union drbd_state, union drbd_state);
1883
drbd_thread_stop(struct drbd_thread * thi)1884 static inline void drbd_thread_stop(struct drbd_thread *thi)
1885 {
1886 _drbd_thread_stop(thi, false, true);
1887 }
1888
drbd_thread_stop_nowait(struct drbd_thread * thi)1889 static inline void drbd_thread_stop_nowait(struct drbd_thread *thi)
1890 {
1891 _drbd_thread_stop(thi, false, false);
1892 }
1893
drbd_thread_restart_nowait(struct drbd_thread * thi)1894 static inline void drbd_thread_restart_nowait(struct drbd_thread *thi)
1895 {
1896 _drbd_thread_stop(thi, true, false);
1897 }
1898
1899 /* counts how many answer packets packets we expect from our peer,
1900 * for either explicit application requests,
1901 * or implicit barrier packets as necessary.
1902 * increased:
1903 * w_send_barrier
1904 * _req_mod(req, QUEUE_FOR_NET_WRITE or QUEUE_FOR_NET_READ);
1905 * it is much easier and equally valid to count what we queue for the
1906 * worker, even before it actually was queued or send.
1907 * (drbd_make_request_common; recovery path on read io-error)
1908 * decreased:
1909 * got_BarrierAck (respective tl_clear, tl_clear_barrier)
1910 * _req_mod(req, DATA_RECEIVED)
1911 * [from receive_DataReply]
1912 * _req_mod(req, WRITE_ACKED_BY_PEER or RECV_ACKED_BY_PEER or NEG_ACKED)
1913 * [from got_BlockAck (P_WRITE_ACK, P_RECV_ACK)]
1914 * for some reason it is NOT decreased in got_NegAck,
1915 * but in the resulting cleanup code from report_params.
1916 * we should try to remember the reason for that...
1917 * _req_mod(req, SEND_FAILED or SEND_CANCELED)
1918 * _req_mod(req, CONNECTION_LOST_WHILE_PENDING)
1919 * [from tl_clear_barrier]
1920 */
inc_ap_pending(struct drbd_device * device)1921 static inline void inc_ap_pending(struct drbd_device *device)
1922 {
1923 atomic_inc(&device->ap_pending_cnt);
1924 }
1925
1926 #define dec_ap_pending(device) ((void)expect((device), __dec_ap_pending(device) >= 0))
__dec_ap_pending(struct drbd_device * device)1927 static inline int __dec_ap_pending(struct drbd_device *device)
1928 {
1929 int ap_pending_cnt = atomic_dec_return(&device->ap_pending_cnt);
1930
1931 if (ap_pending_cnt == 0)
1932 wake_up(&device->misc_wait);
1933 return ap_pending_cnt;
1934 }
1935
1936 /* counts how many resync-related answers we still expect from the peer
1937 * increase decrease
1938 * C_SYNC_TARGET sends P_RS_DATA_REQUEST (and expects P_RS_DATA_REPLY)
1939 * C_SYNC_SOURCE sends P_RS_DATA_REPLY (and expects P_WRITE_ACK with ID_SYNCER)
1940 * (or P_NEG_ACK with ID_SYNCER)
1941 */
inc_rs_pending(struct drbd_peer_device * peer_device)1942 static inline void inc_rs_pending(struct drbd_peer_device *peer_device)
1943 {
1944 atomic_inc(&peer_device->device->rs_pending_cnt);
1945 }
1946
1947 #define dec_rs_pending(peer_device) \
1948 ((void)expect((peer_device), __dec_rs_pending(peer_device) >= 0))
__dec_rs_pending(struct drbd_peer_device * peer_device)1949 static inline int __dec_rs_pending(struct drbd_peer_device *peer_device)
1950 {
1951 return atomic_dec_return(&peer_device->device->rs_pending_cnt);
1952 }
1953
1954 /* counts how many answers we still need to send to the peer.
1955 * increased on
1956 * receive_Data unless protocol A;
1957 * we need to send a P_RECV_ACK (proto B)
1958 * or P_WRITE_ACK (proto C)
1959 * receive_RSDataReply (recv_resync_read) we need to send a P_WRITE_ACK
1960 * receive_DataRequest (receive_RSDataRequest) we need to send back P_DATA
1961 * receive_Barrier_* we need to send a P_BARRIER_ACK
1962 */
inc_unacked(struct drbd_device * device)1963 static inline void inc_unacked(struct drbd_device *device)
1964 {
1965 atomic_inc(&device->unacked_cnt);
1966 }
1967
1968 #define dec_unacked(device) ((void)expect(device, __dec_unacked(device) >= 0))
__dec_unacked(struct drbd_device * device)1969 static inline int __dec_unacked(struct drbd_device *device)
1970 {
1971 return atomic_dec_return(&device->unacked_cnt);
1972 }
1973
1974 #define sub_unacked(device, n) ((void)expect(device, __sub_unacked(device) >= 0))
__sub_unacked(struct drbd_device * device,int n)1975 static inline int __sub_unacked(struct drbd_device *device, int n)
1976 {
1977 return atomic_sub_return(n, &device->unacked_cnt);
1978 }
1979
is_sync_target_state(enum drbd_conns connection_state)1980 static inline bool is_sync_target_state(enum drbd_conns connection_state)
1981 {
1982 return connection_state == C_SYNC_TARGET ||
1983 connection_state == C_PAUSED_SYNC_T;
1984 }
1985
is_sync_source_state(enum drbd_conns connection_state)1986 static inline bool is_sync_source_state(enum drbd_conns connection_state)
1987 {
1988 return connection_state == C_SYNC_SOURCE ||
1989 connection_state == C_PAUSED_SYNC_S;
1990 }
1991
is_sync_state(enum drbd_conns connection_state)1992 static inline bool is_sync_state(enum drbd_conns connection_state)
1993 {
1994 return is_sync_source_state(connection_state) ||
1995 is_sync_target_state(connection_state);
1996 }
1997
1998 /**
1999 * get_ldev() - Increase the ref count on device->ldev. Returns 0 if there is no ldev
2000 * @_device: DRBD device.
2001 * @_min_state: Minimum device state required for success.
2002 *
2003 * You have to call put_ldev() when finished working with device->ldev.
2004 */
2005 #define get_ldev_if_state(_device, _min_state) \
2006 (_get_ldev_if_state((_device), (_min_state)) ? \
2007 ({ __acquire(x); true; }) : false)
2008 #define get_ldev(_device) get_ldev_if_state(_device, D_INCONSISTENT)
2009
put_ldev(struct drbd_device * device)2010 static inline void put_ldev(struct drbd_device *device)
2011 {
2012 enum drbd_disk_state disk_state = device->state.disk;
2013 /* We must check the state *before* the atomic_dec becomes visible,
2014 * or we have a theoretical race where someone hitting zero,
2015 * while state still D_FAILED, will then see D_DISKLESS in the
2016 * condition below and calling into destroy, where he must not, yet. */
2017 int i = atomic_dec_return(&device->local_cnt);
2018
2019 /* This may be called from some endio handler,
2020 * so we must not sleep here. */
2021
2022 __release(local);
2023 D_ASSERT(device, i >= 0);
2024 if (i == 0) {
2025 if (disk_state == D_DISKLESS)
2026 /* even internal references gone, safe to destroy */
2027 drbd_device_post_work(device, DESTROY_DISK);
2028 if (disk_state == D_FAILED)
2029 /* all application IO references gone. */
2030 if (!test_and_set_bit(GOING_DISKLESS, &device->flags))
2031 drbd_device_post_work(device, GO_DISKLESS);
2032 wake_up(&device->misc_wait);
2033 }
2034 }
2035
2036 #ifndef __CHECKER__
_get_ldev_if_state(struct drbd_device * device,enum drbd_disk_state mins)2037 static inline int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_state mins)
2038 {
2039 int io_allowed;
2040
2041 /* never get a reference while D_DISKLESS */
2042 if (device->state.disk == D_DISKLESS)
2043 return 0;
2044
2045 atomic_inc(&device->local_cnt);
2046 io_allowed = (device->state.disk >= mins);
2047 if (!io_allowed)
2048 put_ldev(device);
2049 return io_allowed;
2050 }
2051 #else
2052 extern int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_state mins);
2053 #endif
2054
2055 /* this throttles on-the-fly application requests
2056 * according to max_buffers settings;
2057 * maybe re-implement using semaphores? */
drbd_get_max_buffers(struct drbd_device * device)2058 static inline int drbd_get_max_buffers(struct drbd_device *device)
2059 {
2060 struct net_conf *nc;
2061 int mxb;
2062
2063 rcu_read_lock();
2064 nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
2065 mxb = nc ? nc->max_buffers : 1000000; /* arbitrary limit on open requests */
2066 rcu_read_unlock();
2067
2068 return mxb;
2069 }
2070
drbd_state_is_stable(struct drbd_device * device)2071 static inline int drbd_state_is_stable(struct drbd_device *device)
2072 {
2073 union drbd_dev_state s = device->state;
2074
2075 /* DO NOT add a default clause, we want the compiler to warn us
2076 * for any newly introduced state we may have forgotten to add here */
2077
2078 switch ((enum drbd_conns)s.conn) {
2079 /* new io only accepted when there is no connection, ... */
2080 case C_STANDALONE:
2081 case C_WF_CONNECTION:
2082 /* ... or there is a well established connection. */
2083 case C_CONNECTED:
2084 case C_SYNC_SOURCE:
2085 case C_SYNC_TARGET:
2086 case C_VERIFY_S:
2087 case C_VERIFY_T:
2088 case C_PAUSED_SYNC_S:
2089 case C_PAUSED_SYNC_T:
2090 case C_AHEAD:
2091 case C_BEHIND:
2092 /* transitional states, IO allowed */
2093 case C_DISCONNECTING:
2094 case C_UNCONNECTED:
2095 case C_TIMEOUT:
2096 case C_BROKEN_PIPE:
2097 case C_NETWORK_FAILURE:
2098 case C_PROTOCOL_ERROR:
2099 case C_TEAR_DOWN:
2100 case C_WF_REPORT_PARAMS:
2101 case C_STARTING_SYNC_S:
2102 case C_STARTING_SYNC_T:
2103 break;
2104
2105 /* Allow IO in BM exchange states with new protocols */
2106 case C_WF_BITMAP_S:
2107 if (first_peer_device(device)->connection->agreed_pro_version < 96)
2108 return 0;
2109 break;
2110
2111 /* no new io accepted in these states */
2112 case C_WF_BITMAP_T:
2113 case C_WF_SYNC_UUID:
2114 case C_MASK:
2115 /* not "stable" */
2116 return 0;
2117 }
2118
2119 switch ((enum drbd_disk_state)s.disk) {
2120 case D_DISKLESS:
2121 case D_INCONSISTENT:
2122 case D_OUTDATED:
2123 case D_CONSISTENT:
2124 case D_UP_TO_DATE:
2125 case D_FAILED:
2126 /* disk state is stable as well. */
2127 break;
2128
2129 /* no new io accepted during transitional states */
2130 case D_ATTACHING:
2131 case D_NEGOTIATING:
2132 case D_UNKNOWN:
2133 case D_MASK:
2134 /* not "stable" */
2135 return 0;
2136 }
2137
2138 return 1;
2139 }
2140
drbd_suspended(struct drbd_device * device)2141 static inline int drbd_suspended(struct drbd_device *device)
2142 {
2143 struct drbd_resource *resource = device->resource;
2144
2145 return resource->susp || resource->susp_fen || resource->susp_nod;
2146 }
2147
may_inc_ap_bio(struct drbd_device * device)2148 static inline bool may_inc_ap_bio(struct drbd_device *device)
2149 {
2150 int mxb = drbd_get_max_buffers(device);
2151
2152 if (drbd_suspended(device))
2153 return false;
2154 if (atomic_read(&device->suspend_cnt))
2155 return false;
2156
2157 /* to avoid potential deadlock or bitmap corruption,
2158 * in various places, we only allow new application io
2159 * to start during "stable" states. */
2160
2161 /* no new io accepted when attaching or detaching the disk */
2162 if (!drbd_state_is_stable(device))
2163 return false;
2164
2165 /* since some older kernels don't have atomic_add_unless,
2166 * and we are within the spinlock anyways, we have this workaround. */
2167 if (atomic_read(&device->ap_bio_cnt) > mxb)
2168 return false;
2169 if (test_bit(BITMAP_IO, &device->flags))
2170 return false;
2171 return true;
2172 }
2173
inc_ap_bio_cond(struct drbd_device * device)2174 static inline bool inc_ap_bio_cond(struct drbd_device *device)
2175 {
2176 bool rv = false;
2177
2178 spin_lock_irq(&device->resource->req_lock);
2179 rv = may_inc_ap_bio(device);
2180 if (rv)
2181 atomic_inc(&device->ap_bio_cnt);
2182 spin_unlock_irq(&device->resource->req_lock);
2183
2184 return rv;
2185 }
2186
inc_ap_bio(struct drbd_device * device)2187 static inline void inc_ap_bio(struct drbd_device *device)
2188 {
2189 /* we wait here
2190 * as long as the device is suspended
2191 * until the bitmap is no longer on the fly during connection
2192 * handshake as long as we would exceed the max_buffer limit.
2193 *
2194 * to avoid races with the reconnect code,
2195 * we need to atomic_inc within the spinlock. */
2196
2197 wait_event(device->misc_wait, inc_ap_bio_cond(device));
2198 }
2199
dec_ap_bio(struct drbd_device * device)2200 static inline void dec_ap_bio(struct drbd_device *device)
2201 {
2202 int mxb = drbd_get_max_buffers(device);
2203 int ap_bio = atomic_dec_return(&device->ap_bio_cnt);
2204
2205 D_ASSERT(device, ap_bio >= 0);
2206
2207 if (ap_bio == 0 && test_bit(BITMAP_IO, &device->flags)) {
2208 if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))
2209 drbd_queue_work(&first_peer_device(device)->
2210 connection->sender_work,
2211 &device->bm_io_work.w);
2212 }
2213
2214 /* this currently does wake_up for every dec_ap_bio!
2215 * maybe rather introduce some type of hysteresis?
2216 * e.g. (ap_bio == mxb/2 || ap_bio == 0) ? */
2217 if (ap_bio < mxb)
2218 wake_up(&device->misc_wait);
2219 }
2220
verify_can_do_stop_sector(struct drbd_device * device)2221 static inline bool verify_can_do_stop_sector(struct drbd_device *device)
2222 {
2223 return first_peer_device(device)->connection->agreed_pro_version >= 97 &&
2224 first_peer_device(device)->connection->agreed_pro_version != 100;
2225 }
2226
drbd_set_ed_uuid(struct drbd_device * device,u64 val)2227 static inline int drbd_set_ed_uuid(struct drbd_device *device, u64 val)
2228 {
2229 int changed = device->ed_uuid != val;
2230 device->ed_uuid = val;
2231 return changed;
2232 }
2233
drbd_queue_order_type(struct drbd_device * device)2234 static inline int drbd_queue_order_type(struct drbd_device *device)
2235 {
2236 /* sorry, we currently have no working implementation
2237 * of distributed TCQ stuff */
2238 #ifndef QUEUE_ORDERED_NONE
2239 #define QUEUE_ORDERED_NONE 0
2240 #endif
2241 return QUEUE_ORDERED_NONE;
2242 }
2243
first_connection(struct drbd_resource * resource)2244 static inline struct drbd_connection *first_connection(struct drbd_resource *resource)
2245 {
2246 return list_first_entry_or_null(&resource->connections,
2247 struct drbd_connection, connections);
2248 }
2249
2250 #endif
2251