1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 2001 Jens Axboe <axboe@suse.de>
4 */
5 #ifndef __LINUX_BIO_H
6 #define __LINUX_BIO_H
7
8 #include <linux/mempool.h>
9 /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */
10 #include <linux/blk_types.h>
11 #include <linux/uio.h>
12
13 #define BIO_MAX_VECS 256U
14
15 struct queue_limits;
16
bio_max_segs(unsigned int nr_segs)17 static inline unsigned int bio_max_segs(unsigned int nr_segs)
18 {
19 return min(nr_segs, BIO_MAX_VECS);
20 }
21
22 #define bio_prio(bio) (bio)->bi_ioprio
23 #define bio_set_prio(bio, prio) ((bio)->bi_ioprio = prio)
24
25 #define bio_iter_iovec(bio, iter) \
26 bvec_iter_bvec((bio)->bi_io_vec, (iter))
27
28 #define bio_iter_page(bio, iter) \
29 bvec_iter_page((bio)->bi_io_vec, (iter))
30 #define bio_iter_len(bio, iter) \
31 bvec_iter_len((bio)->bi_io_vec, (iter))
32 #define bio_iter_offset(bio, iter) \
33 bvec_iter_offset((bio)->bi_io_vec, (iter))
34
35 #define bio_page(bio) bio_iter_page((bio), (bio)->bi_iter)
36 #define bio_offset(bio) bio_iter_offset((bio), (bio)->bi_iter)
37 #define bio_iovec(bio) bio_iter_iovec((bio), (bio)->bi_iter)
38
39 #define bvec_iter_sectors(iter) ((iter).bi_size >> 9)
40 #define bvec_iter_end_sector(iter) ((iter).bi_sector + bvec_iter_sectors((iter)))
41
42 #define bio_sectors(bio) bvec_iter_sectors((bio)->bi_iter)
43 #define bio_end_sector(bio) bvec_iter_end_sector((bio)->bi_iter)
44
45 /*
46 * Return the data direction, READ or WRITE.
47 */
48 #define bio_data_dir(bio) \
49 (op_is_write(bio_op(bio)) ? WRITE : READ)
50
51 /*
52 * Check whether this bio carries any data or not. A NULL bio is allowed.
53 */
bio_has_data(struct bio * bio)54 static inline bool bio_has_data(struct bio *bio)
55 {
56 if (bio &&
57 bio->bi_iter.bi_size &&
58 bio_op(bio) != REQ_OP_DISCARD &&
59 bio_op(bio) != REQ_OP_SECURE_ERASE &&
60 bio_op(bio) != REQ_OP_WRITE_ZEROES)
61 return true;
62
63 return false;
64 }
65
bio_no_advance_iter(const struct bio * bio)66 static inline bool bio_no_advance_iter(const struct bio *bio)
67 {
68 return bio_op(bio) == REQ_OP_DISCARD ||
69 bio_op(bio) == REQ_OP_SECURE_ERASE ||
70 bio_op(bio) == REQ_OP_WRITE_ZEROES;
71 }
72
bio_data(struct bio * bio)73 static inline void *bio_data(struct bio *bio)
74 {
75 if (bio_has_data(bio))
76 return page_address(bio_page(bio)) + bio_offset(bio);
77
78 return NULL;
79 }
80
bio_next_segment(const struct bio * bio,struct bvec_iter_all * iter)81 static inline bool bio_next_segment(const struct bio *bio,
82 struct bvec_iter_all *iter)
83 {
84 if (iter->idx >= bio->bi_vcnt)
85 return false;
86
87 bvec_advance(&bio->bi_io_vec[iter->idx], iter);
88 return true;
89 }
90
91 /*
92 * drivers should _never_ use the all version - the bio may have been split
93 * before it got to the driver and the driver won't own all of it
94 */
95 #define bio_for_each_segment_all(bvl, bio, iter) \
96 for (bvl = bvec_init_iter_all(&iter); bio_next_segment((bio), &iter); )
97
bio_advance_iter(const struct bio * bio,struct bvec_iter * iter,unsigned int bytes)98 static inline void bio_advance_iter(const struct bio *bio,
99 struct bvec_iter *iter, unsigned int bytes)
100 {
101 iter->bi_sector += bytes >> 9;
102
103 if (bio_no_advance_iter(bio))
104 iter->bi_size -= bytes;
105 else
106 bvec_iter_advance(bio->bi_io_vec, iter, bytes);
107 /* TODO: It is reasonable to complete bio with error here. */
108 }
109
110 /* @bytes should be less or equal to bvec[i->bi_idx].bv_len */
bio_advance_iter_single(const struct bio * bio,struct bvec_iter * iter,unsigned int bytes)111 static inline void bio_advance_iter_single(const struct bio *bio,
112 struct bvec_iter *iter,
113 unsigned int bytes)
114 {
115 iter->bi_sector += bytes >> 9;
116
117 if (bio_no_advance_iter(bio))
118 iter->bi_size -= bytes;
119 else
120 bvec_iter_advance_single(bio->bi_io_vec, iter, bytes);
121 }
122
123 void __bio_advance(struct bio *, unsigned bytes);
124
125 /**
126 * bio_advance - increment/complete a bio by some number of bytes
127 * @bio: bio to advance
128 * @nbytes: number of bytes to complete
129 *
130 * This updates bi_sector, bi_size and bi_idx; if the number of bytes to
131 * complete doesn't align with a bvec boundary, then bv_len and bv_offset will
132 * be updated on the last bvec as well.
133 *
134 * @bio will then represent the remaining, uncompleted portion of the io.
135 */
bio_advance(struct bio * bio,unsigned int nbytes)136 static inline void bio_advance(struct bio *bio, unsigned int nbytes)
137 {
138 if (nbytes == bio->bi_iter.bi_size) {
139 bio->bi_iter.bi_size = 0;
140 return;
141 }
142 __bio_advance(bio, nbytes);
143 }
144
145 #define __bio_for_each_segment(bvl, bio, iter, start) \
146 for (iter = (start); \
147 (iter).bi_size && \
148 ((bvl = bio_iter_iovec((bio), (iter))), 1); \
149 bio_advance_iter_single((bio), &(iter), (bvl).bv_len))
150
151 #define bio_for_each_segment(bvl, bio, iter) \
152 __bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)
153
154 #define __bio_for_each_bvec(bvl, bio, iter, start) \
155 for (iter = (start); \
156 (iter).bi_size && \
157 ((bvl = mp_bvec_iter_bvec((bio)->bi_io_vec, (iter))), 1); \
158 bio_advance_iter_single((bio), &(iter), (bvl).bv_len))
159
160 /* iterate over multi-page bvec */
161 #define bio_for_each_bvec(bvl, bio, iter) \
162 __bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter)
163
164 /*
165 * Iterate over all multi-page bvecs. Drivers shouldn't use this version for the
166 * same reasons as bio_for_each_segment_all().
167 */
168 #define bio_for_each_bvec_all(bvl, bio, i) \
169 for (i = 0, bvl = bio_first_bvec_all(bio); \
170 i < (bio)->bi_vcnt; i++, bvl++)
171
172 #define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len)
173
bio_segments(struct bio * bio)174 static inline unsigned bio_segments(struct bio *bio)
175 {
176 unsigned segs = 0;
177 struct bio_vec bv;
178 struct bvec_iter iter;
179
180 /*
181 * We special case discard/write same/write zeroes, because they
182 * interpret bi_size differently:
183 */
184
185 switch (bio_op(bio)) {
186 case REQ_OP_DISCARD:
187 case REQ_OP_SECURE_ERASE:
188 case REQ_OP_WRITE_ZEROES:
189 return 0;
190 default:
191 break;
192 }
193
194 bio_for_each_segment(bv, bio, iter)
195 segs++;
196
197 return segs;
198 }
199
200 /*
201 * get a reference to a bio, so it won't disappear. the intended use is
202 * something like:
203 *
204 * bio_get(bio);
205 * submit_bio(rw, bio);
206 * if (bio->bi_flags ...)
207 * do_something
208 * bio_put(bio);
209 *
210 * without the bio_get(), it could potentially complete I/O before submit_bio
211 * returns. and then bio would be freed memory when if (bio->bi_flags ...)
212 * runs
213 */
bio_get(struct bio * bio)214 static inline void bio_get(struct bio *bio)
215 {
216 bio->bi_flags |= (1 << BIO_REFFED);
217 smp_mb__before_atomic();
218 atomic_inc(&bio->__bi_cnt);
219 }
220
bio_cnt_set(struct bio * bio,unsigned int count)221 static inline void bio_cnt_set(struct bio *bio, unsigned int count)
222 {
223 if (count != 1) {
224 bio->bi_flags |= (1 << BIO_REFFED);
225 smp_mb();
226 }
227 atomic_set(&bio->__bi_cnt, count);
228 }
229
bio_flagged(struct bio * bio,unsigned int bit)230 static inline bool bio_flagged(struct bio *bio, unsigned int bit)
231 {
232 return bio->bi_flags & (1U << bit);
233 }
234
bio_set_flag(struct bio * bio,unsigned int bit)235 static inline void bio_set_flag(struct bio *bio, unsigned int bit)
236 {
237 bio->bi_flags |= (1U << bit);
238 }
239
bio_clear_flag(struct bio * bio,unsigned int bit)240 static inline void bio_clear_flag(struct bio *bio, unsigned int bit)
241 {
242 bio->bi_flags &= ~(1U << bit);
243 }
244
bio_first_bvec_all(struct bio * bio)245 static inline struct bio_vec *bio_first_bvec_all(struct bio *bio)
246 {
247 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
248 return bio->bi_io_vec;
249 }
250
bio_first_page_all(struct bio * bio)251 static inline struct page *bio_first_page_all(struct bio *bio)
252 {
253 return bio_first_bvec_all(bio)->bv_page;
254 }
255
bio_first_folio_all(struct bio * bio)256 static inline struct folio *bio_first_folio_all(struct bio *bio)
257 {
258 return page_folio(bio_first_page_all(bio));
259 }
260
bio_last_bvec_all(struct bio * bio)261 static inline struct bio_vec *bio_last_bvec_all(struct bio *bio)
262 {
263 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
264 return &bio->bi_io_vec[bio->bi_vcnt - 1];
265 }
266
267 /**
268 * struct folio_iter - State for iterating all folios in a bio.
269 * @folio: The current folio we're iterating. NULL after the last folio.
270 * @offset: The byte offset within the current folio.
271 * @length: The number of bytes in this iteration (will not cross folio
272 * boundary).
273 */
274 struct folio_iter {
275 struct folio *folio;
276 size_t offset;
277 size_t length;
278 /* private: for use by the iterator */
279 struct folio *_next;
280 size_t _seg_count;
281 int _i;
282 };
283
bio_first_folio(struct folio_iter * fi,struct bio * bio,int i)284 static inline void bio_first_folio(struct folio_iter *fi, struct bio *bio,
285 int i)
286 {
287 struct bio_vec *bvec = bio_first_bvec_all(bio) + i;
288
289 if (unlikely(i >= bio->bi_vcnt)) {
290 fi->folio = NULL;
291 return;
292 }
293
294 fi->folio = page_folio(bvec->bv_page);
295 fi->offset = bvec->bv_offset +
296 PAGE_SIZE * (bvec->bv_page - &fi->folio->page);
297 fi->_seg_count = bvec->bv_len;
298 fi->length = min(folio_size(fi->folio) - fi->offset, fi->_seg_count);
299 fi->_next = folio_next(fi->folio);
300 fi->_i = i;
301 }
302
bio_next_folio(struct folio_iter * fi,struct bio * bio)303 static inline void bio_next_folio(struct folio_iter *fi, struct bio *bio)
304 {
305 fi->_seg_count -= fi->length;
306 if (fi->_seg_count) {
307 fi->folio = fi->_next;
308 fi->offset = 0;
309 fi->length = min(folio_size(fi->folio), fi->_seg_count);
310 fi->_next = folio_next(fi->folio);
311 } else {
312 bio_first_folio(fi, bio, fi->_i + 1);
313 }
314 }
315
316 /**
317 * bio_for_each_folio_all - Iterate over each folio in a bio.
318 * @fi: struct folio_iter which is updated for each folio.
319 * @bio: struct bio to iterate over.
320 */
321 #define bio_for_each_folio_all(fi, bio) \
322 for (bio_first_folio(&fi, bio, 0); fi.folio; bio_next_folio(&fi, bio))
323
324 enum bip_flags {
325 BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */
326 BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */
327 BIP_CTRL_NOCHECK = 1 << 2, /* disable HBA integrity checking */
328 BIP_DISK_NOCHECK = 1 << 3, /* disable disk integrity checking */
329 BIP_IP_CHECKSUM = 1 << 4, /* IP checksum */
330 BIP_INTEGRITY_USER = 1 << 5, /* Integrity payload is user address */
331 BIP_COPY_USER = 1 << 6, /* Kernel bounce buffer in use */
332 };
333
334 /*
335 * bio integrity payload
336 */
337 struct bio_integrity_payload {
338 struct bio *bip_bio; /* parent bio */
339
340 struct bvec_iter bip_iter;
341
342 unsigned short bip_vcnt; /* # of integrity bio_vecs */
343 unsigned short bip_max_vcnt; /* integrity bio_vec slots */
344 unsigned short bip_flags; /* control flags */
345
346 struct bvec_iter bio_iter; /* for rewinding parent bio */
347
348 struct work_struct bip_work; /* I/O completion */
349
350 struct bio_vec *bip_vec;
351 struct bio_vec bip_inline_vecs[];/* embedded bvec array */
352 };
353
354 #if defined(CONFIG_BLK_DEV_INTEGRITY)
355
bio_integrity(struct bio * bio)356 static inline struct bio_integrity_payload *bio_integrity(struct bio *bio)
357 {
358 if (bio->bi_opf & REQ_INTEGRITY)
359 return bio->bi_integrity;
360
361 return NULL;
362 }
363
bio_integrity_flagged(struct bio * bio,enum bip_flags flag)364 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
365 {
366 struct bio_integrity_payload *bip = bio_integrity(bio);
367
368 if (bip)
369 return bip->bip_flags & flag;
370
371 return false;
372 }
373
bip_get_seed(struct bio_integrity_payload * bip)374 static inline sector_t bip_get_seed(struct bio_integrity_payload *bip)
375 {
376 return bip->bip_iter.bi_sector;
377 }
378
bip_set_seed(struct bio_integrity_payload * bip,sector_t seed)379 static inline void bip_set_seed(struct bio_integrity_payload *bip,
380 sector_t seed)
381 {
382 bip->bip_iter.bi_sector = seed;
383 }
384
385 #endif /* CONFIG_BLK_DEV_INTEGRITY */
386
387 void bio_trim(struct bio *bio, sector_t offset, sector_t size);
388 extern struct bio *bio_split(struct bio *bio, int sectors,
389 gfp_t gfp, struct bio_set *bs);
390 struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim,
391 unsigned *segs, struct bio_set *bs, unsigned max_bytes);
392
393 /**
394 * bio_next_split - get next @sectors from a bio, splitting if necessary
395 * @bio: bio to split
396 * @sectors: number of sectors to split from the front of @bio
397 * @gfp: gfp mask
398 * @bs: bio set to allocate from
399 *
400 * Return: a bio representing the next @sectors of @bio - if the bio is smaller
401 * than @sectors, returns the original bio unchanged.
402 */
bio_next_split(struct bio * bio,int sectors,gfp_t gfp,struct bio_set * bs)403 static inline struct bio *bio_next_split(struct bio *bio, int sectors,
404 gfp_t gfp, struct bio_set *bs)
405 {
406 if (sectors >= bio_sectors(bio))
407 return bio;
408
409 return bio_split(bio, sectors, gfp, bs);
410 }
411
412 enum {
413 BIOSET_NEED_BVECS = BIT(0),
414 BIOSET_NEED_RESCUER = BIT(1),
415 BIOSET_PERCPU_CACHE = BIT(2),
416 };
417 extern int bioset_init(struct bio_set *, unsigned int, unsigned int, int flags);
418 extern void bioset_exit(struct bio_set *);
419 extern int biovec_init_pool(mempool_t *pool, int pool_entries);
420
421 struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
422 blk_opf_t opf, gfp_t gfp_mask,
423 struct bio_set *bs);
424 struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask);
425 extern void bio_put(struct bio *);
426
427 struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src,
428 gfp_t gfp, struct bio_set *bs);
429 int bio_init_clone(struct block_device *bdev, struct bio *bio,
430 struct bio *bio_src, gfp_t gfp);
431
432 extern struct bio_set fs_bio_set;
433
bio_alloc(struct block_device * bdev,unsigned short nr_vecs,blk_opf_t opf,gfp_t gfp_mask)434 static inline struct bio *bio_alloc(struct block_device *bdev,
435 unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp_mask)
436 {
437 return bio_alloc_bioset(bdev, nr_vecs, opf, gfp_mask, &fs_bio_set);
438 }
439
440 void submit_bio(struct bio *bio);
441
442 extern void bio_endio(struct bio *);
443
bio_io_error(struct bio * bio)444 static inline void bio_io_error(struct bio *bio)
445 {
446 bio->bi_status = BLK_STS_IOERR;
447 bio_endio(bio);
448 }
449
bio_wouldblock_error(struct bio * bio)450 static inline void bio_wouldblock_error(struct bio *bio)
451 {
452 bio_set_flag(bio, BIO_QUIET);
453 bio->bi_status = BLK_STS_AGAIN;
454 bio_endio(bio);
455 }
456
457 /*
458 * Calculate number of bvec segments that should be allocated to fit data
459 * pointed by @iter. If @iter is backed by bvec it's going to be reused
460 * instead of allocating a new one.
461 */
bio_iov_vecs_to_alloc(struct iov_iter * iter,int max_segs)462 static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs)
463 {
464 if (iov_iter_is_bvec(iter))
465 return 0;
466 return iov_iter_npages(iter, max_segs);
467 }
468
469 struct request_queue;
470
471 extern int submit_bio_wait(struct bio *bio);
472 void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table,
473 unsigned short max_vecs, blk_opf_t opf);
474 extern void bio_uninit(struct bio *);
475 void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf);
476 void bio_chain(struct bio *, struct bio *);
477
478 int __must_check bio_add_page(struct bio *bio, struct page *page, unsigned len,
479 unsigned off);
480 bool __must_check bio_add_folio(struct bio *bio, struct folio *folio,
481 size_t len, size_t off);
482 extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
483 unsigned int, unsigned int);
484 int bio_add_zone_append_page(struct bio *bio, struct page *page,
485 unsigned int len, unsigned int offset);
486 void __bio_add_page(struct bio *bio, struct page *page,
487 unsigned int len, unsigned int off);
488 void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len,
489 size_t off);
490 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);
491 void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter);
492 void __bio_release_pages(struct bio *bio, bool mark_dirty);
493 extern void bio_set_pages_dirty(struct bio *bio);
494 extern void bio_check_pages_dirty(struct bio *bio);
495
496 extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
497 struct bio *src, struct bvec_iter *src_iter);
498 extern void bio_copy_data(struct bio *dst, struct bio *src);
499 extern void bio_free_pages(struct bio *bio);
500 void guard_bio_eod(struct bio *bio);
501 void zero_fill_bio_iter(struct bio *bio, struct bvec_iter iter);
502
zero_fill_bio(struct bio * bio)503 static inline void zero_fill_bio(struct bio *bio)
504 {
505 zero_fill_bio_iter(bio, bio->bi_iter);
506 }
507
bio_release_pages(struct bio * bio,bool mark_dirty)508 static inline void bio_release_pages(struct bio *bio, bool mark_dirty)
509 {
510 if (bio_flagged(bio, BIO_PAGE_PINNED))
511 __bio_release_pages(bio, mark_dirty);
512 }
513
514 #define bio_dev(bio) \
515 disk_devt((bio)->bi_bdev->bd_disk)
516
517 #ifdef CONFIG_BLK_CGROUP
518 void bio_associate_blkg(struct bio *bio);
519 void bio_associate_blkg_from_css(struct bio *bio,
520 struct cgroup_subsys_state *css);
521 void bio_clone_blkg_association(struct bio *dst, struct bio *src);
522 void blkcg_punt_bio_submit(struct bio *bio);
523 #else /* CONFIG_BLK_CGROUP */
bio_associate_blkg(struct bio * bio)524 static inline void bio_associate_blkg(struct bio *bio) { }
bio_associate_blkg_from_css(struct bio * bio,struct cgroup_subsys_state * css)525 static inline void bio_associate_blkg_from_css(struct bio *bio,
526 struct cgroup_subsys_state *css)
527 { }
bio_clone_blkg_association(struct bio * dst,struct bio * src)528 static inline void bio_clone_blkg_association(struct bio *dst,
529 struct bio *src) { }
blkcg_punt_bio_submit(struct bio * bio)530 static inline void blkcg_punt_bio_submit(struct bio *bio)
531 {
532 submit_bio(bio);
533 }
534 #endif /* CONFIG_BLK_CGROUP */
535
bio_set_dev(struct bio * bio,struct block_device * bdev)536 static inline void bio_set_dev(struct bio *bio, struct block_device *bdev)
537 {
538 bio_clear_flag(bio, BIO_REMAPPED);
539 if (bio->bi_bdev != bdev)
540 bio_clear_flag(bio, BIO_BPS_THROTTLED);
541 bio->bi_bdev = bdev;
542 bio_associate_blkg(bio);
543 }
544
545 /*
546 * BIO list management for use by remapping drivers (e.g. DM or MD) and loop.
547 *
548 * A bio_list anchors a singly-linked list of bios chained through the bi_next
549 * member of the bio. The bio_list also caches the last list member to allow
550 * fast access to the tail.
551 */
552 struct bio_list {
553 struct bio *head;
554 struct bio *tail;
555 };
556
bio_list_empty(const struct bio_list * bl)557 static inline int bio_list_empty(const struct bio_list *bl)
558 {
559 return bl->head == NULL;
560 }
561
bio_list_init(struct bio_list * bl)562 static inline void bio_list_init(struct bio_list *bl)
563 {
564 bl->head = bl->tail = NULL;
565 }
566
567 #define BIO_EMPTY_LIST { NULL, NULL }
568
569 #define bio_list_for_each(bio, bl) \
570 for (bio = (bl)->head; bio; bio = bio->bi_next)
571
bio_list_size(const struct bio_list * bl)572 static inline unsigned bio_list_size(const struct bio_list *bl)
573 {
574 unsigned sz = 0;
575 struct bio *bio;
576
577 bio_list_for_each(bio, bl)
578 sz++;
579
580 return sz;
581 }
582
bio_list_add(struct bio_list * bl,struct bio * bio)583 static inline void bio_list_add(struct bio_list *bl, struct bio *bio)
584 {
585 bio->bi_next = NULL;
586
587 if (bl->tail)
588 bl->tail->bi_next = bio;
589 else
590 bl->head = bio;
591
592 bl->tail = bio;
593 }
594
bio_list_add_head(struct bio_list * bl,struct bio * bio)595 static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio)
596 {
597 bio->bi_next = bl->head;
598
599 bl->head = bio;
600
601 if (!bl->tail)
602 bl->tail = bio;
603 }
604
bio_list_merge(struct bio_list * bl,struct bio_list * bl2)605 static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2)
606 {
607 if (!bl2->head)
608 return;
609
610 if (bl->tail)
611 bl->tail->bi_next = bl2->head;
612 else
613 bl->head = bl2->head;
614
615 bl->tail = bl2->tail;
616 }
617
bio_list_merge_init(struct bio_list * bl,struct bio_list * bl2)618 static inline void bio_list_merge_init(struct bio_list *bl,
619 struct bio_list *bl2)
620 {
621 bio_list_merge(bl, bl2);
622 bio_list_init(bl2);
623 }
624
bio_list_merge_head(struct bio_list * bl,struct bio_list * bl2)625 static inline void bio_list_merge_head(struct bio_list *bl,
626 struct bio_list *bl2)
627 {
628 if (!bl2->head)
629 return;
630
631 if (bl->head)
632 bl2->tail->bi_next = bl->head;
633 else
634 bl->tail = bl2->tail;
635
636 bl->head = bl2->head;
637 }
638
bio_list_peek(struct bio_list * bl)639 static inline struct bio *bio_list_peek(struct bio_list *bl)
640 {
641 return bl->head;
642 }
643
bio_list_pop(struct bio_list * bl)644 static inline struct bio *bio_list_pop(struct bio_list *bl)
645 {
646 struct bio *bio = bl->head;
647
648 if (bio) {
649 bl->head = bl->head->bi_next;
650 if (!bl->head)
651 bl->tail = NULL;
652
653 bio->bi_next = NULL;
654 }
655
656 return bio;
657 }
658
bio_list_get(struct bio_list * bl)659 static inline struct bio *bio_list_get(struct bio_list *bl)
660 {
661 struct bio *bio = bl->head;
662
663 bl->head = bl->tail = NULL;
664
665 return bio;
666 }
667
668 /*
669 * Increment chain count for the bio. Make sure the CHAIN flag update
670 * is visible before the raised count.
671 */
bio_inc_remaining(struct bio * bio)672 static inline void bio_inc_remaining(struct bio *bio)
673 {
674 bio_set_flag(bio, BIO_CHAIN);
675 smp_mb__before_atomic();
676 atomic_inc(&bio->__bi_remaining);
677 }
678
679 /*
680 * bio_set is used to allow other portions of the IO system to
681 * allocate their own private memory pools for bio and iovec structures.
682 * These memory pools in turn all allocate from the bio_slab
683 * and the bvec_slabs[].
684 */
685 #define BIO_POOL_SIZE 2
686
687 struct bio_set {
688 struct kmem_cache *bio_slab;
689 unsigned int front_pad;
690
691 /*
692 * per-cpu bio alloc cache
693 */
694 struct bio_alloc_cache __percpu *cache;
695
696 mempool_t bio_pool;
697 mempool_t bvec_pool;
698 #if defined(CONFIG_BLK_DEV_INTEGRITY)
699 mempool_t bio_integrity_pool;
700 mempool_t bvec_integrity_pool;
701 #endif
702
703 unsigned int back_pad;
704 /*
705 * Deadlock avoidance for stacking block drivers: see comments in
706 * bio_alloc_bioset() for details
707 */
708 spinlock_t rescue_lock;
709 struct bio_list rescue_list;
710 struct work_struct rescue_work;
711 struct workqueue_struct *rescue_workqueue;
712
713 /*
714 * Hot un-plug notifier for the per-cpu cache, if used
715 */
716 struct hlist_node cpuhp_dead;
717 };
718
bioset_initialized(struct bio_set * bs)719 static inline bool bioset_initialized(struct bio_set *bs)
720 {
721 return bs->bio_slab != NULL;
722 }
723
724 #if defined(CONFIG_BLK_DEV_INTEGRITY)
725
726 #define bip_for_each_vec(bvl, bip, iter) \
727 for_each_bvec(bvl, (bip)->bip_vec, iter, (bip)->bip_iter)
728
729 #define bio_for_each_integrity_vec(_bvl, _bio, _iter) \
730 for_each_bio(_bio) \
731 bip_for_each_vec(_bvl, _bio->bi_integrity, _iter)
732
733 int bio_integrity_map_user(struct bio *bio, void __user *ubuf, ssize_t len, u32 seed);
734 extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
735 extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
736 extern bool bio_integrity_prep(struct bio *);
737 extern void bio_integrity_advance(struct bio *, unsigned int);
738 extern void bio_integrity_trim(struct bio *);
739 extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
740 extern int bioset_integrity_create(struct bio_set *, int);
741 extern void bioset_integrity_free(struct bio_set *);
742 extern void bio_integrity_init(void);
743
744 #else /* CONFIG_BLK_DEV_INTEGRITY */
745
bio_integrity(struct bio * bio)746 static inline void *bio_integrity(struct bio *bio)
747 {
748 return NULL;
749 }
750
bioset_integrity_create(struct bio_set * bs,int pool_size)751 static inline int bioset_integrity_create(struct bio_set *bs, int pool_size)
752 {
753 return 0;
754 }
755
bioset_integrity_free(struct bio_set * bs)756 static inline void bioset_integrity_free (struct bio_set *bs)
757 {
758 return;
759 }
760
bio_integrity_prep(struct bio * bio)761 static inline bool bio_integrity_prep(struct bio *bio)
762 {
763 return true;
764 }
765
bio_integrity_clone(struct bio * bio,struct bio * bio_src,gfp_t gfp_mask)766 static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
767 gfp_t gfp_mask)
768 {
769 return 0;
770 }
771
bio_integrity_advance(struct bio * bio,unsigned int bytes_done)772 static inline void bio_integrity_advance(struct bio *bio,
773 unsigned int bytes_done)
774 {
775 return;
776 }
777
bio_integrity_trim(struct bio * bio)778 static inline void bio_integrity_trim(struct bio *bio)
779 {
780 return;
781 }
782
bio_integrity_init(void)783 static inline void bio_integrity_init(void)
784 {
785 return;
786 }
787
bio_integrity_flagged(struct bio * bio,enum bip_flags flag)788 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
789 {
790 return false;
791 }
792
bio_integrity_alloc(struct bio * bio,gfp_t gfp,unsigned int nr)793 static inline void *bio_integrity_alloc(struct bio * bio, gfp_t gfp,
794 unsigned int nr)
795 {
796 return ERR_PTR(-EINVAL);
797 }
798
bio_integrity_add_page(struct bio * bio,struct page * page,unsigned int len,unsigned int offset)799 static inline int bio_integrity_add_page(struct bio *bio, struct page *page,
800 unsigned int len, unsigned int offset)
801 {
802 return 0;
803 }
804
bio_integrity_map_user(struct bio * bio,void __user * ubuf,ssize_t len,u32 seed)805 static inline int bio_integrity_map_user(struct bio *bio, void __user *ubuf,
806 ssize_t len, u32 seed)
807 {
808 return -EINVAL;
809 }
810
811 #endif /* CONFIG_BLK_DEV_INTEGRITY */
812
813 /*
814 * Mark a bio as polled. Note that for async polled IO, the caller must
815 * expect -EWOULDBLOCK if we cannot allocate a request (or other resources).
816 * We cannot block waiting for requests on polled IO, as those completions
817 * must be found by the caller. This is different than IRQ driven IO, where
818 * it's safe to wait for IO to complete.
819 */
bio_set_polled(struct bio * bio,struct kiocb * kiocb)820 static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb)
821 {
822 bio->bi_opf |= REQ_POLLED;
823 if (kiocb->ki_flags & IOCB_NOWAIT)
824 bio->bi_opf |= REQ_NOWAIT;
825 }
826
bio_clear_polled(struct bio * bio)827 static inline void bio_clear_polled(struct bio *bio)
828 {
829 bio->bi_opf &= ~REQ_POLLED;
830 }
831
832 struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev,
833 unsigned int nr_pages, blk_opf_t opf, gfp_t gfp);
834 struct bio *bio_chain_and_submit(struct bio *prev, struct bio *new);
835
836 struct bio *blk_alloc_discard_bio(struct block_device *bdev,
837 sector_t *sector, sector_t *nr_sects, gfp_t gfp_mask);
838
839 #endif /* __LINUX_BIO_H */
840