1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2001 Jens Axboe <axboe@suse.de> 4 */ 5 #ifndef __LINUX_BIO_H 6 #define __LINUX_BIO_H 7 8 #include <linux/mempool.h> 9 /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */ 10 #include <linux/blk_types.h> 11 #include <linux/uio.h> 12 13 #define BIO_MAX_VECS 256U 14 15 struct queue_limits; 16 17 static inline unsigned int bio_max_segs(unsigned int nr_segs) 18 { 19 return min(nr_segs, BIO_MAX_VECS); 20 } 21 22 #define bio_prio(bio) (bio)->bi_ioprio 23 #define bio_set_prio(bio, prio) ((bio)->bi_ioprio = prio) 24 25 #define bio_iter_iovec(bio, iter) \ 26 bvec_iter_bvec((bio)->bi_io_vec, (iter)) 27 28 #define bio_iter_page(bio, iter) \ 29 bvec_iter_page((bio)->bi_io_vec, (iter)) 30 #define bio_iter_len(bio, iter) \ 31 bvec_iter_len((bio)->bi_io_vec, (iter)) 32 #define bio_iter_offset(bio, iter) \ 33 bvec_iter_offset((bio)->bi_io_vec, (iter)) 34 35 #define bio_page(bio) bio_iter_page((bio), (bio)->bi_iter) 36 #define bio_offset(bio) bio_iter_offset((bio), (bio)->bi_iter) 37 #define bio_iovec(bio) bio_iter_iovec((bio), (bio)->bi_iter) 38 39 #define bvec_iter_sectors(iter) ((iter).bi_size >> 9) 40 #define bvec_iter_end_sector(iter) ((iter).bi_sector + bvec_iter_sectors((iter))) 41 42 #define bio_sectors(bio) bvec_iter_sectors((bio)->bi_iter) 43 #define bio_end_sector(bio) bvec_iter_end_sector((bio)->bi_iter) 44 45 /* 46 * Return the data direction, READ or WRITE. 47 */ 48 #define bio_data_dir(bio) \ 49 (op_is_write(bio_op(bio)) ? WRITE : READ) 50 51 /* 52 * Check whether this bio carries any data or not. A NULL bio is allowed. 53 */ 54 static inline bool bio_has_data(struct bio *bio) 55 { 56 if (bio && 57 bio->bi_iter.bi_size && 58 bio_op(bio) != REQ_OP_DISCARD && 59 bio_op(bio) != REQ_OP_SECURE_ERASE && 60 bio_op(bio) != REQ_OP_WRITE_ZEROES) 61 return true; 62 63 return false; 64 } 65 66 static inline bool bio_no_advance_iter(const struct bio *bio) 67 { 68 return bio_op(bio) == REQ_OP_DISCARD || 69 bio_op(bio) == REQ_OP_SECURE_ERASE || 70 bio_op(bio) == REQ_OP_WRITE_ZEROES; 71 } 72 73 static inline void *bio_data(struct bio *bio) 74 { 75 if (bio_has_data(bio)) 76 return page_address(bio_page(bio)) + bio_offset(bio); 77 78 return NULL; 79 } 80 81 static inline bool bio_next_segment(const struct bio *bio, 82 struct bvec_iter_all *iter) 83 { 84 if (iter->idx >= bio->bi_vcnt) 85 return false; 86 87 bvec_advance(&bio->bi_io_vec[iter->idx], iter); 88 return true; 89 } 90 91 /* 92 * drivers should _never_ use the all version - the bio may have been split 93 * before it got to the driver and the driver won't own all of it 94 */ 95 #define bio_for_each_segment_all(bvl, bio, iter) \ 96 for (bvl = bvec_init_iter_all(&iter); bio_next_segment((bio), &iter); ) 97 98 static inline void bio_advance_iter(const struct bio *bio, 99 struct bvec_iter *iter, unsigned int bytes) 100 { 101 iter->bi_sector += bytes >> 9; 102 103 if (bio_no_advance_iter(bio)) 104 iter->bi_size -= bytes; 105 else 106 bvec_iter_advance(bio->bi_io_vec, iter, bytes); 107 /* TODO: It is reasonable to complete bio with error here. */ 108 } 109 110 /* @bytes should be less or equal to bvec[i->bi_idx].bv_len */ 111 static inline void bio_advance_iter_single(const struct bio *bio, 112 struct bvec_iter *iter, 113 unsigned int bytes) 114 { 115 iter->bi_sector += bytes >> 9; 116 117 if (bio_no_advance_iter(bio)) 118 iter->bi_size -= bytes; 119 else 120 bvec_iter_advance_single(bio->bi_io_vec, iter, bytes); 121 } 122 123 void __bio_advance(struct bio *, unsigned bytes); 124 125 /** 126 * bio_advance - increment/complete a bio by some number of bytes 127 * @bio: bio to advance 128 * @nbytes: number of bytes to complete 129 * 130 * This updates bi_sector, bi_size and bi_idx; if the number of bytes to 131 * complete doesn't align with a bvec boundary, then bv_len and bv_offset will 132 * be updated on the last bvec as well. 133 * 134 * @bio will then represent the remaining, uncompleted portion of the io. 135 */ 136 static inline void bio_advance(struct bio *bio, unsigned int nbytes) 137 { 138 if (nbytes == bio->bi_iter.bi_size) { 139 bio->bi_iter.bi_size = 0; 140 return; 141 } 142 __bio_advance(bio, nbytes); 143 } 144 145 #define __bio_for_each_segment(bvl, bio, iter, start) \ 146 for (iter = (start); \ 147 (iter).bi_size && \ 148 ((bvl = bio_iter_iovec((bio), (iter))), 1); \ 149 bio_advance_iter_single((bio), &(iter), (bvl).bv_len)) 150 151 #define bio_for_each_segment(bvl, bio, iter) \ 152 __bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter) 153 154 #define __bio_for_each_bvec(bvl, bio, iter, start) \ 155 for (iter = (start); \ 156 (iter).bi_size && \ 157 ((bvl = mp_bvec_iter_bvec((bio)->bi_io_vec, (iter))), 1); \ 158 bio_advance_iter_single((bio), &(iter), (bvl).bv_len)) 159 160 /* iterate over multi-page bvec */ 161 #define bio_for_each_bvec(bvl, bio, iter) \ 162 __bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter) 163 164 /* 165 * Iterate over all multi-page bvecs. Drivers shouldn't use this version for the 166 * same reasons as bio_for_each_segment_all(). 167 */ 168 #define bio_for_each_bvec_all(bvl, bio, i) \ 169 for (i = 0, bvl = bio_first_bvec_all(bio); \ 170 i < (bio)->bi_vcnt; i++, bvl++) 171 172 #define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len) 173 174 static inline unsigned bio_segments(struct bio *bio) 175 { 176 unsigned segs = 0; 177 struct bio_vec bv; 178 struct bvec_iter iter; 179 180 /* 181 * We special case discard/write same/write zeroes, because they 182 * interpret bi_size differently: 183 */ 184 185 switch (bio_op(bio)) { 186 case REQ_OP_DISCARD: 187 case REQ_OP_SECURE_ERASE: 188 case REQ_OP_WRITE_ZEROES: 189 return 0; 190 default: 191 break; 192 } 193 194 bio_for_each_segment(bv, bio, iter) 195 segs++; 196 197 return segs; 198 } 199 200 /* 201 * get a reference to a bio, so it won't disappear. the intended use is 202 * something like: 203 * 204 * bio_get(bio); 205 * submit_bio(rw, bio); 206 * if (bio->bi_flags ...) 207 * do_something 208 * bio_put(bio); 209 * 210 * without the bio_get(), it could potentially complete I/O before submit_bio 211 * returns. and then bio would be freed memory when if (bio->bi_flags ...) 212 * runs 213 */ 214 static inline void bio_get(struct bio *bio) 215 { 216 bio->bi_flags |= (1 << BIO_REFFED); 217 smp_mb__before_atomic(); 218 atomic_inc(&bio->__bi_cnt); 219 } 220 221 static inline void bio_cnt_set(struct bio *bio, unsigned int count) 222 { 223 if (count != 1) { 224 bio->bi_flags |= (1 << BIO_REFFED); 225 smp_mb(); 226 } 227 atomic_set(&bio->__bi_cnt, count); 228 } 229 230 static inline bool bio_flagged(struct bio *bio, unsigned int bit) 231 { 232 return bio->bi_flags & (1U << bit); 233 } 234 235 static inline void bio_set_flag(struct bio *bio, unsigned int bit) 236 { 237 bio->bi_flags |= (1U << bit); 238 } 239 240 static inline void bio_clear_flag(struct bio *bio, unsigned int bit) 241 { 242 bio->bi_flags &= ~(1U << bit); 243 } 244 245 static inline struct bio_vec *bio_first_bvec_all(struct bio *bio) 246 { 247 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); 248 return bio->bi_io_vec; 249 } 250 251 static inline struct page *bio_first_page_all(struct bio *bio) 252 { 253 return bio_first_bvec_all(bio)->bv_page; 254 } 255 256 static inline struct folio *bio_first_folio_all(struct bio *bio) 257 { 258 return page_folio(bio_first_page_all(bio)); 259 } 260 261 static inline struct bio_vec *bio_last_bvec_all(struct bio *bio) 262 { 263 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); 264 return &bio->bi_io_vec[bio->bi_vcnt - 1]; 265 } 266 267 /** 268 * struct folio_iter - State for iterating all folios in a bio. 269 * @folio: The current folio we're iterating. NULL after the last folio. 270 * @offset: The byte offset within the current folio. 271 * @length: The number of bytes in this iteration (will not cross folio 272 * boundary). 273 */ 274 struct folio_iter { 275 struct folio *folio; 276 size_t offset; 277 size_t length; 278 /* private: for use by the iterator */ 279 struct folio *_next; 280 size_t _seg_count; 281 int _i; 282 }; 283 284 static inline void bio_first_folio(struct folio_iter *fi, struct bio *bio, 285 int i) 286 { 287 struct bio_vec *bvec = bio_first_bvec_all(bio) + i; 288 289 if (unlikely(i >= bio->bi_vcnt)) { 290 fi->folio = NULL; 291 return; 292 } 293 294 fi->folio = page_folio(bvec->bv_page); 295 fi->offset = bvec->bv_offset + 296 PAGE_SIZE * (bvec->bv_page - &fi->folio->page); 297 fi->_seg_count = bvec->bv_len; 298 fi->length = min(folio_size(fi->folio) - fi->offset, fi->_seg_count); 299 fi->_next = folio_next(fi->folio); 300 fi->_i = i; 301 } 302 303 static inline void bio_next_folio(struct folio_iter *fi, struct bio *bio) 304 { 305 fi->_seg_count -= fi->length; 306 if (fi->_seg_count) { 307 fi->folio = fi->_next; 308 fi->offset = 0; 309 fi->length = min(folio_size(fi->folio), fi->_seg_count); 310 fi->_next = folio_next(fi->folio); 311 } else { 312 bio_first_folio(fi, bio, fi->_i + 1); 313 } 314 } 315 316 /** 317 * bio_for_each_folio_all - Iterate over each folio in a bio. 318 * @fi: struct folio_iter which is updated for each folio. 319 * @bio: struct bio to iterate over. 320 */ 321 #define bio_for_each_folio_all(fi, bio) \ 322 for (bio_first_folio(&fi, bio, 0); fi.folio; bio_next_folio(&fi, bio)) 323 324 enum bip_flags { 325 BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */ 326 BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */ 327 BIP_CTRL_NOCHECK = 1 << 2, /* disable HBA integrity checking */ 328 BIP_DISK_NOCHECK = 1 << 3, /* disable disk integrity checking */ 329 BIP_IP_CHECKSUM = 1 << 4, /* IP checksum */ 330 BIP_INTEGRITY_USER = 1 << 5, /* Integrity payload is user address */ 331 BIP_COPY_USER = 1 << 6, /* Kernel bounce buffer in use */ 332 }; 333 334 /* 335 * bio integrity payload 336 */ 337 struct bio_integrity_payload { 338 struct bio *bip_bio; /* parent bio */ 339 340 struct bvec_iter bip_iter; 341 342 unsigned short bip_vcnt; /* # of integrity bio_vecs */ 343 unsigned short bip_max_vcnt; /* integrity bio_vec slots */ 344 unsigned short bip_flags; /* control flags */ 345 346 struct bvec_iter bio_iter; /* for rewinding parent bio */ 347 348 struct work_struct bip_work; /* I/O completion */ 349 350 struct bio_vec *bip_vec; 351 struct bio_vec bip_inline_vecs[];/* embedded bvec array */ 352 }; 353 354 #if defined(CONFIG_BLK_DEV_INTEGRITY) 355 356 static inline struct bio_integrity_payload *bio_integrity(struct bio *bio) 357 { 358 if (bio->bi_opf & REQ_INTEGRITY) 359 return bio->bi_integrity; 360 361 return NULL; 362 } 363 364 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag) 365 { 366 struct bio_integrity_payload *bip = bio_integrity(bio); 367 368 if (bip) 369 return bip->bip_flags & flag; 370 371 return false; 372 } 373 374 static inline sector_t bip_get_seed(struct bio_integrity_payload *bip) 375 { 376 return bip->bip_iter.bi_sector; 377 } 378 379 static inline void bip_set_seed(struct bio_integrity_payload *bip, 380 sector_t seed) 381 { 382 bip->bip_iter.bi_sector = seed; 383 } 384 385 #endif /* CONFIG_BLK_DEV_INTEGRITY */ 386 387 void bio_trim(struct bio *bio, sector_t offset, sector_t size); 388 extern struct bio *bio_split(struct bio *bio, int sectors, 389 gfp_t gfp, struct bio_set *bs); 390 struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim, 391 unsigned *segs, struct bio_set *bs, unsigned max_bytes); 392 393 /** 394 * bio_next_split - get next @sectors from a bio, splitting if necessary 395 * @bio: bio to split 396 * @sectors: number of sectors to split from the front of @bio 397 * @gfp: gfp mask 398 * @bs: bio set to allocate from 399 * 400 * Return: a bio representing the next @sectors of @bio - if the bio is smaller 401 * than @sectors, returns the original bio unchanged. 402 */ 403 static inline struct bio *bio_next_split(struct bio *bio, int sectors, 404 gfp_t gfp, struct bio_set *bs) 405 { 406 if (sectors >= bio_sectors(bio)) 407 return bio; 408 409 return bio_split(bio, sectors, gfp, bs); 410 } 411 412 enum { 413 BIOSET_NEED_BVECS = BIT(0), 414 BIOSET_NEED_RESCUER = BIT(1), 415 BIOSET_PERCPU_CACHE = BIT(2), 416 }; 417 extern int bioset_init(struct bio_set *, unsigned int, unsigned int, int flags); 418 extern void bioset_exit(struct bio_set *); 419 extern int biovec_init_pool(mempool_t *pool, int pool_entries); 420 421 struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs, 422 blk_opf_t opf, gfp_t gfp_mask, 423 struct bio_set *bs); 424 struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask); 425 extern void bio_put(struct bio *); 426 427 struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src, 428 gfp_t gfp, struct bio_set *bs); 429 int bio_init_clone(struct block_device *bdev, struct bio *bio, 430 struct bio *bio_src, gfp_t gfp); 431 432 extern struct bio_set fs_bio_set; 433 434 static inline struct bio *bio_alloc(struct block_device *bdev, 435 unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp_mask) 436 { 437 return bio_alloc_bioset(bdev, nr_vecs, opf, gfp_mask, &fs_bio_set); 438 } 439 440 void submit_bio(struct bio *bio); 441 442 extern void bio_endio(struct bio *); 443 444 static inline void bio_io_error(struct bio *bio) 445 { 446 bio->bi_status = BLK_STS_IOERR; 447 bio_endio(bio); 448 } 449 450 static inline void bio_wouldblock_error(struct bio *bio) 451 { 452 bio_set_flag(bio, BIO_QUIET); 453 bio->bi_status = BLK_STS_AGAIN; 454 bio_endio(bio); 455 } 456 457 /* 458 * Calculate number of bvec segments that should be allocated to fit data 459 * pointed by @iter. If @iter is backed by bvec it's going to be reused 460 * instead of allocating a new one. 461 */ 462 static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs) 463 { 464 if (iov_iter_is_bvec(iter)) 465 return 0; 466 return iov_iter_npages(iter, max_segs); 467 } 468 469 struct request_queue; 470 471 extern int submit_bio_wait(struct bio *bio); 472 void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, 473 unsigned short max_vecs, blk_opf_t opf); 474 extern void bio_uninit(struct bio *); 475 void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf); 476 void bio_chain(struct bio *, struct bio *); 477 478 int __must_check bio_add_page(struct bio *bio, struct page *page, unsigned len, 479 unsigned off); 480 bool __must_check bio_add_folio(struct bio *bio, struct folio *folio, 481 size_t len, size_t off); 482 extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *, 483 unsigned int, unsigned int); 484 int bio_add_zone_append_page(struct bio *bio, struct page *page, 485 unsigned int len, unsigned int offset); 486 void __bio_add_page(struct bio *bio, struct page *page, 487 unsigned int len, unsigned int off); 488 void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len, 489 size_t off); 490 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter); 491 void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter); 492 void __bio_release_pages(struct bio *bio, bool mark_dirty); 493 extern void bio_set_pages_dirty(struct bio *bio); 494 extern void bio_check_pages_dirty(struct bio *bio); 495 496 extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter, 497 struct bio *src, struct bvec_iter *src_iter); 498 extern void bio_copy_data(struct bio *dst, struct bio *src); 499 extern void bio_free_pages(struct bio *bio); 500 void guard_bio_eod(struct bio *bio); 501 void zero_fill_bio_iter(struct bio *bio, struct bvec_iter iter); 502 503 static inline void zero_fill_bio(struct bio *bio) 504 { 505 zero_fill_bio_iter(bio, bio->bi_iter); 506 } 507 508 static inline void bio_release_pages(struct bio *bio, bool mark_dirty) 509 { 510 if (bio_flagged(bio, BIO_PAGE_PINNED)) 511 __bio_release_pages(bio, mark_dirty); 512 } 513 514 #define bio_dev(bio) \ 515 disk_devt((bio)->bi_bdev->bd_disk) 516 517 #ifdef CONFIG_BLK_CGROUP 518 void bio_associate_blkg(struct bio *bio); 519 void bio_associate_blkg_from_css(struct bio *bio, 520 struct cgroup_subsys_state *css); 521 void bio_clone_blkg_association(struct bio *dst, struct bio *src); 522 void blkcg_punt_bio_submit(struct bio *bio); 523 #else /* CONFIG_BLK_CGROUP */ 524 static inline void bio_associate_blkg(struct bio *bio) { } 525 static inline void bio_associate_blkg_from_css(struct bio *bio, 526 struct cgroup_subsys_state *css) 527 { } 528 static inline void bio_clone_blkg_association(struct bio *dst, 529 struct bio *src) { } 530 static inline void blkcg_punt_bio_submit(struct bio *bio) 531 { 532 submit_bio(bio); 533 } 534 #endif /* CONFIG_BLK_CGROUP */ 535 536 static inline void bio_set_dev(struct bio *bio, struct block_device *bdev) 537 { 538 bio_clear_flag(bio, BIO_REMAPPED); 539 if (bio->bi_bdev != bdev) 540 bio_clear_flag(bio, BIO_BPS_THROTTLED); 541 bio->bi_bdev = bdev; 542 bio_associate_blkg(bio); 543 } 544 545 /* 546 * BIO list management for use by remapping drivers (e.g. DM or MD) and loop. 547 * 548 * A bio_list anchors a singly-linked list of bios chained through the bi_next 549 * member of the bio. The bio_list also caches the last list member to allow 550 * fast access to the tail. 551 */ 552 struct bio_list { 553 struct bio *head; 554 struct bio *tail; 555 }; 556 557 static inline int bio_list_empty(const struct bio_list *bl) 558 { 559 return bl->head == NULL; 560 } 561 562 static inline void bio_list_init(struct bio_list *bl) 563 { 564 bl->head = bl->tail = NULL; 565 } 566 567 #define BIO_EMPTY_LIST { NULL, NULL } 568 569 #define bio_list_for_each(bio, bl) \ 570 for (bio = (bl)->head; bio; bio = bio->bi_next) 571 572 static inline unsigned bio_list_size(const struct bio_list *bl) 573 { 574 unsigned sz = 0; 575 struct bio *bio; 576 577 bio_list_for_each(bio, bl) 578 sz++; 579 580 return sz; 581 } 582 583 static inline void bio_list_add(struct bio_list *bl, struct bio *bio) 584 { 585 bio->bi_next = NULL; 586 587 if (bl->tail) 588 bl->tail->bi_next = bio; 589 else 590 bl->head = bio; 591 592 bl->tail = bio; 593 } 594 595 static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio) 596 { 597 bio->bi_next = bl->head; 598 599 bl->head = bio; 600 601 if (!bl->tail) 602 bl->tail = bio; 603 } 604 605 static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2) 606 { 607 if (!bl2->head) 608 return; 609 610 if (bl->tail) 611 bl->tail->bi_next = bl2->head; 612 else 613 bl->head = bl2->head; 614 615 bl->tail = bl2->tail; 616 } 617 618 static inline void bio_list_merge_head(struct bio_list *bl, 619 struct bio_list *bl2) 620 { 621 if (!bl2->head) 622 return; 623 624 if (bl->head) 625 bl2->tail->bi_next = bl->head; 626 else 627 bl->tail = bl2->tail; 628 629 bl->head = bl2->head; 630 } 631 632 static inline struct bio *bio_list_peek(struct bio_list *bl) 633 { 634 return bl->head; 635 } 636 637 static inline struct bio *bio_list_pop(struct bio_list *bl) 638 { 639 struct bio *bio = bl->head; 640 641 if (bio) { 642 bl->head = bl->head->bi_next; 643 if (!bl->head) 644 bl->tail = NULL; 645 646 bio->bi_next = NULL; 647 } 648 649 return bio; 650 } 651 652 static inline struct bio *bio_list_get(struct bio_list *bl) 653 { 654 struct bio *bio = bl->head; 655 656 bl->head = bl->tail = NULL; 657 658 return bio; 659 } 660 661 /* 662 * Increment chain count for the bio. Make sure the CHAIN flag update 663 * is visible before the raised count. 664 */ 665 static inline void bio_inc_remaining(struct bio *bio) 666 { 667 bio_set_flag(bio, BIO_CHAIN); 668 smp_mb__before_atomic(); 669 atomic_inc(&bio->__bi_remaining); 670 } 671 672 /* 673 * bio_set is used to allow other portions of the IO system to 674 * allocate their own private memory pools for bio and iovec structures. 675 * These memory pools in turn all allocate from the bio_slab 676 * and the bvec_slabs[]. 677 */ 678 #define BIO_POOL_SIZE 2 679 680 struct bio_set { 681 struct kmem_cache *bio_slab; 682 unsigned int front_pad; 683 684 /* 685 * per-cpu bio alloc cache 686 */ 687 struct bio_alloc_cache __percpu *cache; 688 689 mempool_t bio_pool; 690 mempool_t bvec_pool; 691 #if defined(CONFIG_BLK_DEV_INTEGRITY) 692 mempool_t bio_integrity_pool; 693 mempool_t bvec_integrity_pool; 694 #endif 695 696 unsigned int back_pad; 697 /* 698 * Deadlock avoidance for stacking block drivers: see comments in 699 * bio_alloc_bioset() for details 700 */ 701 spinlock_t rescue_lock; 702 struct bio_list rescue_list; 703 struct work_struct rescue_work; 704 struct workqueue_struct *rescue_workqueue; 705 706 /* 707 * Hot un-plug notifier for the per-cpu cache, if used 708 */ 709 struct hlist_node cpuhp_dead; 710 }; 711 712 static inline bool bioset_initialized(struct bio_set *bs) 713 { 714 return bs->bio_slab != NULL; 715 } 716 717 #if defined(CONFIG_BLK_DEV_INTEGRITY) 718 719 #define bip_for_each_vec(bvl, bip, iter) \ 720 for_each_bvec(bvl, (bip)->bip_vec, iter, (bip)->bip_iter) 721 722 #define bio_for_each_integrity_vec(_bvl, _bio, _iter) \ 723 for_each_bio(_bio) \ 724 bip_for_each_vec(_bvl, _bio->bi_integrity, _iter) 725 726 int bio_integrity_map_user(struct bio *bio, void __user *ubuf, ssize_t len, u32 seed); 727 extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int); 728 extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int); 729 extern bool bio_integrity_prep(struct bio *); 730 extern void bio_integrity_advance(struct bio *, unsigned int); 731 extern void bio_integrity_trim(struct bio *); 732 extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t); 733 extern int bioset_integrity_create(struct bio_set *, int); 734 extern void bioset_integrity_free(struct bio_set *); 735 extern void bio_integrity_init(void); 736 737 #else /* CONFIG_BLK_DEV_INTEGRITY */ 738 739 static inline void *bio_integrity(struct bio *bio) 740 { 741 return NULL; 742 } 743 744 static inline int bioset_integrity_create(struct bio_set *bs, int pool_size) 745 { 746 return 0; 747 } 748 749 static inline void bioset_integrity_free (struct bio_set *bs) 750 { 751 return; 752 } 753 754 static inline bool bio_integrity_prep(struct bio *bio) 755 { 756 return true; 757 } 758 759 static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src, 760 gfp_t gfp_mask) 761 { 762 return 0; 763 } 764 765 static inline void bio_integrity_advance(struct bio *bio, 766 unsigned int bytes_done) 767 { 768 return; 769 } 770 771 static inline void bio_integrity_trim(struct bio *bio) 772 { 773 return; 774 } 775 776 static inline void bio_integrity_init(void) 777 { 778 return; 779 } 780 781 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag) 782 { 783 return false; 784 } 785 786 static inline void *bio_integrity_alloc(struct bio * bio, gfp_t gfp, 787 unsigned int nr) 788 { 789 return ERR_PTR(-EINVAL); 790 } 791 792 static inline int bio_integrity_add_page(struct bio *bio, struct page *page, 793 unsigned int len, unsigned int offset) 794 { 795 return 0; 796 } 797 798 static inline int bio_integrity_map_user(struct bio *bio, void __user *ubuf, 799 ssize_t len, u32 seed) 800 { 801 return -EINVAL; 802 } 803 804 #endif /* CONFIG_BLK_DEV_INTEGRITY */ 805 806 /* 807 * Mark a bio as polled. Note that for async polled IO, the caller must 808 * expect -EWOULDBLOCK if we cannot allocate a request (or other resources). 809 * We cannot block waiting for requests on polled IO, as those completions 810 * must be found by the caller. This is different than IRQ driven IO, where 811 * it's safe to wait for IO to complete. 812 */ 813 static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb) 814 { 815 bio->bi_opf |= REQ_POLLED; 816 if (kiocb->ki_flags & IOCB_NOWAIT) 817 bio->bi_opf |= REQ_NOWAIT; 818 } 819 820 static inline void bio_clear_polled(struct bio *bio) 821 { 822 bio->bi_opf &= ~REQ_POLLED; 823 } 824 825 struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev, 826 unsigned int nr_pages, blk_opf_t opf, gfp_t gfp); 827 828 #endif /* __LINUX_BIO_H */ 829