1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (C) 2011 Lawrence Livermore National Security, LLC. 24 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 25 * Written by Brian Behlendorf <behlendorf1@llnl.gov>. 26 * LLNL-CODE-403049. 27 */ 28 29 #ifndef _ZFS_BLKDEV_H 30 #define _ZFS_BLKDEV_H 31 32 #include <linux/blkdev.h> 33 #include <linux/backing-dev.h> 34 #include <linux/hdreg.h> 35 #include <linux/major.h> 36 #include <linux/msdos_fs.h> /* for SECTOR_* */ 37 #include <linux/bio.h> 38 39 #ifdef HAVE_BLK_MQ 40 #include <linux/blk-mq.h> 41 #endif 42 43 #ifndef HAVE_BLK_QUEUE_FLAG_SET 44 static inline void 45 blk_queue_flag_set(unsigned int flag, struct request_queue *q) 46 { 47 queue_flag_set(flag, q); 48 } 49 #endif 50 51 #ifndef HAVE_BLK_QUEUE_FLAG_CLEAR 52 static inline void 53 blk_queue_flag_clear(unsigned int flag, struct request_queue *q) 54 { 55 queue_flag_clear(flag, q); 56 } 57 #endif 58 59 /* 60 * 4.7 API, 61 * The blk_queue_write_cache() interface has replaced blk_queue_flush() 62 * interface. However, the new interface is GPL-only thus we implement 63 * our own trivial wrapper when the GPL-only version is detected. 64 * 65 * 2.6.36 - 4.6 API, 66 * The blk_queue_flush() interface has replaced blk_queue_ordered() 67 * interface. However, while the old interface was available to all the 68 * new one is GPL-only. Thus if the GPL-only version is detected we 69 * implement our own trivial helper. 70 */ 71 static inline void 72 blk_queue_set_write_cache(struct request_queue *q, bool wc, bool fua) 73 { 74 #if defined(HAVE_BLK_QUEUE_WRITE_CACHE_GPL_ONLY) 75 if (wc) 76 blk_queue_flag_set(QUEUE_FLAG_WC, q); 77 else 78 blk_queue_flag_clear(QUEUE_FLAG_WC, q); 79 if (fua) 80 blk_queue_flag_set(QUEUE_FLAG_FUA, q); 81 else 82 blk_queue_flag_clear(QUEUE_FLAG_FUA, q); 83 #elif defined(HAVE_BLK_QUEUE_WRITE_CACHE) 84 blk_queue_write_cache(q, wc, fua); 85 #elif defined(HAVE_BLK_QUEUE_FLUSH_GPL_ONLY) 86 if (wc) 87 q->flush_flags |= REQ_FLUSH; 88 if (fua) 89 q->flush_flags |= REQ_FUA; 90 #elif defined(HAVE_BLK_QUEUE_FLUSH) 91 blk_queue_flush(q, (wc ? REQ_FLUSH : 0) | (fua ? REQ_FUA : 0)); 92 #else 93 #error "Unsupported kernel" 94 #endif 95 } 96 97 static inline void 98 blk_queue_set_read_ahead(struct request_queue *q, unsigned long ra_pages) 99 { 100 #if !defined(HAVE_BLK_QUEUE_UPDATE_READAHEAD) && \ 101 !defined(HAVE_DISK_UPDATE_READAHEAD) 102 #ifdef HAVE_BLK_QUEUE_BDI_DYNAMIC 103 q->backing_dev_info->ra_pages = ra_pages; 104 #else 105 q->backing_dev_info.ra_pages = ra_pages; 106 #endif 107 #endif 108 } 109 110 #ifdef HAVE_BIO_BVEC_ITER 111 #define BIO_BI_SECTOR(bio) (bio)->bi_iter.bi_sector 112 #define BIO_BI_SIZE(bio) (bio)->bi_iter.bi_size 113 #define BIO_BI_IDX(bio) (bio)->bi_iter.bi_idx 114 #define BIO_BI_SKIP(bio) (bio)->bi_iter.bi_bvec_done 115 #define bio_for_each_segment4(bv, bvp, b, i) \ 116 bio_for_each_segment((bv), (b), (i)) 117 typedef struct bvec_iter bvec_iterator_t; 118 #else 119 #define BIO_BI_SECTOR(bio) (bio)->bi_sector 120 #define BIO_BI_SIZE(bio) (bio)->bi_size 121 #define BIO_BI_IDX(bio) (bio)->bi_idx 122 #define BIO_BI_SKIP(bio) (0) 123 #define bio_for_each_segment4(bv, bvp, b, i) \ 124 bio_for_each_segment((bvp), (b), (i)) 125 typedef int bvec_iterator_t; 126 #endif 127 128 static inline void 129 bio_set_flags_failfast(struct block_device *bdev, int *flags, bool dev, 130 bool transport, bool driver) 131 { 132 #ifdef CONFIG_BUG 133 /* 134 * Disable FAILFAST for loopback devices because of the 135 * following incorrect BUG_ON() in loop_make_request(). 136 * This support is also disabled for md devices because the 137 * test suite layers md devices on top of loopback devices. 138 * This may be removed when the loopback driver is fixed. 139 * 140 * BUG_ON(!lo || (rw != READ && rw != WRITE)); 141 */ 142 if ((MAJOR(bdev->bd_dev) == LOOP_MAJOR) || 143 (MAJOR(bdev->bd_dev) == MD_MAJOR)) 144 return; 145 146 #ifdef BLOCK_EXT_MAJOR 147 if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR) 148 return; 149 #endif /* BLOCK_EXT_MAJOR */ 150 #endif /* CONFIG_BUG */ 151 152 if (dev) 153 *flags |= REQ_FAILFAST_DEV; 154 if (transport) 155 *flags |= REQ_FAILFAST_TRANSPORT; 156 if (driver) 157 *flags |= REQ_FAILFAST_DRIVER; 158 } 159 160 /* 161 * Maximum disk label length, it may be undefined for some kernels. 162 */ 163 #if !defined(DISK_NAME_LEN) 164 #define DISK_NAME_LEN 32 165 #endif /* DISK_NAME_LEN */ 166 167 #ifdef HAVE_BIO_BI_STATUS 168 static inline int 169 bi_status_to_errno(blk_status_t status) 170 { 171 switch (status) { 172 case BLK_STS_OK: 173 return (0); 174 case BLK_STS_NOTSUPP: 175 return (EOPNOTSUPP); 176 case BLK_STS_TIMEOUT: 177 return (ETIMEDOUT); 178 case BLK_STS_NOSPC: 179 return (ENOSPC); 180 case BLK_STS_TRANSPORT: 181 return (ENOLINK); 182 case BLK_STS_TARGET: 183 return (EREMOTEIO); 184 #ifdef HAVE_BLK_STS_RESV_CONFLICT 185 case BLK_STS_RESV_CONFLICT: 186 #else 187 case BLK_STS_NEXUS: 188 #endif 189 return (EBADE); 190 case BLK_STS_MEDIUM: 191 return (ENODATA); 192 case BLK_STS_PROTECTION: 193 return (EILSEQ); 194 case BLK_STS_RESOURCE: 195 return (ENOMEM); 196 case BLK_STS_AGAIN: 197 return (EAGAIN); 198 case BLK_STS_IOERR: 199 return (EIO); 200 default: 201 return (EIO); 202 } 203 } 204 205 static inline blk_status_t 206 errno_to_bi_status(int error) 207 { 208 switch (error) { 209 case 0: 210 return (BLK_STS_OK); 211 case EOPNOTSUPP: 212 return (BLK_STS_NOTSUPP); 213 case ETIMEDOUT: 214 return (BLK_STS_TIMEOUT); 215 case ENOSPC: 216 return (BLK_STS_NOSPC); 217 case ENOLINK: 218 return (BLK_STS_TRANSPORT); 219 case EREMOTEIO: 220 return (BLK_STS_TARGET); 221 case EBADE: 222 #ifdef HAVE_BLK_STS_RESV_CONFLICT 223 return (BLK_STS_RESV_CONFLICT); 224 #else 225 return (BLK_STS_NEXUS); 226 #endif 227 case ENODATA: 228 return (BLK_STS_MEDIUM); 229 case EILSEQ: 230 return (BLK_STS_PROTECTION); 231 case ENOMEM: 232 return (BLK_STS_RESOURCE); 233 case EAGAIN: 234 return (BLK_STS_AGAIN); 235 case EIO: 236 return (BLK_STS_IOERR); 237 default: 238 return (BLK_STS_IOERR); 239 } 240 } 241 #endif /* HAVE_BIO_BI_STATUS */ 242 243 /* 244 * 4.3 API change 245 * The bio_endio() prototype changed slightly. These are helper 246 * macro's to ensure the prototype and invocation are handled. 247 */ 248 #ifdef HAVE_1ARG_BIO_END_IO_T 249 #ifdef HAVE_BIO_BI_STATUS 250 #define BIO_END_IO_ERROR(bio) bi_status_to_errno(bio->bi_status) 251 #define BIO_END_IO_PROTO(fn, x, z) static void fn(struct bio *x) 252 #define BIO_END_IO(bio, error) bio_set_bi_status(bio, error) 253 static inline void 254 bio_set_bi_status(struct bio *bio, int error) 255 { 256 ASSERT3S(error, <=, 0); 257 bio->bi_status = errno_to_bi_status(-error); 258 bio_endio(bio); 259 } 260 #else 261 #define BIO_END_IO_ERROR(bio) (-(bio->bi_error)) 262 #define BIO_END_IO_PROTO(fn, x, z) static void fn(struct bio *x) 263 #define BIO_END_IO(bio, error) bio_set_bi_error(bio, error) 264 static inline void 265 bio_set_bi_error(struct bio *bio, int error) 266 { 267 ASSERT3S(error, <=, 0); 268 bio->bi_error = error; 269 bio_endio(bio); 270 } 271 #endif /* HAVE_BIO_BI_STATUS */ 272 273 #else 274 #define BIO_END_IO_PROTO(fn, x, z) static void fn(struct bio *x, int z) 275 #define BIO_END_IO(bio, error) bio_endio(bio, error); 276 #endif /* HAVE_1ARG_BIO_END_IO_T */ 277 278 /* 279 * 5.15 MACRO, 280 * GD_DEAD 281 * 282 * 2.6.36 - 5.14 MACRO, 283 * GENHD_FL_UP 284 * 285 * Check the disk status and return B_TRUE if alive 286 * otherwise B_FALSE 287 */ 288 static inline boolean_t 289 zfs_check_disk_status(struct block_device *bdev) 290 { 291 #if defined(GENHD_FL_UP) 292 return (!!(bdev->bd_disk->flags & GENHD_FL_UP)); 293 #elif defined(GD_DEAD) 294 return (!test_bit(GD_DEAD, &bdev->bd_disk->state)); 295 #else 296 /* 297 * This is encountered if neither GENHD_FL_UP nor GD_DEAD is available in 298 * the kernel - likely due to an MACRO change that needs to be chased down. 299 */ 300 #error "Unsupported kernel: no usable disk status check" 301 #endif 302 } 303 304 /* 305 * 4.1 API, 306 * 3.10.0 CentOS 7.x API, 307 * blkdev_reread_part() 308 * 309 * For older kernels trigger a re-reading of the partition table by calling 310 * check_disk_change() which calls flush_disk() to invalidate the device. 311 * 312 * For newer kernels (as of 5.10), bdev_check_media_change is used, in favor of 313 * check_disk_change(), with the modification that invalidation is no longer 314 * forced. 315 */ 316 #ifdef HAVE_CHECK_DISK_CHANGE 317 #define zfs_check_media_change(bdev) check_disk_change(bdev) 318 #ifdef HAVE_BLKDEV_REREAD_PART 319 #define vdev_bdev_reread_part(bdev) blkdev_reread_part(bdev) 320 #else 321 #define vdev_bdev_reread_part(bdev) check_disk_change(bdev) 322 #endif /* HAVE_BLKDEV_REREAD_PART */ 323 #else 324 #ifdef HAVE_BDEV_CHECK_MEDIA_CHANGE 325 static inline int 326 zfs_check_media_change(struct block_device *bdev) 327 { 328 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK 329 struct gendisk *gd = bdev->bd_disk; 330 const struct block_device_operations *bdo = gd->fops; 331 #endif 332 333 if (!bdev_check_media_change(bdev)) 334 return (0); 335 336 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK 337 /* 338 * Force revalidation, to mimic the old behavior of 339 * check_disk_change() 340 */ 341 if (bdo->revalidate_disk) 342 bdo->revalidate_disk(gd); 343 #endif 344 345 return (0); 346 } 347 #define vdev_bdev_reread_part(bdev) zfs_check_media_change(bdev) 348 #elif defined(HAVE_DISK_CHECK_MEDIA_CHANGE) 349 #define vdev_bdev_reread_part(bdev) disk_check_media_change(bdev->bd_disk) 350 #define zfs_check_media_change(bdev) disk_check_media_change(bdev->bd_disk) 351 #else 352 /* 353 * This is encountered if check_disk_change() and bdev_check_media_change() 354 * are not available in the kernel - likely due to an API change that needs 355 * to be chased down. 356 */ 357 #error "Unsupported kernel: no usable disk change check" 358 #endif /* HAVE_BDEV_CHECK_MEDIA_CHANGE */ 359 #endif /* HAVE_CHECK_DISK_CHANGE */ 360 361 /* 362 * 2.6.27 API change 363 * The function was exported for use, prior to this it existed but the 364 * symbol was not exported. 365 * 366 * 4.4.0-6.21 API change for Ubuntu 367 * lookup_bdev() gained a second argument, FMODE_*, to check inode permissions. 368 * 369 * 5.11 API change 370 * Changed to take a dev_t argument which is set on success and return a 371 * non-zero error code on failure. 372 */ 373 static inline int 374 vdev_lookup_bdev(const char *path, dev_t *dev) 375 { 376 #if defined(HAVE_DEVT_LOOKUP_BDEV) 377 return (lookup_bdev(path, dev)); 378 #elif defined(HAVE_1ARG_LOOKUP_BDEV) 379 struct block_device *bdev = lookup_bdev(path); 380 if (IS_ERR(bdev)) 381 return (PTR_ERR(bdev)); 382 383 *dev = bdev->bd_dev; 384 bdput(bdev); 385 386 return (0); 387 #elif defined(HAVE_MODE_LOOKUP_BDEV) 388 struct block_device *bdev = lookup_bdev(path, FMODE_READ); 389 if (IS_ERR(bdev)) 390 return (PTR_ERR(bdev)); 391 392 *dev = bdev->bd_dev; 393 bdput(bdev); 394 395 return (0); 396 #else 397 #error "Unsupported kernel" 398 #endif 399 } 400 401 #if defined(HAVE_BLK_MODE_T) 402 #define blk_mode_is_open_write(flag) ((flag) & BLK_OPEN_WRITE) 403 #else 404 #define blk_mode_is_open_write(flag) ((flag) & FMODE_WRITE) 405 #endif 406 407 /* 408 * Kernels without bio_set_op_attrs use bi_rw for the bio flags. 409 */ 410 #if !defined(HAVE_BIO_SET_OP_ATTRS) 411 static inline void 412 bio_set_op_attrs(struct bio *bio, unsigned rw, unsigned flags) 413 { 414 #if defined(HAVE_BIO_BI_OPF) 415 bio->bi_opf = rw | flags; 416 #else 417 bio->bi_rw |= rw | flags; 418 #endif /* HAVE_BIO_BI_OPF */ 419 } 420 #endif 421 422 /* 423 * bio_set_flush - Set the appropriate flags in a bio to guarantee 424 * data are on non-volatile media on completion. 425 * 426 * 2.6.37 - 4.8 API, 427 * Introduce WRITE_FLUSH, WRITE_FUA, and WRITE_FLUSH_FUA flags as a 428 * replacement for WRITE_BARRIER to allow expressing richer semantics 429 * to the block layer. It's up to the block layer to implement the 430 * semantics correctly. Use the WRITE_FLUSH_FUA flag combination. 431 * 432 * 4.8 - 4.9 API, 433 * REQ_FLUSH was renamed to REQ_PREFLUSH. For consistency with previous 434 * OpenZFS releases, prefer the WRITE_FLUSH_FUA flag set if it's available. 435 * 436 * 4.10 API, 437 * The read/write flags and their modifiers, including WRITE_FLUSH, 438 * WRITE_FUA and WRITE_FLUSH_FUA were removed from fs.h in 439 * torvalds/linux@70fd7614 and replaced by direct flag modification 440 * of the REQ_ flags in bio->bi_opf. Use REQ_PREFLUSH. 441 */ 442 static inline void 443 bio_set_flush(struct bio *bio) 444 { 445 #if defined(HAVE_REQ_PREFLUSH) /* >= 4.10 */ 446 bio_set_op_attrs(bio, 0, REQ_PREFLUSH | REQ_OP_WRITE); 447 #elif defined(WRITE_FLUSH_FUA) /* >= 2.6.37 and <= 4.9 */ 448 bio_set_op_attrs(bio, 0, WRITE_FLUSH_FUA); 449 #else 450 #error "Allowing the build will cause bio_set_flush requests to be ignored." 451 #endif 452 } 453 454 /* 455 * 4.8 API, 456 * REQ_OP_FLUSH 457 * 458 * 4.8-rc0 - 4.8-rc1, 459 * REQ_PREFLUSH 460 * 461 * 2.6.36 - 4.7 API, 462 * REQ_FLUSH 463 * 464 * in all cases but may have a performance impact for some kernels. It 465 * has the advantage of minimizing kernel specific changes in the zvol code. 466 * 467 */ 468 static inline boolean_t 469 bio_is_flush(struct bio *bio) 470 { 471 #if defined(HAVE_REQ_OP_FLUSH) && defined(HAVE_BIO_BI_OPF) 472 return ((bio_op(bio) == REQ_OP_FLUSH) || (bio->bi_opf & REQ_PREFLUSH)); 473 #elif defined(HAVE_REQ_PREFLUSH) && defined(HAVE_BIO_BI_OPF) 474 return (bio->bi_opf & REQ_PREFLUSH); 475 #elif defined(HAVE_REQ_PREFLUSH) && !defined(HAVE_BIO_BI_OPF) 476 return (bio->bi_rw & REQ_PREFLUSH); 477 #elif defined(HAVE_REQ_FLUSH) 478 return (bio->bi_rw & REQ_FLUSH); 479 #else 480 #error "Unsupported kernel" 481 #endif 482 } 483 484 /* 485 * 4.8 API, 486 * REQ_FUA flag moved to bio->bi_opf 487 * 488 * 2.6.x - 4.7 API, 489 * REQ_FUA 490 */ 491 static inline boolean_t 492 bio_is_fua(struct bio *bio) 493 { 494 #if defined(HAVE_BIO_BI_OPF) 495 return (bio->bi_opf & REQ_FUA); 496 #elif defined(REQ_FUA) 497 return (bio->bi_rw & REQ_FUA); 498 #else 499 #error "Allowing the build will cause fua requests to be ignored." 500 #endif 501 } 502 503 /* 504 * 4.8 API, 505 * REQ_OP_DISCARD 506 * 507 * 2.6.36 - 4.7 API, 508 * REQ_DISCARD 509 * 510 * In all cases the normal I/O path is used for discards. The only 511 * difference is how the kernel tags individual I/Os as discards. 512 */ 513 static inline boolean_t 514 bio_is_discard(struct bio *bio) 515 { 516 #if defined(HAVE_REQ_OP_DISCARD) 517 return (bio_op(bio) == REQ_OP_DISCARD); 518 #elif defined(HAVE_REQ_DISCARD) 519 return (bio->bi_rw & REQ_DISCARD); 520 #else 521 #error "Unsupported kernel" 522 #endif 523 } 524 525 /* 526 * 4.8 API, 527 * REQ_OP_SECURE_ERASE 528 * 529 * 2.6.36 - 4.7 API, 530 * REQ_SECURE 531 */ 532 static inline boolean_t 533 bio_is_secure_erase(struct bio *bio) 534 { 535 #if defined(HAVE_REQ_OP_SECURE_ERASE) 536 return (bio_op(bio) == REQ_OP_SECURE_ERASE); 537 #elif defined(REQ_SECURE) 538 return (bio->bi_rw & REQ_SECURE); 539 #else 540 return (0); 541 #endif 542 } 543 544 /* 545 * 2.6.33 API change 546 * Discard granularity and alignment restrictions may now be set. For 547 * older kernels which do not support this it is safe to skip it. 548 */ 549 static inline void 550 blk_queue_discard_granularity(struct request_queue *q, unsigned int dg) 551 { 552 q->limits.discard_granularity = dg; 553 } 554 555 /* 556 * 5.19 API, 557 * bdev_max_discard_sectors() 558 * 559 * 2.6.32 API, 560 * blk_queue_discard() 561 */ 562 static inline boolean_t 563 bdev_discard_supported(struct block_device *bdev) 564 { 565 #if defined(HAVE_BDEV_MAX_DISCARD_SECTORS) 566 return (!!bdev_max_discard_sectors(bdev)); 567 #elif defined(HAVE_BLK_QUEUE_DISCARD) 568 return (!!blk_queue_discard(bdev_get_queue(bdev))); 569 #else 570 #error "Unsupported kernel" 571 #endif 572 } 573 574 /* 575 * 5.19 API, 576 * bdev_max_secure_erase_sectors() 577 * 578 * 4.8 API, 579 * blk_queue_secure_erase() 580 * 581 * 2.6.36 - 4.7 API, 582 * blk_queue_secdiscard() 583 */ 584 static inline boolean_t 585 bdev_secure_discard_supported(struct block_device *bdev) 586 { 587 #if defined(HAVE_BDEV_MAX_SECURE_ERASE_SECTORS) 588 return (!!bdev_max_secure_erase_sectors(bdev)); 589 #elif defined(HAVE_BLK_QUEUE_SECURE_ERASE) 590 return (!!blk_queue_secure_erase(bdev_get_queue(bdev))); 591 #elif defined(HAVE_BLK_QUEUE_SECDISCARD) 592 return (!!blk_queue_secdiscard(bdev_get_queue(bdev))); 593 #else 594 #error "Unsupported kernel" 595 #endif 596 } 597 598 /* 599 * A common holder for vdev_bdev_open() is used to relax the exclusive open 600 * semantics slightly. Internal vdev disk callers may pass VDEV_HOLDER to 601 * allow them to open the device multiple times. Other kernel callers and 602 * user space processes which don't pass this value will get EBUSY. This is 603 * currently required for the correct operation of hot spares. 604 */ 605 #define VDEV_HOLDER ((void *)0x2401de7) 606 607 static inline unsigned long 608 blk_generic_start_io_acct(struct request_queue *q __attribute__((unused)), 609 struct gendisk *disk __attribute__((unused)), 610 int rw __attribute__((unused)), struct bio *bio) 611 { 612 #if defined(HAVE_BDEV_IO_ACCT_63) 613 return (bdev_start_io_acct(bio->bi_bdev, bio_op(bio), 614 jiffies)); 615 #elif defined(HAVE_BDEV_IO_ACCT_OLD) 616 return (bdev_start_io_acct(bio->bi_bdev, bio_sectors(bio), 617 bio_op(bio), jiffies)); 618 #elif defined(HAVE_DISK_IO_ACCT) 619 return (disk_start_io_acct(disk, bio_sectors(bio), bio_op(bio))); 620 #elif defined(HAVE_BIO_IO_ACCT) 621 return (bio_start_io_acct(bio)); 622 #elif defined(HAVE_GENERIC_IO_ACCT_3ARG) 623 unsigned long start_time = jiffies; 624 generic_start_io_acct(rw, bio_sectors(bio), &disk->part0); 625 return (start_time); 626 #elif defined(HAVE_GENERIC_IO_ACCT_4ARG) 627 unsigned long start_time = jiffies; 628 generic_start_io_acct(q, rw, bio_sectors(bio), &disk->part0); 629 return (start_time); 630 #else 631 /* Unsupported */ 632 return (0); 633 #endif 634 } 635 636 static inline void 637 blk_generic_end_io_acct(struct request_queue *q __attribute__((unused)), 638 struct gendisk *disk __attribute__((unused)), 639 int rw __attribute__((unused)), struct bio *bio, unsigned long start_time) 640 { 641 #if defined(HAVE_BDEV_IO_ACCT_63) 642 bdev_end_io_acct(bio->bi_bdev, bio_op(bio), bio_sectors(bio), 643 start_time); 644 #elif defined(HAVE_BDEV_IO_ACCT_OLD) 645 bdev_end_io_acct(bio->bi_bdev, bio_op(bio), start_time); 646 #elif defined(HAVE_DISK_IO_ACCT) 647 disk_end_io_acct(disk, bio_op(bio), start_time); 648 #elif defined(HAVE_BIO_IO_ACCT) 649 bio_end_io_acct(bio, start_time); 650 #elif defined(HAVE_GENERIC_IO_ACCT_3ARG) 651 generic_end_io_acct(rw, &disk->part0, start_time); 652 #elif defined(HAVE_GENERIC_IO_ACCT_4ARG) 653 generic_end_io_acct(q, rw, &disk->part0, start_time); 654 #endif 655 } 656 657 #ifndef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS 658 static inline struct request_queue * 659 blk_generic_alloc_queue(make_request_fn make_request, int node_id) 660 { 661 #if defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN) 662 return (blk_alloc_queue(make_request, node_id)); 663 #elif defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN_RH) 664 return (blk_alloc_queue_rh(make_request, node_id)); 665 #else 666 struct request_queue *q = blk_alloc_queue(GFP_KERNEL); 667 if (q != NULL) 668 blk_queue_make_request(q, make_request); 669 670 return (q); 671 #endif 672 } 673 #endif /* !HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */ 674 675 /* 676 * All the io_*() helper functions below can operate on a bio, or a rq, but 677 * not both. The older submit_bio() codepath will pass a bio, and the 678 * newer blk-mq codepath will pass a rq. 679 */ 680 static inline int 681 io_data_dir(struct bio *bio, struct request *rq) 682 { 683 #ifdef HAVE_BLK_MQ 684 if (rq != NULL) { 685 if (op_is_write(req_op(rq))) { 686 return (WRITE); 687 } else { 688 return (READ); 689 } 690 } 691 #else 692 ASSERT3P(rq, ==, NULL); 693 #endif 694 return (bio_data_dir(bio)); 695 } 696 697 static inline int 698 io_is_flush(struct bio *bio, struct request *rq) 699 { 700 #ifdef HAVE_BLK_MQ 701 if (rq != NULL) 702 return (req_op(rq) == REQ_OP_FLUSH); 703 #else 704 ASSERT3P(rq, ==, NULL); 705 #endif 706 return (bio_is_flush(bio)); 707 } 708 709 static inline int 710 io_is_discard(struct bio *bio, struct request *rq) 711 { 712 #ifdef HAVE_BLK_MQ 713 if (rq != NULL) 714 return (req_op(rq) == REQ_OP_DISCARD); 715 #else 716 ASSERT3P(rq, ==, NULL); 717 #endif 718 return (bio_is_discard(bio)); 719 } 720 721 static inline int 722 io_is_secure_erase(struct bio *bio, struct request *rq) 723 { 724 #ifdef HAVE_BLK_MQ 725 if (rq != NULL) 726 return (req_op(rq) == REQ_OP_SECURE_ERASE); 727 #else 728 ASSERT3P(rq, ==, NULL); 729 #endif 730 return (bio_is_secure_erase(bio)); 731 } 732 733 static inline int 734 io_is_fua(struct bio *bio, struct request *rq) 735 { 736 #ifdef HAVE_BLK_MQ 737 if (rq != NULL) 738 return (rq->cmd_flags & REQ_FUA); 739 #else 740 ASSERT3P(rq, ==, NULL); 741 #endif 742 return (bio_is_fua(bio)); 743 } 744 745 746 static inline uint64_t 747 io_offset(struct bio *bio, struct request *rq) 748 { 749 #ifdef HAVE_BLK_MQ 750 if (rq != NULL) 751 return (blk_rq_pos(rq) << 9); 752 #else 753 ASSERT3P(rq, ==, NULL); 754 #endif 755 return (BIO_BI_SECTOR(bio) << 9); 756 } 757 758 static inline uint64_t 759 io_size(struct bio *bio, struct request *rq) 760 { 761 #ifdef HAVE_BLK_MQ 762 if (rq != NULL) 763 return (blk_rq_bytes(rq)); 764 #else 765 ASSERT3P(rq, ==, NULL); 766 #endif 767 return (BIO_BI_SIZE(bio)); 768 } 769 770 static inline int 771 io_has_data(struct bio *bio, struct request *rq) 772 { 773 #ifdef HAVE_BLK_MQ 774 if (rq != NULL) 775 return (bio_has_data(rq->bio)); 776 #else 777 ASSERT3P(rq, ==, NULL); 778 #endif 779 return (bio_has_data(bio)); 780 } 781 #endif /* _ZFS_BLKDEV_H */ 782