1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Zoned block device handling 4 * 5 * Copyright (c) 2015, Hannes Reinecke 6 * Copyright (c) 2015, SUSE Linux GmbH 7 * 8 * Copyright (c) 2016, Damien Le Moal 9 * Copyright (c) 2016, Western Digital 10 */ 11 12 #include <linux/kernel.h> 13 #include <linux/module.h> 14 #include <linux/rbtree.h> 15 #include <linux/blkdev.h> 16 #include <linux/blk-mq.h> 17 #include <linux/mm.h> 18 #include <linux/vmalloc.h> 19 #include <linux/sched/mm.h> 20 21 #include "blk.h" 22 23 #define ZONE_COND_NAME(name) [BLK_ZONE_COND_##name] = #name 24 static const char *const zone_cond_name[] = { 25 ZONE_COND_NAME(NOT_WP), 26 ZONE_COND_NAME(EMPTY), 27 ZONE_COND_NAME(IMP_OPEN), 28 ZONE_COND_NAME(EXP_OPEN), 29 ZONE_COND_NAME(CLOSED), 30 ZONE_COND_NAME(READONLY), 31 ZONE_COND_NAME(FULL), 32 ZONE_COND_NAME(OFFLINE), 33 }; 34 #undef ZONE_COND_NAME 35 36 /** 37 * blk_zone_cond_str - Return string XXX in BLK_ZONE_COND_XXX. 38 * @zone_cond: BLK_ZONE_COND_XXX. 39 * 40 * Description: Centralize block layer function to convert BLK_ZONE_COND_XXX 41 * into string format. Useful in the debugging and tracing zone conditions. For 42 * invalid BLK_ZONE_COND_XXX it returns string "UNKNOWN". 43 */ 44 const char *blk_zone_cond_str(enum blk_zone_cond zone_cond) 45 { 46 static const char *zone_cond_str = "UNKNOWN"; 47 48 if (zone_cond < ARRAY_SIZE(zone_cond_name) && zone_cond_name[zone_cond]) 49 zone_cond_str = zone_cond_name[zone_cond]; 50 51 return zone_cond_str; 52 } 53 EXPORT_SYMBOL_GPL(blk_zone_cond_str); 54 55 /* 56 * Return true if a request is a write requests that needs zone write locking. 57 */ 58 bool blk_req_needs_zone_write_lock(struct request *rq) 59 { 60 if (!rq->q->seq_zones_wlock) 61 return false; 62 63 if (blk_rq_is_passthrough(rq)) 64 return false; 65 66 switch (req_op(rq)) { 67 case REQ_OP_WRITE_ZEROES: 68 case REQ_OP_WRITE_SAME: 69 case REQ_OP_WRITE: 70 return blk_rq_zone_is_seq(rq); 71 default: 72 return false; 73 } 74 } 75 EXPORT_SYMBOL_GPL(blk_req_needs_zone_write_lock); 76 77 bool blk_req_zone_write_trylock(struct request *rq) 78 { 79 unsigned int zno = blk_rq_zone_no(rq); 80 81 if (test_and_set_bit(zno, rq->q->seq_zones_wlock)) 82 return false; 83 84 WARN_ON_ONCE(rq->rq_flags & RQF_ZONE_WRITE_LOCKED); 85 rq->rq_flags |= RQF_ZONE_WRITE_LOCKED; 86 87 return true; 88 } 89 EXPORT_SYMBOL_GPL(blk_req_zone_write_trylock); 90 91 void __blk_req_zone_write_lock(struct request *rq) 92 { 93 if (WARN_ON_ONCE(test_and_set_bit(blk_rq_zone_no(rq), 94 rq->q->seq_zones_wlock))) 95 return; 96 97 WARN_ON_ONCE(rq->rq_flags & RQF_ZONE_WRITE_LOCKED); 98 rq->rq_flags |= RQF_ZONE_WRITE_LOCKED; 99 } 100 EXPORT_SYMBOL_GPL(__blk_req_zone_write_lock); 101 102 void __blk_req_zone_write_unlock(struct request *rq) 103 { 104 rq->rq_flags &= ~RQF_ZONE_WRITE_LOCKED; 105 if (rq->q->seq_zones_wlock) 106 WARN_ON_ONCE(!test_and_clear_bit(blk_rq_zone_no(rq), 107 rq->q->seq_zones_wlock)); 108 } 109 EXPORT_SYMBOL_GPL(__blk_req_zone_write_unlock); 110 111 /** 112 * blkdev_nr_zones - Get number of zones 113 * @disk: Target gendisk 114 * 115 * Return the total number of zones of a zoned block device. For a block 116 * device without zone capabilities, the number of zones is always 0. 117 */ 118 unsigned int blkdev_nr_zones(struct gendisk *disk) 119 { 120 sector_t zone_sectors = blk_queue_zone_sectors(disk->queue); 121 122 if (!blk_queue_is_zoned(disk->queue)) 123 return 0; 124 return (get_capacity(disk) + zone_sectors - 1) >> ilog2(zone_sectors); 125 } 126 EXPORT_SYMBOL_GPL(blkdev_nr_zones); 127 128 /** 129 * blkdev_report_zones - Get zones information 130 * @bdev: Target block device 131 * @sector: Sector from which to report zones 132 * @nr_zones: Maximum number of zones to report 133 * @cb: Callback function called for each reported zone 134 * @data: Private data for the callback 135 * 136 * Description: 137 * Get zone information starting from the zone containing @sector for at most 138 * @nr_zones, and call @cb for each zone reported by the device. 139 * To report all zones in a device starting from @sector, the BLK_ALL_ZONES 140 * constant can be passed to @nr_zones. 141 * Returns the number of zones reported by the device, or a negative errno 142 * value in case of failure. 143 * 144 * Note: The caller must use memalloc_noXX_save/restore() calls to control 145 * memory allocations done within this function. 146 */ 147 int blkdev_report_zones(struct block_device *bdev, sector_t sector, 148 unsigned int nr_zones, report_zones_cb cb, void *data) 149 { 150 struct gendisk *disk = bdev->bd_disk; 151 sector_t capacity = get_capacity(disk); 152 153 if (!blk_queue_is_zoned(bdev_get_queue(bdev)) || 154 WARN_ON_ONCE(!disk->fops->report_zones)) 155 return -EOPNOTSUPP; 156 157 if (!nr_zones || sector >= capacity) 158 return 0; 159 160 return disk->fops->report_zones(disk, sector, nr_zones, cb, data); 161 } 162 EXPORT_SYMBOL_GPL(blkdev_report_zones); 163 164 static inline unsigned long *blk_alloc_zone_bitmap(int node, 165 unsigned int nr_zones) 166 { 167 return kcalloc_node(BITS_TO_LONGS(nr_zones), sizeof(unsigned long), 168 GFP_NOIO, node); 169 } 170 171 static int blk_zone_need_reset_cb(struct blk_zone *zone, unsigned int idx, 172 void *data) 173 { 174 /* 175 * For an all-zones reset, ignore conventional, empty, read-only 176 * and offline zones. 177 */ 178 switch (zone->cond) { 179 case BLK_ZONE_COND_NOT_WP: 180 case BLK_ZONE_COND_EMPTY: 181 case BLK_ZONE_COND_READONLY: 182 case BLK_ZONE_COND_OFFLINE: 183 return 0; 184 default: 185 set_bit(idx, (unsigned long *)data); 186 return 0; 187 } 188 } 189 190 static int blkdev_zone_reset_all_emulated(struct block_device *bdev, 191 gfp_t gfp_mask) 192 { 193 struct request_queue *q = bdev_get_queue(bdev); 194 sector_t capacity = get_capacity(bdev->bd_disk); 195 sector_t zone_sectors = blk_queue_zone_sectors(q); 196 unsigned long *need_reset; 197 struct bio *bio = NULL; 198 sector_t sector = 0; 199 int ret; 200 201 need_reset = blk_alloc_zone_bitmap(q->node, q->nr_zones); 202 if (!need_reset) 203 return -ENOMEM; 204 205 ret = bdev->bd_disk->fops->report_zones(bdev->bd_disk, 0, 206 q->nr_zones, blk_zone_need_reset_cb, 207 need_reset); 208 if (ret < 0) 209 goto out_free_need_reset; 210 211 ret = 0; 212 while (sector < capacity) { 213 if (!test_bit(blk_queue_zone_no(q, sector), need_reset)) { 214 sector += zone_sectors; 215 continue; 216 } 217 218 bio = blk_next_bio(bio, bdev, 0, REQ_OP_ZONE_RESET | REQ_SYNC, 219 gfp_mask); 220 bio->bi_iter.bi_sector = sector; 221 sector += zone_sectors; 222 223 /* This may take a while, so be nice to others */ 224 cond_resched(); 225 } 226 227 if (bio) { 228 ret = submit_bio_wait(bio); 229 bio_put(bio); 230 } 231 232 out_free_need_reset: 233 kfree(need_reset); 234 return ret; 235 } 236 237 static int blkdev_zone_reset_all(struct block_device *bdev, gfp_t gfp_mask) 238 { 239 struct bio bio; 240 241 bio_init(&bio, NULL, 0); 242 bio_set_dev(&bio, bdev); 243 bio.bi_opf = REQ_OP_ZONE_RESET_ALL | REQ_SYNC; 244 245 return submit_bio_wait(&bio); 246 } 247 248 /** 249 * blkdev_zone_mgmt - Execute a zone management operation on a range of zones 250 * @bdev: Target block device 251 * @op: Operation to be performed on the zones 252 * @sector: Start sector of the first zone to operate on 253 * @nr_sectors: Number of sectors, should be at least the length of one zone and 254 * must be zone size aligned. 255 * @gfp_mask: Memory allocation flags (for bio_alloc) 256 * 257 * Description: 258 * Perform the specified operation on the range of zones specified by 259 * @sector..@sector+@nr_sectors. Specifying the entire disk sector range 260 * is valid, but the specified range should not contain conventional zones. 261 * The operation to execute on each zone can be a zone reset, open, close 262 * or finish request. 263 */ 264 int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op, 265 sector_t sector, sector_t nr_sectors, 266 gfp_t gfp_mask) 267 { 268 struct request_queue *q = bdev_get_queue(bdev); 269 sector_t zone_sectors = blk_queue_zone_sectors(q); 270 sector_t capacity = get_capacity(bdev->bd_disk); 271 sector_t end_sector = sector + nr_sectors; 272 struct bio *bio = NULL; 273 int ret = 0; 274 275 if (!blk_queue_is_zoned(q)) 276 return -EOPNOTSUPP; 277 278 if (bdev_read_only(bdev)) 279 return -EPERM; 280 281 if (!op_is_zone_mgmt(op)) 282 return -EOPNOTSUPP; 283 284 if (end_sector <= sector || end_sector > capacity) 285 /* Out of range */ 286 return -EINVAL; 287 288 /* Check alignment (handle eventual smaller last zone) */ 289 if (sector & (zone_sectors - 1)) 290 return -EINVAL; 291 292 if ((nr_sectors & (zone_sectors - 1)) && end_sector != capacity) 293 return -EINVAL; 294 295 /* 296 * In the case of a zone reset operation over all zones, 297 * REQ_OP_ZONE_RESET_ALL can be used with devices supporting this 298 * command. For other devices, we emulate this command behavior by 299 * identifying the zones needing a reset. 300 */ 301 if (op == REQ_OP_ZONE_RESET && sector == 0 && nr_sectors == capacity) { 302 if (!blk_queue_zone_resetall(q)) 303 return blkdev_zone_reset_all_emulated(bdev, gfp_mask); 304 return blkdev_zone_reset_all(bdev, gfp_mask); 305 } 306 307 while (sector < end_sector) { 308 bio = blk_next_bio(bio, bdev, 0, op | REQ_SYNC, gfp_mask); 309 bio->bi_iter.bi_sector = sector; 310 sector += zone_sectors; 311 312 /* This may take a while, so be nice to others */ 313 cond_resched(); 314 } 315 316 ret = submit_bio_wait(bio); 317 bio_put(bio); 318 319 return ret; 320 } 321 EXPORT_SYMBOL_GPL(blkdev_zone_mgmt); 322 323 struct zone_report_args { 324 struct blk_zone __user *zones; 325 }; 326 327 static int blkdev_copy_zone_to_user(struct blk_zone *zone, unsigned int idx, 328 void *data) 329 { 330 struct zone_report_args *args = data; 331 332 if (copy_to_user(&args->zones[idx], zone, sizeof(struct blk_zone))) 333 return -EFAULT; 334 return 0; 335 } 336 337 /* 338 * BLKREPORTZONE ioctl processing. 339 * Called from blkdev_ioctl. 340 */ 341 int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode, 342 unsigned int cmd, unsigned long arg) 343 { 344 void __user *argp = (void __user *)arg; 345 struct zone_report_args args; 346 struct request_queue *q; 347 struct blk_zone_report rep; 348 int ret; 349 350 if (!argp) 351 return -EINVAL; 352 353 q = bdev_get_queue(bdev); 354 if (!q) 355 return -ENXIO; 356 357 if (!blk_queue_is_zoned(q)) 358 return -ENOTTY; 359 360 if (copy_from_user(&rep, argp, sizeof(struct blk_zone_report))) 361 return -EFAULT; 362 363 if (!rep.nr_zones) 364 return -EINVAL; 365 366 args.zones = argp + sizeof(struct blk_zone_report); 367 ret = blkdev_report_zones(bdev, rep.sector, rep.nr_zones, 368 blkdev_copy_zone_to_user, &args); 369 if (ret < 0) 370 return ret; 371 372 rep.nr_zones = ret; 373 rep.flags = BLK_ZONE_REP_CAPACITY; 374 if (copy_to_user(argp, &rep, sizeof(struct blk_zone_report))) 375 return -EFAULT; 376 return 0; 377 } 378 379 static int blkdev_truncate_zone_range(struct block_device *bdev, fmode_t mode, 380 const struct blk_zone_range *zrange) 381 { 382 loff_t start, end; 383 384 if (zrange->sector + zrange->nr_sectors <= zrange->sector || 385 zrange->sector + zrange->nr_sectors > get_capacity(bdev->bd_disk)) 386 /* Out of range */ 387 return -EINVAL; 388 389 start = zrange->sector << SECTOR_SHIFT; 390 end = ((zrange->sector + zrange->nr_sectors) << SECTOR_SHIFT) - 1; 391 392 return truncate_bdev_range(bdev, mode, start, end); 393 } 394 395 /* 396 * BLKRESETZONE, BLKOPENZONE, BLKCLOSEZONE and BLKFINISHZONE ioctl processing. 397 * Called from blkdev_ioctl. 398 */ 399 int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode, 400 unsigned int cmd, unsigned long arg) 401 { 402 void __user *argp = (void __user *)arg; 403 struct request_queue *q; 404 struct blk_zone_range zrange; 405 enum req_opf op; 406 int ret; 407 408 if (!argp) 409 return -EINVAL; 410 411 q = bdev_get_queue(bdev); 412 if (!q) 413 return -ENXIO; 414 415 if (!blk_queue_is_zoned(q)) 416 return -ENOTTY; 417 418 if (!(mode & FMODE_WRITE)) 419 return -EBADF; 420 421 if (copy_from_user(&zrange, argp, sizeof(struct blk_zone_range))) 422 return -EFAULT; 423 424 switch (cmd) { 425 case BLKRESETZONE: 426 op = REQ_OP_ZONE_RESET; 427 428 /* Invalidate the page cache, including dirty pages. */ 429 filemap_invalidate_lock(bdev->bd_inode->i_mapping); 430 ret = blkdev_truncate_zone_range(bdev, mode, &zrange); 431 if (ret) 432 goto fail; 433 break; 434 case BLKOPENZONE: 435 op = REQ_OP_ZONE_OPEN; 436 break; 437 case BLKCLOSEZONE: 438 op = REQ_OP_ZONE_CLOSE; 439 break; 440 case BLKFINISHZONE: 441 op = REQ_OP_ZONE_FINISH; 442 break; 443 default: 444 return -ENOTTY; 445 } 446 447 ret = blkdev_zone_mgmt(bdev, op, zrange.sector, zrange.nr_sectors, 448 GFP_KERNEL); 449 450 fail: 451 if (cmd == BLKRESETZONE) 452 filemap_invalidate_unlock(bdev->bd_inode->i_mapping); 453 454 return ret; 455 } 456 457 void blk_queue_free_zone_bitmaps(struct request_queue *q) 458 { 459 kfree(q->conv_zones_bitmap); 460 q->conv_zones_bitmap = NULL; 461 kfree(q->seq_zones_wlock); 462 q->seq_zones_wlock = NULL; 463 } 464 465 struct blk_revalidate_zone_args { 466 struct gendisk *disk; 467 unsigned long *conv_zones_bitmap; 468 unsigned long *seq_zones_wlock; 469 unsigned int nr_zones; 470 sector_t zone_sectors; 471 sector_t sector; 472 }; 473 474 /* 475 * Helper function to check the validity of zones of a zoned block device. 476 */ 477 static int blk_revalidate_zone_cb(struct blk_zone *zone, unsigned int idx, 478 void *data) 479 { 480 struct blk_revalidate_zone_args *args = data; 481 struct gendisk *disk = args->disk; 482 struct request_queue *q = disk->queue; 483 sector_t capacity = get_capacity(disk); 484 485 /* 486 * All zones must have the same size, with the exception on an eventual 487 * smaller last zone. 488 */ 489 if (zone->start == 0) { 490 if (zone->len == 0 || !is_power_of_2(zone->len)) { 491 pr_warn("%s: Invalid zoned device with non power of two zone size (%llu)\n", 492 disk->disk_name, zone->len); 493 return -ENODEV; 494 } 495 496 args->zone_sectors = zone->len; 497 args->nr_zones = (capacity + zone->len - 1) >> ilog2(zone->len); 498 } else if (zone->start + args->zone_sectors < capacity) { 499 if (zone->len != args->zone_sectors) { 500 pr_warn("%s: Invalid zoned device with non constant zone size\n", 501 disk->disk_name); 502 return -ENODEV; 503 } 504 } else { 505 if (zone->len > args->zone_sectors) { 506 pr_warn("%s: Invalid zoned device with larger last zone size\n", 507 disk->disk_name); 508 return -ENODEV; 509 } 510 } 511 512 /* Check for holes in the zone report */ 513 if (zone->start != args->sector) { 514 pr_warn("%s: Zone gap at sectors %llu..%llu\n", 515 disk->disk_name, args->sector, zone->start); 516 return -ENODEV; 517 } 518 519 /* Check zone type */ 520 switch (zone->type) { 521 case BLK_ZONE_TYPE_CONVENTIONAL: 522 if (!args->conv_zones_bitmap) { 523 args->conv_zones_bitmap = 524 blk_alloc_zone_bitmap(q->node, args->nr_zones); 525 if (!args->conv_zones_bitmap) 526 return -ENOMEM; 527 } 528 set_bit(idx, args->conv_zones_bitmap); 529 break; 530 case BLK_ZONE_TYPE_SEQWRITE_REQ: 531 case BLK_ZONE_TYPE_SEQWRITE_PREF: 532 if (!args->seq_zones_wlock) { 533 args->seq_zones_wlock = 534 blk_alloc_zone_bitmap(q->node, args->nr_zones); 535 if (!args->seq_zones_wlock) 536 return -ENOMEM; 537 } 538 break; 539 default: 540 pr_warn("%s: Invalid zone type 0x%x at sectors %llu\n", 541 disk->disk_name, (int)zone->type, zone->start); 542 return -ENODEV; 543 } 544 545 args->sector += zone->len; 546 return 0; 547 } 548 549 /** 550 * blk_revalidate_disk_zones - (re)allocate and initialize zone bitmaps 551 * @disk: Target disk 552 * @update_driver_data: Callback to update driver data on the frozen disk 553 * 554 * Helper function for low-level device drivers to (re) allocate and initialize 555 * a disk request queue zone bitmaps. This functions should normally be called 556 * within the disk ->revalidate method for blk-mq based drivers. For BIO based 557 * drivers only q->nr_zones needs to be updated so that the sysfs exposed value 558 * is correct. 559 * If the @update_driver_data callback function is not NULL, the callback is 560 * executed with the device request queue frozen after all zones have been 561 * checked. 562 */ 563 int blk_revalidate_disk_zones(struct gendisk *disk, 564 void (*update_driver_data)(struct gendisk *disk)) 565 { 566 struct request_queue *q = disk->queue; 567 struct blk_revalidate_zone_args args = { 568 .disk = disk, 569 }; 570 unsigned int noio_flag; 571 int ret; 572 573 if (WARN_ON_ONCE(!blk_queue_is_zoned(q))) 574 return -EIO; 575 if (WARN_ON_ONCE(!queue_is_mq(q))) 576 return -EIO; 577 578 if (!get_capacity(disk)) 579 return -EIO; 580 581 /* 582 * Ensure that all memory allocations in this context are done as if 583 * GFP_NOIO was specified. 584 */ 585 noio_flag = memalloc_noio_save(); 586 ret = disk->fops->report_zones(disk, 0, UINT_MAX, 587 blk_revalidate_zone_cb, &args); 588 if (!ret) { 589 pr_warn("%s: No zones reported\n", disk->disk_name); 590 ret = -ENODEV; 591 } 592 memalloc_noio_restore(noio_flag); 593 594 /* 595 * If zones where reported, make sure that the entire disk capacity 596 * has been checked. 597 */ 598 if (ret > 0 && args.sector != get_capacity(disk)) { 599 pr_warn("%s: Missing zones from sector %llu\n", 600 disk->disk_name, args.sector); 601 ret = -ENODEV; 602 } 603 604 /* 605 * Install the new bitmaps and update nr_zones only once the queue is 606 * stopped and all I/Os are completed (i.e. a scheduler is not 607 * referencing the bitmaps). 608 */ 609 blk_mq_freeze_queue(q); 610 if (ret > 0) { 611 blk_queue_chunk_sectors(q, args.zone_sectors); 612 q->nr_zones = args.nr_zones; 613 swap(q->seq_zones_wlock, args.seq_zones_wlock); 614 swap(q->conv_zones_bitmap, args.conv_zones_bitmap); 615 if (update_driver_data) 616 update_driver_data(disk); 617 ret = 0; 618 } else { 619 pr_warn("%s: failed to revalidate zones\n", disk->disk_name); 620 blk_queue_free_zone_bitmaps(q); 621 } 622 blk_mq_unfreeze_queue(q); 623 624 kfree(args.seq_zones_wlock); 625 kfree(args.conv_zones_bitmap); 626 return ret; 627 } 628 EXPORT_SYMBOL_GPL(blk_revalidate_disk_zones); 629 630 void blk_queue_clear_zone_settings(struct request_queue *q) 631 { 632 blk_mq_freeze_queue(q); 633 634 blk_queue_free_zone_bitmaps(q); 635 blk_queue_flag_clear(QUEUE_FLAG_ZONE_RESETALL, q); 636 q->required_elevator_features &= ~ELEVATOR_F_ZBD_SEQ_WRITE; 637 q->nr_zones = 0; 638 q->max_open_zones = 0; 639 q->max_active_zones = 0; 640 q->limits.chunk_sectors = 0; 641 q->limits.zone_write_granularity = 0; 642 q->limits.max_zone_append_sectors = 0; 643 644 blk_mq_unfreeze_queue(q); 645 } 646