1 /* 2 * Copyright (c) 2013-2018 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@dragonflybsd.org> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #include "hammer2.h" 36 37 #define HAMMER2_DOP_READ 1 38 #define HAMMER2_DOP_NEW 2 39 #define HAMMER2_DOP_NEWNZ 3 40 #define HAMMER2_DOP_READQ 4 41 42 /* 43 * Implements an abstraction layer for synchronous and asynchronous 44 * buffered device I/O. Can be used as an OS-abstraction but the main 45 * purpose is to allow larger buffers to be used against hammer2_chain's 46 * using smaller allocations, without causing deadlocks. 47 * 48 * The DIOs also record temporary state with limited persistence. This 49 * feature is used to keep track of dedupable blocks. 50 */ 51 static int hammer2_io_cleanup_callback(hammer2_io_t *dio, void *arg); 52 static void dio_write_stats_update(hammer2_io_t *dio, struct buf *bp); 53 54 static int 55 hammer2_io_cmp(hammer2_io_t *io1, hammer2_io_t *io2) 56 { 57 if (io1->pbase < io2->pbase) 58 return(-1); 59 if (io1->pbase > io2->pbase) 60 return(1); 61 return(0); 62 } 63 64 RB_PROTOTYPE2(hammer2_io_tree, hammer2_io, rbnode, hammer2_io_cmp, off_t); 65 RB_GENERATE2(hammer2_io_tree, hammer2_io, rbnode, hammer2_io_cmp, 66 off_t, pbase); 67 68 struct hammer2_cleanupcb_info { 69 struct hammer2_io_tree tmptree; 70 int count; 71 }; 72 73 #if 0 74 static __inline 75 uint64_t 76 hammer2_io_mask(hammer2_io_t *dio, hammer2_off_t off, u_int bytes) 77 { 78 uint64_t mask; 79 int i; 80 81 if (bytes < 1024) /* smaller chunks not supported */ 82 return 0; 83 84 /* 85 * Calculate crc check mask for larger chunks 86 */ 87 i = (((off & ~HAMMER2_OFF_MASK_RADIX) - dio->pbase) & 88 HAMMER2_PBUFMASK) >> 10; 89 if (i == 0 && bytes == HAMMER2_PBUFSIZE) 90 return((uint64_t)-1); 91 mask = ((uint64_t)1U << (bytes >> 10)) - 1; 92 mask <<= i; 93 94 return mask; 95 } 96 #endif 97 98 #ifdef HAMMER2_IO_DEBUG 99 100 static __inline void 101 DIO_RECORD(hammer2_io_t *dio HAMMER2_IO_DEBUG_ARGS) 102 { 103 int i; 104 105 i = atomic_fetchadd_int(&dio->debug_index, 1) & HAMMER2_IO_DEBUG_MASK; 106 107 dio->debug_file[i] = file; 108 dio->debug_line[i] = line; 109 dio->debug_refs[i] = dio->refs; 110 dio->debug_td[i] = curthread; 111 } 112 113 #else 114 115 #define DIO_RECORD(dio) 116 117 #endif 118 119 /* 120 * Returns the DIO corresponding to the data|radix, creating it if necessary. 121 * 122 * If createit is 0, NULL can be returned indicating that the DIO does not 123 * exist. (btype) is ignored when createit is 0. 124 */ 125 static __inline 126 hammer2_io_t * 127 hammer2_io_alloc(hammer2_dev_t *hmp, hammer2_key_t data_off, uint8_t btype, 128 int createit, int *isgoodp) 129 { 130 hammer2_io_t *dio; 131 hammer2_io_t *xio; 132 hammer2_key_t lbase; 133 hammer2_key_t pbase; 134 hammer2_key_t pmask; 135 hammer2_volume_t *vol; 136 uint64_t refs; 137 int lsize; 138 int psize; 139 140 psize = HAMMER2_PBUFSIZE; 141 pmask = ~(hammer2_off_t)(psize - 1); 142 if ((int)(data_off & HAMMER2_OFF_MASK_RADIX)) 143 lsize = 1 << (int)(data_off & HAMMER2_OFF_MASK_RADIX); 144 else 145 lsize = 0; 146 lbase = data_off & ~HAMMER2_OFF_MASK_RADIX; 147 pbase = lbase & pmask; 148 149 if (pbase == 0 || ((lbase + lsize - 1) & pmask) != pbase) { 150 kprintf("Illegal: %016jx %016jx+%08x / %016jx\n", 151 pbase, lbase, lsize, pmask); 152 } 153 KKASSERT(pbase != 0 && ((lbase + lsize - 1) & pmask) == pbase); 154 *isgoodp = 0; 155 156 /* 157 * Access/Allocate the DIO, bump dio->refs to prevent destruction. 158 * 159 * If DIO_GOOD is set the ref should prevent it from being cleared 160 * out from under us, we can set *isgoodp, and the caller can operate 161 * on the buffer without any further interaction. 162 */ 163 hammer2_spin_sh(&hmp->io_spin); 164 dio = RB_LOOKUP(hammer2_io_tree, &hmp->iotree, pbase); 165 if (dio) { 166 refs = atomic_fetchadd_64(&dio->refs, 1); 167 if ((refs & HAMMER2_DIO_MASK) == 0) { 168 atomic_add_int(&dio->hmp->iofree_count, -1); 169 } 170 if (refs & HAMMER2_DIO_GOOD) 171 *isgoodp = 1; 172 hammer2_spin_unsh(&hmp->io_spin); 173 } else if (createit) { 174 refs = 0; 175 hammer2_spin_unsh(&hmp->io_spin); 176 vol = hammer2_get_volume(hmp, pbase); 177 dio = kmalloc(sizeof(*dio), M_HAMMER2, M_INTWAIT | M_ZERO); 178 dio->hmp = hmp; 179 dio->devvp = vol->dev->devvp; 180 dio->dbase = vol->offset; 181 KKASSERT((dio->dbase & HAMMER2_FREEMAP_LEVEL1_MASK) == 0); 182 dio->pbase = pbase; 183 dio->psize = psize; 184 dio->btype = btype; 185 dio->refs = refs + 1; 186 dio->act = 5; 187 hammer2_spin_ex(&hmp->io_spin); 188 xio = RB_INSERT(hammer2_io_tree, &hmp->iotree, dio); 189 if (xio == NULL) { 190 atomic_add_int(&hammer2_dio_count, 1); 191 hammer2_spin_unex(&hmp->io_spin); 192 } else { 193 refs = atomic_fetchadd_64(&xio->refs, 1); 194 if ((refs & HAMMER2_DIO_MASK) == 0) 195 atomic_add_int(&xio->hmp->iofree_count, -1); 196 if (refs & HAMMER2_DIO_GOOD) 197 *isgoodp = 1; 198 hammer2_spin_unex(&hmp->io_spin); 199 kfree(dio, M_HAMMER2); 200 dio = xio; 201 } 202 } else { 203 hammer2_spin_unsh(&hmp->io_spin); 204 return NULL; 205 } 206 dio->ticks = ticks; 207 if (dio->act < 10) 208 ++dio->act; 209 210 return dio; 211 } 212 213 /* 214 * Acquire the requested dio. If DIO_GOOD is not set we must instantiate 215 * a buffer. If set the buffer already exists and is good to go. 216 */ 217 hammer2_io_t * 218 _hammer2_io_getblk(hammer2_dev_t *hmp, int btype, off_t lbase, 219 int lsize, int op HAMMER2_IO_DEBUG_ARGS) 220 { 221 hammer2_io_t *dio; 222 hammer2_off_t dev_pbase; 223 off_t peof; 224 uint64_t orefs; 225 uint64_t nrefs; 226 int isgood; 227 int error; 228 int hce; 229 int bflags; 230 231 bflags = ((btype == HAMMER2_BREF_TYPE_DATA) ? B_NOTMETA : 0); 232 bflags |= B_KVABIO; 233 234 KKASSERT((1 << (int)(lbase & HAMMER2_OFF_MASK_RADIX)) == lsize); 235 236 if (op == HAMMER2_DOP_READQ) { 237 dio = hammer2_io_alloc(hmp, lbase, btype, 0, &isgood); 238 if (dio == NULL) 239 return NULL; 240 op = HAMMER2_DOP_READ; 241 } else { 242 dio = hammer2_io_alloc(hmp, lbase, btype, 1, &isgood); 243 } 244 245 for (;;) { 246 orefs = dio->refs; 247 cpu_ccfence(); 248 249 /* 250 * Buffer is already good, handle the op and return. 251 */ 252 if (orefs & HAMMER2_DIO_GOOD) { 253 if (isgood == 0) 254 cpu_mfence(); 255 bkvasync(dio->bp); 256 257 switch(op) { 258 case HAMMER2_DOP_NEW: 259 bzero(hammer2_io_data(dio, lbase), lsize); 260 /* fall through */ 261 case HAMMER2_DOP_NEWNZ: 262 atomic_set_long(&dio->refs, HAMMER2_DIO_DIRTY); 263 break; 264 case HAMMER2_DOP_READ: 265 default: 266 /* nothing to do */ 267 break; 268 } 269 DIO_RECORD(dio HAMMER2_IO_DEBUG_CALL); 270 return (dio); 271 } 272 273 /* 274 * Try to own the DIO 275 */ 276 if (orefs & HAMMER2_DIO_INPROG) { 277 nrefs = orefs | HAMMER2_DIO_WAITING; 278 tsleep_interlock(dio, 0); 279 if (atomic_cmpset_64(&dio->refs, orefs, nrefs)) { 280 tsleep(dio, PINTERLOCKED, "h2dio", hz); 281 } 282 /* retry */ 283 } else { 284 nrefs = orefs | HAMMER2_DIO_INPROG; 285 if (atomic_cmpset_64(&dio->refs, orefs, nrefs)) { 286 break; 287 } 288 } 289 } 290 291 /* 292 * We break to here if GOOD is not set and we acquired INPROG for 293 * the I/O. 294 */ 295 KKASSERT(dio->bp == NULL); 296 if (btype == HAMMER2_BREF_TYPE_DATA) 297 hce = hammer2_cluster_data_read; 298 else 299 hce = hammer2_cluster_meta_read; 300 301 error = 0; 302 dev_pbase = dio->pbase - dio->dbase; 303 if (dio->pbase == (lbase & ~HAMMER2_OFF_MASK_RADIX) && 304 dio->psize == lsize) { 305 switch(op) { 306 case HAMMER2_DOP_NEW: 307 case HAMMER2_DOP_NEWNZ: 308 dio->bp = getblk(dio->devvp, 309 dev_pbase, dio->psize, 310 GETBLK_KVABIO, 0); 311 if (op == HAMMER2_DOP_NEW) { 312 bkvasync(dio->bp); 313 bzero(dio->bp->b_data, dio->psize); 314 } 315 atomic_set_long(&dio->refs, HAMMER2_DIO_DIRTY); 316 break; 317 case HAMMER2_DOP_READ: 318 default: 319 KKASSERT(dio->bp == NULL); 320 if (hce > 0) { 321 /* 322 * Synchronous cluster I/O for now. 323 */ 324 peof = (dio->pbase + HAMMER2_SEGMASK64) & 325 ~HAMMER2_SEGMASK64; 326 peof -= dio->dbase; 327 error = cluster_readx(dio->devvp, 328 peof, dev_pbase, 329 dio->psize, bflags, 330 dio->psize, 331 HAMMER2_PBUFSIZE*hce, 332 &dio->bp); 333 } else { 334 error = breadnx(dio->devvp, dev_pbase, 335 dio->psize, bflags, 336 NULL, NULL, 0, &dio->bp); 337 } 338 break; 339 } 340 } else { 341 if (hce > 0) { 342 /* 343 * Synchronous cluster I/O for now. 344 */ 345 peof = (dio->pbase + HAMMER2_SEGMASK64) & 346 ~HAMMER2_SEGMASK64; 347 peof -= dio->dbase; 348 error = cluster_readx(dio->devvp, 349 peof, dev_pbase, dio->psize, 350 bflags, 351 dio->psize, HAMMER2_PBUFSIZE*hce, 352 &dio->bp); 353 } else { 354 error = breadnx(dio->devvp, dev_pbase, 355 dio->psize, bflags, 356 NULL, NULL, 0, &dio->bp); 357 } 358 if (dio->bp) { 359 /* 360 * Handle NEW flags 361 */ 362 switch(op) { 363 case HAMMER2_DOP_NEW: 364 bkvasync(dio->bp); 365 bzero(hammer2_io_data(dio, lbase), lsize); 366 /* fall through */ 367 case HAMMER2_DOP_NEWNZ: 368 atomic_set_long(&dio->refs, HAMMER2_DIO_DIRTY); 369 break; 370 case HAMMER2_DOP_READ: 371 default: 372 break; 373 } 374 375 /* 376 * Tell the kernel that the buffer cache is not 377 * meta-data based on the btype. This allows 378 * swapcache to distinguish between data and 379 * meta-data. 380 */ 381 switch(btype) { 382 case HAMMER2_BREF_TYPE_DATA: 383 dio->bp->b_flags |= B_NOTMETA; 384 break; 385 default: 386 break; 387 } 388 } 389 } 390 391 if (dio->bp) { 392 bkvasync(dio->bp); 393 BUF_KERNPROC(dio->bp); 394 dio->bp->b_flags &= ~B_AGE; 395 /* dio->bp->b_debug_info2 = dio; */ 396 } 397 dio->error = error; 398 399 /* 400 * Clear INPROG and WAITING, set GOOD wake up anyone waiting. 401 */ 402 for (;;) { 403 orefs = dio->refs; 404 cpu_ccfence(); 405 nrefs = orefs & ~(HAMMER2_DIO_INPROG | HAMMER2_DIO_WAITING); 406 if (error == 0) 407 nrefs |= HAMMER2_DIO_GOOD; 408 if (atomic_cmpset_64(&dio->refs, orefs, nrefs)) { 409 if (orefs & HAMMER2_DIO_WAITING) 410 wakeup(dio); 411 break; 412 } 413 cpu_pause(); 414 } 415 416 /* XXX error handling */ 417 DIO_RECORD(dio HAMMER2_IO_DEBUG_CALL); 418 419 return dio; 420 } 421 422 /* 423 * Release our ref on *diop. 424 * 425 * On the 1->0 transition we clear DIO_GOOD, set DIO_INPROG, and dispose 426 * of dio->bp. Then we clean up DIO_INPROG and DIO_WAITING. 427 */ 428 void 429 _hammer2_io_putblk(hammer2_io_t **diop HAMMER2_IO_DEBUG_ARGS) 430 { 431 hammer2_dev_t *hmp; 432 hammer2_io_t *dio; 433 struct buf *bp; 434 off_t pbase; 435 int psize; 436 int dio_limit; 437 uint64_t orefs; 438 uint64_t nrefs; 439 440 dio = *diop; 441 *diop = NULL; 442 hmp = dio->hmp; 443 DIO_RECORD(dio HAMMER2_IO_DEBUG_CALL); 444 445 KKASSERT((dio->refs & HAMMER2_DIO_MASK) != 0); 446 447 /* 448 * Drop refs. 449 * 450 * On the 1->0 transition clear GOOD and set INPROG, and break. 451 * On any other transition we can return early. 452 */ 453 for (;;) { 454 orefs = dio->refs; 455 cpu_ccfence(); 456 457 if ((orefs & HAMMER2_DIO_MASK) == 1 && 458 (orefs & HAMMER2_DIO_INPROG) == 0) { 459 /* 460 * Lastdrop case, INPROG can be set. GOOD must be 461 * cleared to prevent the getblk shortcut. 462 */ 463 nrefs = orefs - 1; 464 nrefs &= ~(HAMMER2_DIO_GOOD | HAMMER2_DIO_DIRTY); 465 nrefs |= HAMMER2_DIO_INPROG; 466 if (atomic_cmpset_64(&dio->refs, orefs, nrefs)) 467 break; 468 } else if ((orefs & HAMMER2_DIO_MASK) == 1) { 469 /* 470 * Lastdrop case, INPROG already set. We must 471 * wait for INPROG to clear. 472 */ 473 nrefs = orefs | HAMMER2_DIO_WAITING; 474 tsleep_interlock(dio, 0); 475 if (atomic_cmpset_64(&dio->refs, orefs, nrefs)) { 476 tsleep(dio, PINTERLOCKED, "h2dio", hz); 477 } 478 /* retry */ 479 } else { 480 /* 481 * Normal drop case. 482 */ 483 nrefs = orefs - 1; 484 if (atomic_cmpset_64(&dio->refs, orefs, nrefs)) 485 return; 486 /* retry */ 487 } 488 cpu_pause(); 489 /* retry */ 490 } 491 492 /* 493 * Lastdrop (1->0 transition). INPROG has been set, GOOD and DIRTY 494 * have been cleared. iofree_count has not yet been incremented, 495 * note that another accessor race will decrement iofree_count so 496 * we have to increment it regardless. 497 * 498 * We can now dispose of the buffer, and should do it before calling 499 * io_complete() in case there's a race against a new reference 500 * which causes io_complete() to chain and instantiate the bp again. 501 */ 502 pbase = dio->pbase; 503 psize = dio->psize; 504 bp = dio->bp; 505 dio->bp = NULL; 506 507 if ((orefs & HAMMER2_DIO_GOOD) && bp) { 508 /* 509 * Non-errored disposal of bp 510 */ 511 if (orefs & HAMMER2_DIO_DIRTY) { 512 dio_write_stats_update(dio, bp); 513 514 /* 515 * Allows dirty buffers to accumulate and 516 * possibly be canceled (e.g. by a 'rm'), 517 * by default we will burst-write later. 518 * 519 * We generally do NOT want to issue an actual 520 * b[a]write() or cluster_write() here. Due to 521 * the way chains are locked, buffers may be cycled 522 * in and out quite often and disposal here can cause 523 * multiple writes or write-read stalls. 524 * 525 * If FLUSH is set we do want to issue the actual 526 * write. This typically occurs in the write-behind 527 * case when writing to large files. 528 */ 529 off_t peof; 530 int hce; 531 if (dio->refs & HAMMER2_DIO_FLUSH) { 532 if ((hce = hammer2_cluster_write) != 0) { 533 peof = (pbase + HAMMER2_SEGMASK64) & 534 ~HAMMER2_SEGMASK64; 535 peof -= dio->dbase; 536 bp->b_flags |= B_CLUSTEROK; 537 cluster_write(bp, peof, psize, hce); 538 } else { 539 bp->b_flags &= ~B_CLUSTEROK; 540 bawrite(bp); 541 } 542 } else { 543 bp->b_flags &= ~B_CLUSTEROK; 544 bdwrite(bp); 545 } 546 } else if (bp->b_flags & (B_ERROR | B_INVAL | B_RELBUF)) { 547 brelse(bp); 548 } else { 549 bqrelse(bp); 550 } 551 } else if (bp) { 552 /* 553 * Errored disposal of bp 554 */ 555 brelse(bp); 556 } 557 558 /* 559 * Update iofree_count before disposing of the dio 560 */ 561 hmp = dio->hmp; 562 atomic_add_int(&hmp->iofree_count, 1); 563 564 /* 565 * Clear INPROG, GOOD, and WAITING (GOOD should already be clear). 566 * 567 * Also clear FLUSH as it was handled above. 568 */ 569 for (;;) { 570 orefs = dio->refs; 571 cpu_ccfence(); 572 nrefs = orefs & ~(HAMMER2_DIO_INPROG | HAMMER2_DIO_GOOD | 573 HAMMER2_DIO_WAITING | HAMMER2_DIO_FLUSH); 574 if (atomic_cmpset_64(&dio->refs, orefs, nrefs)) { 575 if (orefs & HAMMER2_DIO_WAITING) 576 wakeup(dio); 577 break; 578 } 579 cpu_pause(); 580 } 581 582 /* 583 * We cache free buffers so re-use cases can use a shared lock, but 584 * if too many build up we have to clean them out. 585 */ 586 dio_limit = hammer2_dio_limit; 587 if (dio_limit < 256) 588 dio_limit = 256; 589 if (dio_limit > 1024*1024) 590 dio_limit = 1024*1024; 591 if (hmp->iofree_count > dio_limit) { 592 struct hammer2_cleanupcb_info info; 593 594 RB_INIT(&info.tmptree); 595 hammer2_spin_ex(&hmp->io_spin); 596 if (hmp->iofree_count > dio_limit) { 597 info.count = hmp->iofree_count / 5; 598 RB_SCAN(hammer2_io_tree, &hmp->iotree, NULL, 599 hammer2_io_cleanup_callback, &info); 600 } 601 hammer2_spin_unex(&hmp->io_spin); 602 hammer2_io_cleanup(hmp, &info.tmptree); 603 } 604 } 605 606 /* 607 * Cleanup any dio's with (INPROG | refs) == 0. 608 * 609 * Called to clean up cached DIOs on umount after all activity has been 610 * flushed. 611 */ 612 static 613 int 614 hammer2_io_cleanup_callback(hammer2_io_t *dio, void *arg) 615 { 616 struct hammer2_cleanupcb_info *info = arg; 617 hammer2_io_t *xio; 618 619 if ((dio->refs & (HAMMER2_DIO_MASK | HAMMER2_DIO_INPROG)) == 0) { 620 if (dio->act > 0) { 621 int act; 622 623 act = dio->act - (ticks - dio->ticks) / hz - 1; 624 if (act > 0) { 625 dio->act = act; 626 return 0; 627 } 628 dio->act = 0; 629 } 630 KKASSERT(dio->bp == NULL); 631 if (info->count > 0) { 632 RB_REMOVE(hammer2_io_tree, &dio->hmp->iotree, dio); 633 xio = RB_INSERT(hammer2_io_tree, &info->tmptree, dio); 634 KKASSERT(xio == NULL); 635 --info->count; 636 } 637 } 638 return 0; 639 } 640 641 void 642 hammer2_io_cleanup(hammer2_dev_t *hmp, struct hammer2_io_tree *tree) 643 { 644 hammer2_io_t *dio; 645 646 while ((dio = RB_ROOT(tree)) != NULL) { 647 RB_REMOVE(hammer2_io_tree, tree, dio); 648 KKASSERT(dio->bp == NULL && 649 (dio->refs & (HAMMER2_DIO_MASK | HAMMER2_DIO_INPROG)) == 0); 650 if (dio->refs & HAMMER2_DIO_DIRTY) { 651 kprintf("hammer2_io_cleanup: Dirty buffer " 652 "%016jx/%d (bp=%p)\n", 653 dio->pbase, dio->psize, dio->bp); 654 } 655 kfree(dio, M_HAMMER2); 656 atomic_add_int(&hammer2_dio_count, -1); 657 atomic_add_int(&hmp->iofree_count, -1); 658 } 659 } 660 661 /* 662 * Returns a pointer to the requested data. 663 */ 664 char * 665 hammer2_io_data(hammer2_io_t *dio, off_t lbase) 666 { 667 struct buf *bp; 668 int off; 669 670 bp = dio->bp; 671 KKASSERT(bp != NULL); 672 bkvasync(bp); 673 lbase -= dio->dbase; 674 off = (lbase & ~HAMMER2_OFF_MASK_RADIX) - bp->b_loffset; 675 KKASSERT(off >= 0 && off < bp->b_bufsize); 676 return(bp->b_data + off); 677 } 678 679 int 680 hammer2_io_new(hammer2_dev_t *hmp, int btype, off_t lbase, int lsize, 681 hammer2_io_t **diop) 682 { 683 *diop = hammer2_io_getblk(hmp, btype, lbase, lsize, HAMMER2_DOP_NEW); 684 return ((*diop)->error); 685 } 686 687 int 688 hammer2_io_newnz(hammer2_dev_t *hmp, int btype, off_t lbase, int lsize, 689 hammer2_io_t **diop) 690 { 691 *diop = hammer2_io_getblk(hmp, btype, lbase, lsize, HAMMER2_DOP_NEWNZ); 692 return ((*diop)->error); 693 } 694 695 int 696 _hammer2_io_bread(hammer2_dev_t *hmp, int btype, off_t lbase, int lsize, 697 hammer2_io_t **diop HAMMER2_IO_DEBUG_ARGS) 698 { 699 #ifdef HAMMER2_IO_DEBUG 700 hammer2_io_t *dio; 701 #endif 702 703 *diop = _hammer2_io_getblk(hmp, btype, lbase, lsize, 704 HAMMER2_DOP_READ HAMMER2_IO_DEBUG_CALL); 705 #ifdef HAMMER2_IO_DEBUG 706 if ((dio = *diop) != NULL) { 707 int i = (dio->debug_index - 1) & HAMMER2_IO_DEBUG_MASK; 708 dio->debug_data[i] = debug_data; 709 } 710 #endif 711 return ((*diop)->error); 712 } 713 714 hammer2_io_t * 715 _hammer2_io_getquick(hammer2_dev_t *hmp, off_t lbase, 716 int lsize HAMMER2_IO_DEBUG_ARGS) 717 { 718 hammer2_io_t *dio; 719 720 dio = _hammer2_io_getblk(hmp, 0, lbase, lsize, 721 HAMMER2_DOP_READQ HAMMER2_IO_DEBUG_CALL); 722 return dio; 723 } 724 725 void 726 _hammer2_io_bawrite(hammer2_io_t **diop HAMMER2_IO_DEBUG_ARGS) 727 { 728 atomic_set_64(&(*diop)->refs, HAMMER2_DIO_DIRTY | 729 HAMMER2_DIO_FLUSH); 730 _hammer2_io_putblk(diop HAMMER2_IO_DEBUG_CALL); 731 } 732 733 void 734 _hammer2_io_bdwrite(hammer2_io_t **diop HAMMER2_IO_DEBUG_ARGS) 735 { 736 atomic_set_64(&(*diop)->refs, HAMMER2_DIO_DIRTY); 737 _hammer2_io_putblk(diop HAMMER2_IO_DEBUG_CALL); 738 } 739 740 int 741 _hammer2_io_bwrite(hammer2_io_t **diop HAMMER2_IO_DEBUG_ARGS) 742 { 743 atomic_set_64(&(*diop)->refs, HAMMER2_DIO_DIRTY | 744 HAMMER2_DIO_FLUSH); 745 _hammer2_io_putblk(diop HAMMER2_IO_DEBUG_CALL); 746 return (0); /* XXX */ 747 } 748 749 void 750 hammer2_io_setdirty(hammer2_io_t *dio) 751 { 752 atomic_set_64(&dio->refs, HAMMER2_DIO_DIRTY); 753 } 754 755 /* 756 * This routine is called when a MODIFIED chain is being DESTROYED, 757 * in an attempt to allow the related buffer cache buffer to be 758 * invalidated and discarded instead of flushing it to disk. 759 * 760 * At the moment this case is only really useful for file meta-data. 761 * File data is already handled via the logical buffer cache associated 762 * with the vnode, and will be discarded if it was never flushed to disk. 763 * File meta-data may include inodes, directory entries, and indirect blocks. 764 * 765 * XXX 766 * However, our DIO buffers are PBUFSIZE'd (64KB), and the area being 767 * invalidated might be smaller. Most of the meta-data structures above 768 * are in the 'smaller' category. For now, don't try to invalidate the 769 * data areas. 770 */ 771 void 772 hammer2_io_inval(hammer2_io_t *dio, hammer2_off_t data_off, u_int bytes) 773 { 774 /* NOP */ 775 } 776 777 void 778 _hammer2_io_brelse(hammer2_io_t **diop HAMMER2_IO_DEBUG_ARGS) 779 { 780 _hammer2_io_putblk(diop HAMMER2_IO_DEBUG_CALL); 781 } 782 783 void 784 _hammer2_io_bqrelse(hammer2_io_t **diop HAMMER2_IO_DEBUG_ARGS) 785 { 786 _hammer2_io_putblk(diop HAMMER2_IO_DEBUG_CALL); 787 } 788 789 /* 790 * Set dedup validation bits in a DIO. We do not need the buffer cache 791 * buffer for this. This must be done concurrent with setting bits in 792 * the freemap so as to interlock with bulkfree's clearing of those bits. 793 */ 794 void 795 hammer2_io_dedup_set(hammer2_dev_t *hmp, hammer2_blockref_t *bref) 796 { 797 hammer2_io_t *dio; 798 uint64_t mask; 799 int lsize; 800 int isgood; 801 802 dio = hammer2_io_alloc(hmp, bref->data_off, bref->type, 1, &isgood); 803 if ((int)(bref->data_off & HAMMER2_OFF_MASK_RADIX)) 804 lsize = 1 << (int)(bref->data_off & HAMMER2_OFF_MASK_RADIX); 805 else 806 lsize = 0; 807 mask = hammer2_dedup_mask(dio, bref->data_off, lsize); 808 atomic_clear_64(&dio->dedup_valid, mask); 809 atomic_set_64(&dio->dedup_alloc, mask); 810 hammer2_io_putblk(&dio); 811 } 812 813 /* 814 * Clear dedup validation bits in a DIO. This is typically done when 815 * a modified chain is destroyed or by the bulkfree code. No buffer 816 * is needed for this operation. If the DIO no longer exists it is 817 * equivalent to the bits not being set. 818 */ 819 void 820 hammer2_io_dedup_delete(hammer2_dev_t *hmp, uint8_t btype, 821 hammer2_off_t data_off, u_int bytes) 822 { 823 hammer2_io_t *dio; 824 uint64_t mask; 825 int isgood; 826 827 if ((data_off & ~HAMMER2_OFF_MASK_RADIX) == 0) 828 return; 829 if (btype != HAMMER2_BREF_TYPE_DATA) 830 return; 831 dio = hammer2_io_alloc(hmp, data_off, btype, 0, &isgood); 832 if (dio) { 833 if (data_off < dio->pbase || 834 (data_off & ~HAMMER2_OFF_MASK_RADIX) + bytes > 835 dio->pbase + dio->psize) { 836 panic("hammer2_io_dedup_delete: DATAOFF BAD " 837 "%016jx/%d %016jx\n", 838 data_off, bytes, dio->pbase); 839 } 840 mask = hammer2_dedup_mask(dio, data_off, bytes); 841 atomic_clear_64(&dio->dedup_alloc, mask); 842 atomic_clear_64(&dio->dedup_valid, mask); 843 hammer2_io_putblk(&dio); 844 } 845 } 846 847 /* 848 * Assert that dedup allocation bits in a DIO are not set. This operation 849 * does not require a buffer. The DIO does not need to exist. 850 */ 851 void 852 hammer2_io_dedup_assert(hammer2_dev_t *hmp, hammer2_off_t data_off, u_int bytes) 853 { 854 hammer2_io_t *dio; 855 int isgood; 856 857 dio = hammer2_io_alloc(hmp, data_off, HAMMER2_BREF_TYPE_DATA, 858 0, &isgood); 859 if (dio) { 860 KASSERT((dio->dedup_alloc & 861 hammer2_dedup_mask(dio, data_off, bytes)) == 0, 862 ("hammer2_dedup_assert: %016jx/%d %016jx/%016jx", 863 data_off, 864 bytes, 865 hammer2_dedup_mask(dio, data_off, bytes), 866 dio->dedup_alloc)); 867 hammer2_io_putblk(&dio); 868 } 869 } 870 871 static 872 void 873 dio_write_stats_update(hammer2_io_t *dio, struct buf *bp) 874 { 875 if (bp->b_flags & B_DELWRI) 876 return; 877 hammer2_adjwritecounter(dio->btype, dio->psize); 878 } 879 880 void 881 hammer2_io_bkvasync(hammer2_io_t *dio) 882 { 883 KKASSERT(dio->bp != NULL); 884 bkvasync(dio->bp); 885 } 886 887 /* 888 * Ref a dio that is already owned 889 */ 890 void 891 _hammer2_io_ref(hammer2_io_t *dio HAMMER2_IO_DEBUG_ARGS) 892 { 893 DIO_RECORD(dio HAMMER2_IO_DEBUG_CALL); 894 atomic_add_64(&dio->refs, 1); 895 } 896