1 /* 2 * Copyright (c) 2013-2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@dragonflybsd.org> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/fcntl.h> 38 #include <sys/buf.h> 39 #include <sys/proc.h> 40 #include <sys/mount.h> 41 #include <sys/vnode.h> 42 #include <sys/mountctl.h> 43 #include <vm/vm_kern.h> 44 #include <vm/vm_extern.h> 45 46 #include "hammer2.h" 47 48 /* 49 * breadth-first search 50 */ 51 typedef struct hammer2_chain_save { 52 TAILQ_ENTRY(hammer2_chain_save) entry; 53 hammer2_chain_t *chain; 54 int pri; 55 } hammer2_chain_save_t; 56 57 TAILQ_HEAD(hammer2_chain_save_list, hammer2_chain_save); 58 typedef struct hammer2_chain_save_list hammer2_chain_save_list_t; 59 60 typedef struct hammer2_bulkfree_info { 61 hammer2_dev_t *hmp; 62 kmem_anon_desc_t kp; 63 hammer2_off_t sbase; /* sub-loop iteration */ 64 hammer2_off_t sstop; 65 hammer2_bmap_data_t *bmap; 66 int depth; 67 long count_10_00; /* staged->free */ 68 long count_11_10; /* allocated->staged */ 69 long count_00_11; /* (should not happen) */ 70 long count_01_11; /* (should not happen) */ 71 long count_10_11; /* staged->allocated */ 72 long count_l0cleans; 73 long count_linadjusts; 74 long count_inodes_scanned; 75 long count_dirents_scanned; 76 long count_dedup_factor; 77 long count_bytes_scanned; 78 long count_chains_scanned; 79 long count_chains_reported; 80 long bulkfree_calls; 81 int bulkfree_ticks; 82 hammer2_off_t adj_free; 83 hammer2_tid_t mtid; 84 hammer2_tid_t saved_mirror_tid; 85 time_t save_time; 86 hammer2_chain_save_list_t list; 87 hammer2_dedup_t *dedup; 88 int pri; 89 } hammer2_bulkfree_info_t; 90 91 static int h2_bulkfree_test(hammer2_bulkfree_info_t *info, 92 hammer2_blockref_t *bref, int pri, int saved_error); 93 94 /* 95 * General bulk scan function with callback. Called with a referenced 96 * but UNLOCKED parent. The parent is returned in the same state. 97 */ 98 static 99 int 100 hammer2_bulk_scan(hammer2_chain_t *parent, 101 int (*func)(hammer2_bulkfree_info_t *info, 102 hammer2_blockref_t *bref), 103 hammer2_bulkfree_info_t *info) 104 { 105 hammer2_blockref_t bref; 106 hammer2_chain_t *chain; 107 hammer2_chain_save_t *tail; 108 hammer2_chain_save_t *save; 109 int first = 1; 110 int rup_error; 111 int error; 112 int e2; 113 114 ++info->pri; 115 116 chain = NULL; 117 rup_error = 0; 118 error = 0; 119 120 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS | 121 HAMMER2_RESOLVE_SHARED); 122 tail = TAILQ_LAST(&info->list, hammer2_chain_save_list); 123 124 /* 125 * The parent was previously retrieved NODATA and thus has not 126 * tested the CRC. Now that we have locked it normally, check 127 * for a CRC problem and skip it if we found one. The bulk scan 128 * cannot safely traverse invalid block tables (we could end up 129 * in an endless loop or cause a panic). 130 */ 131 if (parent->error & HAMMER2_ERROR_CHECK) { 132 error = parent->error; 133 goto done; 134 } 135 136 /* 137 * Report which PFS is being scanned 138 */ 139 if (parent->bref.type == HAMMER2_BREF_TYPE_INODE && 140 (parent->bref.flags & HAMMER2_BREF_FLAG_PFSROOT)) { 141 kprintf("hammer2_bulkfree: Scanning %s\n", 142 parent->data->ipdata.filename); 143 } 144 145 /* 146 * Generally loop on the contents if we have not been flagged 147 * for abort. 148 * 149 * Remember that these chains are completely isolated from 150 * the frontend, so we can release locks temporarily without 151 * imploding. 152 */ 153 for (;;) { 154 error |= hammer2_chain_scan(parent, &chain, &bref, &first, 155 HAMMER2_LOOKUP_NODATA | 156 HAMMER2_LOOKUP_SHARED); 157 158 /* 159 * Handle EOF or other error at current level. This stops 160 * the bulkfree scan. 161 */ 162 if (error & ~HAMMER2_ERROR_CHECK) 163 break; 164 165 /* 166 * Account for dirents before thre data_off test, since most 167 * dirents do not need a data reference. 168 */ 169 if (bref.type == HAMMER2_BREF_TYPE_DIRENT) 170 ++info->count_dirents_scanned; 171 172 /* 173 * Ignore brefs without data (typically dirents) 174 */ 175 if ((bref.data_off & ~HAMMER2_OFF_MASK_RADIX) == 0) 176 continue; 177 178 /* 179 * Process bref, chain is only non-NULL if the bref 180 * might be recursable (its possible that we sometimes get 181 * a non-NULL chain where the bref cannot be recursed). 182 * 183 * If we already ran down this tree we do not have to do it 184 * again, but we must still recover any cumulative error 185 * recorded from the time we did. 186 */ 187 ++info->pri; 188 e2 = h2_bulkfree_test(info, &bref, 1, 0); 189 if (e2) { 190 error |= e2 & ~HAMMER2_ERROR_EOF; 191 continue; 192 } 193 194 if (bref.type == HAMMER2_BREF_TYPE_INODE) 195 ++info->count_inodes_scanned; 196 197 error |= func(info, &bref); 198 if (error & ~HAMMER2_ERROR_CHECK) 199 break; 200 201 /* 202 * A non-null chain is always returned if it is 203 * recursive, otherwise a non-null chain might be 204 * returned but usually is not when not recursive. 205 */ 206 if (chain == NULL) 207 continue; 208 209 if (chain) { 210 info->count_bytes_scanned += chain->bytes; 211 ++info->count_chains_scanned; 212 213 if (info->count_chains_scanned >= 214 info->count_chains_reported + 1000000 || 215 (info->count_chains_scanned < 1000000 && 216 info->count_chains_scanned >= 217 info->count_chains_reported + 100000)) { 218 kprintf(" chains %-7ld inodes %-7ld " 219 "dirents %-7ld bytes %5ldMB\n", 220 info->count_chains_scanned, 221 info->count_inodes_scanned, 222 info->count_dirents_scanned, 223 info->count_bytes_scanned / 1000000); 224 info->count_chains_reported = 225 info->count_chains_scanned; 226 } 227 } 228 229 /* 230 * Else check type and setup depth-first scan. 231 * 232 * Account for bytes actually read. 233 */ 234 switch(chain->bref.type) { 235 case HAMMER2_BREF_TYPE_INODE: 236 case HAMMER2_BREF_TYPE_FREEMAP_NODE: 237 case HAMMER2_BREF_TYPE_INDIRECT: 238 case HAMMER2_BREF_TYPE_VOLUME: 239 case HAMMER2_BREF_TYPE_FREEMAP: 240 ++info->depth; 241 if (chain->error & HAMMER2_ERROR_CHECK) { 242 /* 243 * Cannot safely recurse chains with crc 244 * errors, even in emergency mode. 245 */ 246 /* NOP */ 247 } else if (info->depth > 16) { 248 save = kmalloc(sizeof(*save), M_HAMMER2, 249 M_WAITOK | M_ZERO); 250 save->chain = chain; 251 hammer2_chain_ref(chain); 252 TAILQ_INSERT_TAIL(&info->list, save, entry); 253 254 /* guess */ 255 info->pri += 10; 256 } else { 257 int savepri = info->pri; 258 259 hammer2_chain_unlock(chain); 260 hammer2_chain_unlock(parent); 261 info->pri = 0; 262 rup_error |= hammer2_bulk_scan(chain, 263 func, info); 264 info->pri += savepri; 265 hammer2_chain_lock(parent, 266 HAMMER2_RESOLVE_ALWAYS | 267 HAMMER2_RESOLVE_SHARED); 268 hammer2_chain_lock(chain, 269 HAMMER2_RESOLVE_ALWAYS | 270 HAMMER2_RESOLVE_SHARED); 271 } 272 --info->depth; 273 break; 274 case HAMMER2_BREF_TYPE_DATA: 275 break; 276 default: 277 /* does not recurse */ 278 break; 279 } 280 if (rup_error & HAMMER2_ERROR_ABORTED) 281 break; 282 } 283 if (chain) { 284 hammer2_chain_unlock(chain); 285 hammer2_chain_drop(chain); 286 } 287 288 /* 289 * If this is a PFSROOT, also re-run any defered elements 290 * added during our scan so we can report any cumulative errors 291 * for the PFS. 292 */ 293 if (parent->bref.type == HAMMER2_BREF_TYPE_INODE && 294 (parent->bref.flags & HAMMER2_BREF_FLAG_PFSROOT)) { 295 for (;;) { 296 int opri; 297 298 save = tail ? TAILQ_NEXT(tail, entry) : 299 TAILQ_FIRST(&info->list); 300 if (save == NULL) 301 break; 302 303 TAILQ_REMOVE(&info->list, save, entry); 304 opri = info->pri; 305 info->pri = 0; 306 rup_error |= hammer2_bulk_scan(save->chain, func, info); 307 hammer2_chain_drop(save->chain); 308 kfree(save, M_HAMMER2); 309 info->pri = opri; 310 } 311 } 312 313 error |= rup_error; 314 315 /* 316 * Report which PFS the errors were encountered in. 317 */ 318 if (parent->bref.type == HAMMER2_BREF_TYPE_INODE && 319 (parent->bref.flags & HAMMER2_BREF_FLAG_PFSROOT) && 320 (error & ~HAMMER2_ERROR_EOF)) { 321 kprintf("hammer2_bulkfree: Encountered errors (%08x) " 322 "while scanning \"%s\"\n", 323 error, parent->data->ipdata.filename); 324 } 325 326 /* 327 * Save with higher pri now that we know what it is. 328 */ 329 h2_bulkfree_test(info, &parent->bref, info->pri + 1, 330 (error & ~HAMMER2_ERROR_EOF)); 331 332 done: 333 hammer2_chain_unlock(parent); 334 335 return (error & ~HAMMER2_ERROR_EOF); 336 } 337 338 /* 339 * Bulkfree algorithm 340 * 341 * Repeat { 342 * Chain flush (partial synchronization) XXX removed 343 * Scan the whole topology - build in-memory freemap (mark 11) 344 * Reconcile the in-memory freemap against the on-disk freemap. 345 * ondisk xx -> ondisk 11 (if allocated) 346 * ondisk 11 -> ondisk 10 (if free in-memory) 347 * ondisk 10 -> ondisk 00 (if free in-memory) - on next pass 348 * } 349 * 350 * The topology scan may have to be performed multiple times to window 351 * freemaps which are too large to fit in kernel memory. 352 * 353 * Races are handled using a double-transition (11->10, 10->00). The bulkfree 354 * scan snapshots the volume root's blockset and thus can run concurrent with 355 * normal operations, as long as a full flush is made between each pass to 356 * synchronize any modified chains (otherwise their blocks might be improperly 357 * freed). 358 * 359 * Temporary memory in multiples of 64KB is required to reconstruct the leaf 360 * hammer2_bmap_data blocks so they can later be compared against the live 361 * freemap. Each 64KB block represents 128 x 16KB x 1024 = ~2 GB of storage. 362 * A 32MB save area thus represents around ~1 TB. The temporary memory 363 * allocated can be specified. If it is not sufficient multiple topology 364 * passes will be made. 365 */ 366 367 /* 368 * Bulkfree callback info 369 */ 370 static void hammer2_bulkfree_thread(void *arg __unused); 371 static void cbinfo_bmap_init(hammer2_bulkfree_info_t *cbinfo, size_t size); 372 static int h2_bulkfree_callback(hammer2_bulkfree_info_t *cbinfo, 373 hammer2_blockref_t *bref); 374 static int h2_bulkfree_sync(hammer2_bulkfree_info_t *cbinfo); 375 static void h2_bulkfree_sync_adjust(hammer2_bulkfree_info_t *cbinfo, 376 hammer2_off_t data_off, hammer2_bmap_data_t *live, 377 hammer2_bmap_data_t *bmap, hammer2_key_t alloc_base); 378 379 void 380 hammer2_bulkfree_init(hammer2_dev_t *hmp) 381 { 382 hammer2_thr_create(&hmp->bfthr, NULL, hmp, 383 hmp->devrepname, -1, -1, 384 hammer2_bulkfree_thread); 385 } 386 387 void 388 hammer2_bulkfree_uninit(hammer2_dev_t *hmp) 389 { 390 hammer2_thr_delete(&hmp->bfthr); 391 } 392 393 static void 394 hammer2_bulkfree_thread(void *arg) 395 { 396 hammer2_thread_t *thr = arg; 397 hammer2_ioc_bulkfree_t bfi; 398 uint32_t flags; 399 400 for (;;) { 401 hammer2_thr_wait_any(thr, 402 HAMMER2_THREAD_STOP | 403 HAMMER2_THREAD_FREEZE | 404 HAMMER2_THREAD_UNFREEZE | 405 HAMMER2_THREAD_REMASTER, 406 hz * 60); 407 408 flags = thr->flags; 409 cpu_ccfence(); 410 if (flags & HAMMER2_THREAD_STOP) 411 break; 412 if (flags & HAMMER2_THREAD_FREEZE) { 413 hammer2_thr_signal2(thr, HAMMER2_THREAD_FROZEN, 414 HAMMER2_THREAD_FREEZE); 415 continue; 416 } 417 if (flags & HAMMER2_THREAD_UNFREEZE) { 418 hammer2_thr_signal2(thr, 0, 419 HAMMER2_THREAD_FROZEN | 420 HAMMER2_THREAD_UNFREEZE); 421 continue; 422 } 423 if (flags & HAMMER2_THREAD_FROZEN) 424 continue; 425 if (flags & HAMMER2_THREAD_REMASTER) { 426 hammer2_thr_signal2(thr, 0, HAMMER2_THREAD_REMASTER); 427 bzero(&bfi, sizeof(bfi)); 428 bfi.size = 8192 * 1024; 429 /* hammer2_bulkfree_pass(thr->hmp, &bfi); */ 430 } 431 } 432 thr->td = NULL; 433 hammer2_thr_signal(thr, HAMMER2_THREAD_STOPPED); 434 /* structure can go invalid at this point */ 435 } 436 437 int 438 hammer2_bulkfree_pass(hammer2_dev_t *hmp, hammer2_chain_t *vchain, 439 hammer2_ioc_bulkfree_t *bfi) 440 { 441 hammer2_bulkfree_info_t cbinfo; 442 hammer2_chain_save_t *save; 443 hammer2_off_t incr; 444 size_t size; 445 int error; 446 447 /* 448 * We have to clear the live dedup cache as it might have entries 449 * that are freeable as of now. Any new entries in the dedup cache 450 * made after this point, even if they become freeable, will have 451 * previously been fully allocated and will be protected by the 452 * 2-stage bulkfree. 453 */ 454 hammer2_dedup_clear(hmp); 455 456 /* 457 * Setup for free pass using the buffer size specified by the 458 * hammer2 utility, 32K-aligned. 459 */ 460 bzero(&cbinfo, sizeof(cbinfo)); 461 size = (bfi->size + HAMMER2_FREEMAP_LEVELN_PSIZE - 1) & 462 ~(size_t)(HAMMER2_FREEMAP_LEVELN_PSIZE - 1); 463 464 /* 465 * Cap at 1/4 physical memory (hammer2 utility will not normally 466 * ever specify a buffer this big, but leave the option available). 467 */ 468 if (size > kmem_lim_size() * 1024 * 1024 / 4) { 469 size = kmem_lim_size() * 1024 * 1024 / 4; 470 kprintf("hammer2: Warning: capping bulkfree buffer at %jdM\n", 471 (intmax_t)size / (1024 * 1024)); 472 } 473 474 #define HAMMER2_FREEMAP_SIZEDIV \ 475 (HAMMER2_FREEMAP_LEVEL1_SIZE / HAMMER2_FREEMAP_LEVELN_PSIZE) 476 #define HAMMER2_FREEMAP_SIZEMASK (HAMMER2_FREEMAP_SIZEDIV - 1) 477 478 /* 479 * Cap at the size needed to cover the whole volume to avoid 480 * making an unnecessarily large allocation. 481 */ 482 if (size > hmp->total_size / HAMMER2_FREEMAP_SIZEDIV) { 483 size = (hmp->total_size + HAMMER2_FREEMAP_SIZEMASK) / 484 HAMMER2_FREEMAP_SIZEDIV; 485 } 486 487 /* 488 * Minimum bitmap buffer size, then align to a LEVELN_PSIZE (32K) 489 * boundary. 490 */ 491 if (size < 1024 * 1024) 492 size = 1024 * 1024; 493 size = (size + HAMMER2_FREEMAP_LEVELN_PSIZE - 1) & 494 ~(size_t)(HAMMER2_FREEMAP_LEVELN_PSIZE - 1); 495 496 cbinfo.hmp = hmp; 497 cbinfo.bmap = kmem_alloc_swapbacked(&cbinfo.kp, size, VM_SUBSYS_HAMMER); 498 cbinfo.saved_mirror_tid = hmp->voldata.mirror_tid; 499 500 cbinfo.dedup = kmalloc(sizeof(*cbinfo.dedup) * HAMMER2_DEDUP_HEUR_SIZE, 501 M_HAMMER2, M_WAITOK | M_ZERO); 502 503 kprintf("hammer2: bulkfree buf=%jdM\n", 504 (intmax_t)size / (1024 * 1024)); 505 506 /* 507 * Normalize start point to a 2GB boundary. We operate on a 508 * 64KB leaf bitmap boundary which represents 2GB of storage. 509 */ 510 cbinfo.sbase = bfi->sbase; 511 if (cbinfo.sbase > hmp->total_size) 512 cbinfo.sbase = hmp->total_size; 513 cbinfo.sbase &= ~HAMMER2_FREEMAP_LEVEL1_MASK; 514 TAILQ_INIT(&cbinfo.list); 515 516 cbinfo.bulkfree_ticks = ticks; 517 518 /* 519 * Loop on a full meta-data scan as many times as required to 520 * get through all available storage. 521 */ 522 error = 0; 523 while (cbinfo.sbase < hmp->total_size) { 524 /* 525 * We have enough ram to represent (incr) bytes of storage. 526 * Each 64KB of ram represents 2GB of storage. 527 * 528 * We must also clean out our de-duplication heuristic for 529 * each (incr) bytes of storage, otherwise we wind up not 530 * scanning meta-data for later areas of storage because 531 * they had already been scanned in earlier areas of storage. 532 * Since the ranging is different, we have to restart 533 * the dedup heuristic too. 534 */ 535 int allmedia; 536 537 cbinfo_bmap_init(&cbinfo, size); 538 bzero(cbinfo.dedup, sizeof(*cbinfo.dedup) * 539 HAMMER2_DEDUP_HEUR_SIZE); 540 cbinfo.count_inodes_scanned = 0; 541 cbinfo.count_dirents_scanned = 0; 542 cbinfo.count_bytes_scanned = 0; 543 cbinfo.count_chains_scanned = 0; 544 cbinfo.count_chains_reported = 0; 545 546 incr = size / HAMMER2_FREEMAP_LEVELN_PSIZE * 547 HAMMER2_FREEMAP_LEVEL1_SIZE; 548 if (hmp->total_size - cbinfo.sbase <= incr) { 549 cbinfo.sstop = hmp->total_size; 550 allmedia = 1; 551 } else { 552 cbinfo.sstop = cbinfo.sbase + incr; 553 allmedia = 0; 554 } 555 kprintf("hammer2: pass %016jx-%016jx ", 556 (intmax_t)cbinfo.sbase, 557 (intmax_t)cbinfo.sstop); 558 if (allmedia && cbinfo.sbase == 0) 559 kprintf("(all media)\n"); 560 else if (allmedia) 561 kprintf("(remaining media)\n"); 562 else 563 kprintf("(%jdGB of media)\n", 564 (intmax_t)incr / (1024L*1024*1024)); 565 566 /* 567 * Scan topology for stuff inside this range. 568 * 569 * NOTE - By not using a transaction the operation can 570 * run concurrent with the frontend as well as 571 * with flushes. 572 * 573 * We cannot safely set a mtid without a transaction, 574 * and in fact we don't want to set one anyway. We 575 * want the bulkfree to be passive and no interfere 576 * with crash recovery. 577 */ 578 #undef HAMMER2_BULKFREE_TRANS /* undef - don't use transaction */ 579 #ifdef HAMMER2_BULKFREE_TRANS 580 hammer2_trans_init(hmp->spmp, 0); 581 cbinfo.mtid = hammer2_trans_sub(hmp->spmp); 582 #else 583 cbinfo.mtid = 0; 584 #endif 585 cbinfo.pri = 0; 586 error |= hammer2_bulk_scan(vchain, 587 h2_bulkfree_callback, &cbinfo); 588 589 while ((save = TAILQ_FIRST(&cbinfo.list)) != NULL && 590 (error & ~HAMMER2_ERROR_CHECK) == 0) { 591 TAILQ_REMOVE(&cbinfo.list, save, entry); 592 cbinfo.pri = 0; 593 error |= hammer2_bulk_scan(save->chain, 594 h2_bulkfree_callback, 595 &cbinfo); 596 hammer2_chain_drop(save->chain); 597 kfree(save, M_HAMMER2); 598 } 599 while (save) { 600 TAILQ_REMOVE(&cbinfo.list, save, entry); 601 hammer2_chain_drop(save->chain); 602 kfree(save, M_HAMMER2); 603 save = TAILQ_FIRST(&cbinfo.list); 604 } 605 606 /* 607 * If the complete scan succeeded we can synchronize our 608 * in-memory freemap against live storage. If an abort 609 * occured we cannot safely synchronize our partially 610 * filled-out in-memory freemap. 611 * 612 * We still synchronize on CHECK failures. That is, we still 613 * want bulkfree to operate even if the filesystem has defects. 614 */ 615 if (error & ~HAMMER2_ERROR_CHECK) { 616 kprintf("bulkfree lastdrop %d %d error=0x%04x\n", 617 vchain->refs, vchain->core.chain_count, error); 618 } else { 619 if (error & HAMMER2_ERROR_CHECK) { 620 kprintf("bulkfree lastdrop %d %d " 621 "(with check errors)\n", 622 vchain->refs, vchain->core.chain_count); 623 } else { 624 kprintf("bulkfree lastdrop %d %d\n", 625 vchain->refs, vchain->core.chain_count); 626 } 627 628 error = h2_bulkfree_sync(&cbinfo); 629 630 hammer2_voldata_lock(hmp); 631 hammer2_voldata_modify(hmp); 632 hmp->voldata.allocator_free += cbinfo.adj_free; 633 hammer2_voldata_unlock(hmp); 634 } 635 636 /* 637 * Cleanup for next loop. 638 */ 639 #ifdef HAMMER2_BULKFREE_TRANS 640 hammer2_trans_done(hmp->spmp, 0); 641 #endif 642 if (error & ~HAMMER2_ERROR_CHECK) 643 break; 644 cbinfo.sbase = cbinfo.sstop; 645 cbinfo.adj_free = 0; 646 } 647 kmem_free_swapbacked(&cbinfo.kp); 648 kfree(cbinfo.dedup, M_HAMMER2); 649 cbinfo.dedup = NULL; 650 651 bfi->sstop = cbinfo.sbase; 652 653 incr = bfi->sstop / (hmp->total_size / 10000); 654 if (incr > 10000) 655 incr = 10000; 656 657 kprintf("bulkfree pass statistics (%d.%02d%% storage processed):\n", 658 (int)incr / 100, 659 (int)incr % 100); 660 661 if (error & ~HAMMER2_ERROR_CHECK) { 662 kprintf(" bulkfree was aborted\n"); 663 } else { 664 if (error & HAMMER2_ERROR_CHECK) { 665 kprintf(" WARNING: bulkfree " 666 "encountered CRC errors\n"); 667 } 668 kprintf(" transition->free %ld\n", cbinfo.count_10_00); 669 kprintf(" transition->staged %ld\n", cbinfo.count_11_10); 670 kprintf(" ERR(00)->allocated %ld\n", cbinfo.count_00_11); 671 kprintf(" ERR(01)->allocated %ld\n", cbinfo.count_01_11); 672 kprintf(" staged->allocated %ld\n", cbinfo.count_10_11); 673 kprintf(" ~2MB segs cleaned %ld\n", cbinfo.count_l0cleans); 674 kprintf(" linear adjusts %ld\n", 675 cbinfo.count_linadjusts); 676 kprintf(" dedup factor %ld\n", 677 cbinfo.count_dedup_factor); 678 } 679 680 return error; 681 } 682 683 static void 684 cbinfo_bmap_init(hammer2_bulkfree_info_t *cbinfo, size_t size) 685 { 686 hammer2_bmap_data_t *bmap = cbinfo->bmap; 687 hammer2_key_t key = cbinfo->sbase; 688 hammer2_key_t lokey; 689 hammer2_key_t hikey; 690 691 lokey = (cbinfo->hmp->voldata.allocator_beg + HAMMER2_SEGMASK64) & 692 ~HAMMER2_SEGMASK64; 693 hikey = cbinfo->hmp->total_size & ~HAMMER2_SEGMASK64; 694 695 bzero(bmap, size); 696 while (size) { 697 bzero(bmap, sizeof(*bmap)); 698 if (lokey < H2FMBASE(key, HAMMER2_FREEMAP_LEVEL1_RADIX)) 699 lokey = H2FMBASE(key, HAMMER2_FREEMAP_LEVEL1_RADIX); 700 if (lokey < H2FMZONEBASE(key) + HAMMER2_ZONE_SEG64) 701 lokey = H2FMZONEBASE(key) + HAMMER2_ZONE_SEG64; 702 if (key < lokey || key >= hikey) { 703 memset(bmap->bitmapq, -1, 704 sizeof(bmap->bitmapq)); 705 bmap->avail = 0; 706 bmap->linear = HAMMER2_SEGSIZE; 707 } else { 708 bmap->avail = HAMMER2_FREEMAP_LEVEL0_SIZE; 709 } 710 size -= sizeof(*bmap); 711 key += HAMMER2_FREEMAP_LEVEL0_SIZE; 712 ++bmap; 713 } 714 } 715 716 static int 717 h2_bulkfree_callback(hammer2_bulkfree_info_t *cbinfo, hammer2_blockref_t *bref) 718 { 719 hammer2_bmap_data_t *bmap; 720 hammer2_off_t data_off; 721 uint16_t class; 722 size_t bytes; 723 int radix; 724 725 /* 726 * Check for signal and allow yield to userland during scan. 727 */ 728 if (hammer2_signal_check(&cbinfo->save_time)) 729 return HAMMER2_ERROR_ABORTED; 730 731 /* 732 * Deal with kernel thread cpu or I/O hogging by limiting the 733 * number of chains scanned per second to hammer2_bulkfree_tps. 734 * Ignore leaf records (DIRENT and DATA), no per-record I/O is 735 * involved for those since we don't load their data. 736 */ 737 if (bref->type != HAMMER2_BREF_TYPE_DATA && 738 bref->type != HAMMER2_BREF_TYPE_DIRENT) { 739 ++cbinfo->bulkfree_calls; 740 if (cbinfo->bulkfree_calls > hammer2_bulkfree_tps) { 741 int dticks = ticks - cbinfo->bulkfree_ticks; 742 if (dticks < 0) 743 dticks = 0; 744 if (dticks < hz) { 745 tsleep(&cbinfo->bulkfree_ticks, 0, 746 "h2bw", hz - dticks); 747 } 748 cbinfo->bulkfree_calls = 0; 749 cbinfo->bulkfree_ticks = ticks; 750 } 751 } 752 753 /* 754 * Calculate the data offset and determine if it is within 755 * the current freemap range being gathered. 756 */ 757 data_off = bref->data_off & ~HAMMER2_OFF_MASK_RADIX; 758 if (data_off < cbinfo->sbase || data_off >= cbinfo->sstop) 759 return 0; 760 if (data_off < cbinfo->hmp->voldata.allocator_beg) 761 return 0; 762 if (data_off >= cbinfo->hmp->total_size) 763 return 0; 764 765 /* 766 * Calculate the information needed to generate the in-memory 767 * freemap record. 768 * 769 * Hammer2 does not allow allocations to cross the L1 (2GB) boundary, 770 * it's a problem if it does. (Or L0 (2MB) for that matter). 771 */ 772 radix = (int)(bref->data_off & HAMMER2_OFF_MASK_RADIX); 773 KKASSERT(radix != 0); 774 bytes = (size_t)1 << radix; 775 class = (bref->type << 8) | HAMMER2_PBUFRADIX; 776 777 if (data_off + bytes > cbinfo->sstop) { 778 kprintf("hammer2_bulkfree_scan: illegal 2GB boundary " 779 "%016jx %016jx/%d\n", 780 (intmax_t)bref->data_off, 781 (intmax_t)bref->key, 782 bref->keybits); 783 bytes = cbinfo->sstop - data_off; /* XXX */ 784 } 785 786 /* 787 * Convert to a storage offset relative to the beginning of the 788 * storage range we are collecting. Then lookup the level0 bmap entry. 789 */ 790 data_off -= cbinfo->sbase; 791 bmap = cbinfo->bmap + (data_off >> HAMMER2_FREEMAP_LEVEL0_RADIX); 792 793 /* 794 * Convert data_off to a bmap-relative value (~4MB storage range). 795 * Adjust linear, class, and avail. 796 * 797 * Hammer2 does not allow allocations to cross the L0 (4MB) boundary, 798 */ 799 data_off &= HAMMER2_FREEMAP_LEVEL0_MASK; 800 if (data_off + bytes > HAMMER2_FREEMAP_LEVEL0_SIZE) { 801 kprintf("hammer2_bulkfree_scan: illegal 4MB boundary " 802 "%016jx %016jx/%d\n", 803 (intmax_t)bref->data_off, 804 (intmax_t)bref->key, 805 bref->keybits); 806 bytes = HAMMER2_FREEMAP_LEVEL0_SIZE - data_off; 807 } 808 809 if (bmap->class == 0) { 810 bmap->class = class; 811 bmap->avail = HAMMER2_FREEMAP_LEVEL0_SIZE; 812 } 813 814 /* 815 * NOTE: bmap->class does not have to match class. Classification 816 * is relaxed when free space is low, so some mixing can occur. 817 */ 818 #if 0 819 /* 820 * XXX removed 821 */ 822 if (bmap->class != class) { 823 kprintf("hammer2_bulkfree_scan: illegal mixed class " 824 "%016jx %016jx/%d (%04x vs %04x)\n", 825 (intmax_t)bref->data_off, 826 (intmax_t)bref->key, 827 bref->keybits, 828 class, bmap->class); 829 } 830 #endif 831 832 /* 833 * Just record the highest byte-granular offset for now. Do not 834 * match against allocations which are in multiples of whole blocks. 835 * 836 * Make sure that any in-block linear offset at least covers the 837 * data range. This can cause bmap->linear to become block-aligned. 838 */ 839 if (bytes & HAMMER2_FREEMAP_BLOCK_MASK) { 840 if (bmap->linear < (int32_t)data_off + (int32_t)bytes) 841 bmap->linear = (int32_t)data_off + (int32_t)bytes; 842 } else if (bmap->linear >= (int32_t)data_off && 843 bmap->linear < (int32_t)data_off + (int32_t)bytes) { 844 bmap->linear = (int32_t)data_off + (int32_t)bytes; 845 } 846 847 /* 848 * Adjust the hammer2_bitmap_t bitmap[HAMMER2_BMAP_ELEMENTS]. 849 * 64-bit entries, 2 bits per entry, to code 11. 850 * 851 * NOTE: data_off mask to 524288, shift right by 14 (radix for 16384), 852 * and multiply shift amount by 2 for sets of 2 bits. 853 * 854 * NOTE: The allocation can be smaller than HAMMER2_FREEMAP_BLOCK_SIZE. 855 * also, data_off may not be FREEMAP_BLOCK_SIZE aligned. 856 */ 857 while (bytes > 0) { 858 hammer2_bitmap_t bmask; 859 int bindex; 860 861 bindex = (int)data_off >> (HAMMER2_FREEMAP_BLOCK_RADIX + 862 HAMMER2_BMAP_INDEX_RADIX); 863 bmask = (hammer2_bitmap_t)3 << 864 ((((int)data_off & HAMMER2_BMAP_INDEX_MASK) >> 865 HAMMER2_FREEMAP_BLOCK_RADIX) << 1); 866 867 /* 868 * NOTE! The (avail) calculation is bitmap-granular. Multiple 869 * sub-granular records can wind up at the same bitmap 870 * position. 871 */ 872 if ((bmap->bitmapq[bindex] & bmask) == 0) { 873 if (bytes < HAMMER2_FREEMAP_BLOCK_SIZE) { 874 bmap->avail -= HAMMER2_FREEMAP_BLOCK_SIZE; 875 } else { 876 bmap->avail -= bytes; 877 } 878 bmap->bitmapq[bindex] |= bmask; 879 } 880 data_off += HAMMER2_FREEMAP_BLOCK_SIZE; 881 if (bytes < HAMMER2_FREEMAP_BLOCK_SIZE) 882 bytes = 0; 883 else 884 bytes -= HAMMER2_FREEMAP_BLOCK_SIZE; 885 } 886 return 0; 887 } 888 889 /* 890 * Synchronize the in-memory bitmap with the live freemap. This is not a 891 * direct copy. Instead the bitmaps must be compared: 892 * 893 * In-memory Live-freemap 894 * 00 11 -> 10 (do nothing if live modified) 895 * 10 -> 00 (do nothing if live modified) 896 * 11 10 -> 11 handles race against live 897 * ** -> 11 nominally warn of corruption 898 * 899 * We must also fixup the hints in HAMMER2_BREF_TYPE_FREEMAP_LEAF. 900 */ 901 static int 902 h2_bulkfree_sync(hammer2_bulkfree_info_t *cbinfo) 903 { 904 hammer2_off_t data_off; 905 hammer2_key_t key; 906 hammer2_key_t key_dummy; 907 hammer2_bmap_data_t *bmap; 908 hammer2_bmap_data_t *live; 909 hammer2_chain_t *live_parent; 910 hammer2_chain_t *live_chain; 911 int bmapindex; 912 int error; 913 914 kprintf("hammer2_bulkfree - range "); 915 916 if (cbinfo->sbase < cbinfo->hmp->voldata.allocator_beg) 917 kprintf("%016jx-", 918 (intmax_t)cbinfo->hmp->voldata.allocator_beg); 919 else 920 kprintf("%016jx-", 921 (intmax_t)cbinfo->sbase); 922 923 if (cbinfo->sstop > cbinfo->hmp->total_size) 924 kprintf("%016jx\n", 925 (intmax_t)cbinfo->hmp->total_size); 926 else 927 kprintf("%016jx\n", 928 (intmax_t)cbinfo->sstop); 929 930 data_off = cbinfo->sbase; 931 bmap = cbinfo->bmap; 932 933 live_parent = &cbinfo->hmp->fchain; 934 hammer2_chain_ref(live_parent); 935 hammer2_chain_lock(live_parent, HAMMER2_RESOLVE_ALWAYS); 936 live_chain = NULL; 937 error = 0; 938 939 /* 940 * Iterate each hammer2_bmap_data_t line (128 bytes) managing 941 * 4MB of storage. 942 */ 943 while (data_off < cbinfo->sstop) { 944 /* 945 * The freemap is not used below allocator_beg or beyond 946 * total_size. 947 */ 948 949 if (data_off < cbinfo->hmp->voldata.allocator_beg) 950 goto next; 951 if (data_off >= cbinfo->hmp->total_size) 952 goto next; 953 954 /* 955 * Locate the freemap leaf on the live filesystem 956 */ 957 key = (data_off & ~HAMMER2_FREEMAP_LEVEL1_MASK); 958 959 if (live_chain == NULL || live_chain->bref.key != key) { 960 if (live_chain) { 961 hammer2_chain_unlock(live_chain); 962 hammer2_chain_drop(live_chain); 963 } 964 live_chain = hammer2_chain_lookup( 965 &live_parent, 966 &key_dummy, 967 key, 968 key + HAMMER2_FREEMAP_LEVEL1_MASK, 969 &error, 970 HAMMER2_LOOKUP_ALWAYS); 971 if (error) { 972 kprintf("hammer2_bulkfree: freemap lookup " 973 "error near %016jx, error %s\n", 974 (intmax_t)data_off, 975 hammer2_error_str(live_chain->error)); 976 break; 977 } 978 } 979 if (live_chain == NULL) { 980 /* 981 * XXX if we implement a full recovery mode we need 982 * to create/recreate missing freemap chains if our 983 * bmap has any allocated blocks. 984 */ 985 if (bmap->class && 986 bmap->avail != HAMMER2_FREEMAP_LEVEL0_SIZE) { 987 kprintf("hammer2_bulkfree: cannot locate " 988 "live leaf for allocated data " 989 "near %016jx\n", 990 (intmax_t)data_off); 991 } 992 goto next; 993 } 994 if (live_chain->error) { 995 kprintf("hammer2_bulkfree: unable to access freemap " 996 "near %016jx, error %s\n", 997 (intmax_t)data_off, 998 hammer2_error_str(live_chain->error)); 999 hammer2_chain_unlock(live_chain); 1000 hammer2_chain_drop(live_chain); 1001 live_chain = NULL; 1002 goto next; 1003 } 1004 1005 bmapindex = (data_off & HAMMER2_FREEMAP_LEVEL1_MASK) >> 1006 HAMMER2_FREEMAP_LEVEL0_RADIX; 1007 live = &live_chain->data->bmdata[bmapindex]; 1008 1009 /* 1010 * Shortcut if the bitmaps match and the live linear 1011 * indicator is sane. We can't do a perfect check of 1012 * live->linear because the only real requirement is that 1013 * if it is not block-aligned, that it not cover the space 1014 * within its current block which overlaps one of the data 1015 * ranges we scan. We don't retain enough fine-grained 1016 * data in our scan to be able to set it exactly. 1017 * 1018 * TODO - we could shortcut this by testing that both 1019 * live->class and bmap->class are 0, and both avails are 1020 * set to HAMMER2_FREEMAP_LEVEL0_SIZE (4MB). 1021 */ 1022 if (bcmp(live->bitmapq, bmap->bitmapq, 1023 sizeof(bmap->bitmapq)) == 0 && 1024 live->linear >= bmap->linear) { 1025 goto next; 1026 } 1027 if (hammer2_debug & 1) { 1028 kprintf("live %016jx %04d.%04x (avail=%d)\n", 1029 data_off, bmapindex, live->class, live->avail); 1030 } 1031 1032 hammer2_chain_modify(live_chain, cbinfo->mtid, 0, 0); 1033 live_chain->bref.check.freemap.bigmask = -1; 1034 cbinfo->hmp->freemap_relaxed = 0; /* reset heuristic */ 1035 live = &live_chain->data->bmdata[bmapindex]; 1036 1037 h2_bulkfree_sync_adjust(cbinfo, data_off, live, bmap, 1038 live_chain->bref.key + 1039 bmapindex * 1040 HAMMER2_FREEMAP_LEVEL0_SIZE); 1041 next: 1042 data_off += HAMMER2_FREEMAP_LEVEL0_SIZE; 1043 ++bmap; 1044 } 1045 if (live_chain) { 1046 hammer2_chain_unlock(live_chain); 1047 hammer2_chain_drop(live_chain); 1048 } 1049 if (live_parent) { 1050 hammer2_chain_unlock(live_parent); 1051 hammer2_chain_drop(live_parent); 1052 } 1053 return error; 1054 } 1055 1056 /* 1057 * Merge the bulkfree bitmap against the existing bitmap. 1058 */ 1059 static 1060 void 1061 h2_bulkfree_sync_adjust(hammer2_bulkfree_info_t *cbinfo, 1062 hammer2_off_t data_off, hammer2_bmap_data_t *live, 1063 hammer2_bmap_data_t *bmap, hammer2_key_t alloc_base) 1064 { 1065 int bindex; 1066 int scount; 1067 hammer2_off_t tmp_off; 1068 hammer2_bitmap_t lmask; 1069 hammer2_bitmap_t mmask; 1070 1071 tmp_off = data_off; 1072 1073 for (bindex = 0; bindex < HAMMER2_BMAP_ELEMENTS; ++bindex) { 1074 lmask = live->bitmapq[bindex]; /* live */ 1075 mmask = bmap->bitmapq[bindex]; /* snapshotted bulkfree */ 1076 if (lmask == mmask) { 1077 tmp_off += HAMMER2_BMAP_INDEX_SIZE; 1078 continue; 1079 } 1080 1081 for (scount = 0; 1082 scount < HAMMER2_BMAP_BITS_PER_ELEMENT; 1083 scount += 2) { 1084 if ((mmask & 3) == 0) { 1085 /* 1086 * in-memory 00 live 11 -> 10 1087 * live 10 -> 00 1088 * 1089 * Storage might be marked allocated or 1090 * staged and must be remarked staged or 1091 * free. 1092 */ 1093 switch (lmask & 3) { 1094 case 0: /* 00 */ 1095 break; 1096 case 1: /* 01 */ 1097 kprintf("hammer2_bulkfree: cannot " 1098 "transition m=00/l=01\n"); 1099 break; 1100 case 2: /* 10 -> 00 */ 1101 live->bitmapq[bindex] &= 1102 ~((hammer2_bitmap_t)2 << scount); 1103 live->avail += 1104 HAMMER2_FREEMAP_BLOCK_SIZE; 1105 if (live->avail > 1106 HAMMER2_FREEMAP_LEVEL0_SIZE) { 1107 live->avail = 1108 HAMMER2_FREEMAP_LEVEL0_SIZE; 1109 } 1110 cbinfo->adj_free += 1111 HAMMER2_FREEMAP_BLOCK_SIZE; 1112 ++cbinfo->count_10_00; 1113 hammer2_io_dedup_assert( 1114 cbinfo->hmp, 1115 tmp_off | 1116 HAMMER2_FREEMAP_BLOCK_RADIX, 1117 HAMMER2_FREEMAP_BLOCK_SIZE); 1118 break; 1119 case 3: /* 11 -> 10 */ 1120 live->bitmapq[bindex] &= 1121 ~((hammer2_bitmap_t)1 << scount); 1122 ++cbinfo->count_11_10; 1123 hammer2_io_dedup_delete( 1124 cbinfo->hmp, 1125 HAMMER2_BREF_TYPE_DATA, 1126 tmp_off | 1127 HAMMER2_FREEMAP_BLOCK_RADIX, 1128 HAMMER2_FREEMAP_BLOCK_SIZE); 1129 break; 1130 } 1131 } else if ((mmask & 3) == 3) { 1132 /* 1133 * in-memory 11 live 10 -> 11 1134 * live ** -> 11 1135 * 1136 * Storage might be incorrectly marked free 1137 * or staged and must be remarked fully 1138 * allocated. 1139 */ 1140 switch (lmask & 3) { 1141 case 0: /* 00 */ 1142 ++cbinfo->count_00_11; 1143 cbinfo->adj_free -= 1144 HAMMER2_FREEMAP_BLOCK_SIZE; 1145 live->avail -= 1146 HAMMER2_FREEMAP_BLOCK_SIZE; 1147 if ((int32_t)live->avail < 0) 1148 live->avail = 0; 1149 break; 1150 case 1: /* 01 */ 1151 ++cbinfo->count_01_11; 1152 break; 1153 case 2: /* 10 -> 11 */ 1154 ++cbinfo->count_10_11; 1155 break; 1156 case 3: /* 11 */ 1157 break; 1158 } 1159 live->bitmapq[bindex] |= 1160 ((hammer2_bitmap_t)3 << scount); 1161 } 1162 mmask >>= 2; 1163 lmask >>= 2; 1164 tmp_off += HAMMER2_FREEMAP_BLOCK_SIZE; 1165 } 1166 } 1167 1168 /* 1169 * Determine if the live bitmap is completely free and reset its 1170 * fields if so. Otherwise check to see if we can reduce the linear 1171 * offset. 1172 */ 1173 for (bindex = HAMMER2_BMAP_ELEMENTS - 1; bindex >= 0; --bindex) { 1174 if (live->bitmapq[bindex] != 0) 1175 break; 1176 } 1177 if (bindex < 0) { 1178 /* 1179 * Completely empty, reset entire segment 1180 */ 1181 #if 0 1182 kprintf("hammer2: cleanseg %016jx.%04x (%d)\n", 1183 alloc_base, live->class, live->avail); 1184 #endif 1185 live->avail = HAMMER2_FREEMAP_LEVEL0_SIZE; 1186 live->class = 0; 1187 live->linear = 0; 1188 ++cbinfo->count_l0cleans; 1189 } else if (bindex < 7) { 1190 /* 1191 * Partially full, bitmapq[bindex] != 0. Our bulkfree pass 1192 * does not record enough information to set live->linear 1193 * exactly. 1194 * 1195 * NOTE: Setting live->linear to a sub-block (16K) boundary 1196 * forces the live code to iterate to the next fully 1197 * free block. It does NOT mean that all blocks above 1198 * live->linear are available. 1199 * 1200 * Setting live->linear to a fragmentary (less than 1201 * 16K) boundary allows allocations to iterate within 1202 * that sub-block. 1203 */ 1204 if (live->linear < bmap->linear && 1205 ((live->linear ^ bmap->linear) & 1206 ~HAMMER2_FREEMAP_BLOCK_MASK) == 0) { 1207 /* 1208 * If greater than but still within the same 1209 * sub-block as live we can adjust linear upward. 1210 */ 1211 live->linear = bmap->linear; 1212 ++cbinfo->count_linadjusts; 1213 } else { 1214 /* 1215 * Otherwise adjust to the nearest higher or same 1216 * sub-block boundary. The live system may have 1217 * bounced live->linear around so we cannot make any 1218 * assumptions with regards to available fragmentary 1219 * allocations. 1220 */ 1221 live->linear = 1222 (bmap->linear + HAMMER2_FREEMAP_BLOCK_MASK) & 1223 ~HAMMER2_FREEMAP_BLOCK_MASK; 1224 ++cbinfo->count_linadjusts; 1225 } 1226 } else { 1227 /* 1228 * Completely full, effectively disable the linear iterator 1229 */ 1230 live->linear = HAMMER2_SEGSIZE; 1231 } 1232 1233 #if 0 1234 if (bmap->class) { 1235 kprintf("%016jx %04d.%04x (avail=%7d) " 1236 "%08x %08x %08x %08x %08x %08x %08x %08x\n", 1237 (intmax_t)data_off, 1238 (int)((data_off & 1239 HAMMER2_FREEMAP_LEVEL1_MASK) >> 1240 HAMMER2_FREEMAP_LEVEL0_RADIX), 1241 bmap->class, 1242 bmap->avail, 1243 bmap->bitmap[0], bmap->bitmap[1], 1244 bmap->bitmap[2], bmap->bitmap[3], 1245 bmap->bitmap[4], bmap->bitmap[5], 1246 bmap->bitmap[6], bmap->bitmap[7]); 1247 } 1248 #endif 1249 } 1250 1251 /* 1252 * BULKFREE DEDUP HEURISTIC 1253 * 1254 * WARNING! This code is SMP safe but the heuristic allows SMP collisions. 1255 * All fields must be loaded into locals and validated. 1256 */ 1257 static 1258 int 1259 h2_bulkfree_test(hammer2_bulkfree_info_t *cbinfo, hammer2_blockref_t *bref, 1260 int pri, int saved_error) 1261 { 1262 hammer2_dedup_t *dedup; 1263 int best; 1264 int n; 1265 int i; 1266 1267 n = hammer2_icrc32(&bref->data_off, sizeof(bref->data_off)); 1268 dedup = cbinfo->dedup + (n & (HAMMER2_DEDUP_HEUR_MASK & ~7)); 1269 1270 for (i = best = 0; i < 8; ++i) { 1271 if (dedup[i].data_off == bref->data_off) { 1272 if (dedup[i].ticks < pri) 1273 dedup[i].ticks = pri; 1274 if (pri == 1) 1275 cbinfo->count_dedup_factor += dedup[i].ticks; 1276 return (dedup[i].saved_error | HAMMER2_ERROR_EOF); 1277 } 1278 if (dedup[i].ticks < dedup[best].ticks) 1279 best = i; 1280 } 1281 dedup[best].data_off = bref->data_off; 1282 dedup[best].ticks = pri; 1283 dedup[best].saved_error = saved_error; 1284 1285 return 0; 1286 } 1287