1 /* 2 * Copyright (c) 2011-2018 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@dragonflybsd.org> 6 * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in 16 * the documentation and/or other materials provided with the 17 * distribution. 18 * 3. Neither the name of The DragonFly Project nor the names of its 19 * contributors may be used to endorse or promote products derived 20 * from this software without specific, prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 25 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 26 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 28 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 29 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 30 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 31 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 32 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 */ 35 /* 36 * TRANSACTION AND FLUSH HANDLING 37 * 38 * Deceptively simple but actually fairly difficult to implement properly is 39 * how I would describe it. 40 * 41 * Flushing generally occurs bottom-up but requires a top-down scan to 42 * locate chains with MODIFIED and/or UPDATE bits set. The ONFLUSH flag 43 * tells how to recurse downward to find these chains. 44 */ 45 46 #include <sys/cdefs.h> 47 #include <sys/param.h> 48 #include <sys/systm.h> 49 #include <sys/types.h> 50 #include <sys/lock.h> 51 #include <sys/uuid.h> 52 53 #include "hammer2.h" 54 55 #define FLUSH_DEBUG 0 56 57 #define HAMMER2_FLUSH_DEPTH_LIMIT 60 /* stack recursion limit */ 58 59 60 /* 61 * Recursively flush the specified chain. The chain is locked and 62 * referenced by the caller and will remain so on return. The chain 63 * will remain referenced throughout but can temporarily lose its 64 * lock during the recursion to avoid unnecessarily stalling user 65 * processes. 66 */ 67 struct hammer2_flush_info { 68 hammer2_chain_t *parent; 69 int depth; 70 int error; /* cumulative error */ 71 int flags; 72 #ifdef HAMMER2_SCAN_DEBUG 73 long scan_count; 74 long scan_mod_count; 75 long scan_upd_count; 76 long scan_onf_count; 77 long scan_del_count; 78 long scan_btype[7]; 79 #endif 80 hammer2_chain_t *debug; 81 }; 82 83 typedef struct hammer2_flush_info hammer2_flush_info_t; 84 85 static int hammer2_flush_core(hammer2_flush_info_t *info, 86 hammer2_chain_t *chain, int flags); 87 static int hammer2_flush_recurse(hammer2_chain_t *child, void *data); 88 89 /* 90 * Any per-pfs transaction initialization goes here. 91 */ 92 void 93 hammer2_trans_manage_init(hammer2_pfs_t *pmp) 94 { 95 } 96 97 /* 98 * Transaction support for any modifying operation. Transactions are used 99 * in the pmp layer by the frontend and in the spmp layer by the backend. 100 * 101 * 0 - Normal transaction. Interlocks against just the 102 * COPYQ portion of an ISFLUSH transaction. 103 * 104 * TRANS_ISFLUSH - Flush transaction. Interlocks against other flush 105 * transactions. 106 * 107 * When COPYQ is also specified, waits for the count 108 * to drop to 1. 109 * 110 * TRANS_BUFCACHE - Buffer cache transaction. No interlock. 111 * 112 * TRANS_SIDEQ - Run the sideq (only tested in trans_done()) 113 * 114 * Initializing a new transaction allocates a transaction ID. Typically 115 * passed a pmp (hmp passed as NULL), indicating a cluster transaction. Can 116 * be passed a NULL pmp and non-NULL hmp to indicate a transaction on a single 117 * media target. The latter mode is used by the recovery code. 118 */ 119 void 120 hammer2_trans_init(hammer2_pfs_t *pmp, uint32_t flags) 121 { 122 uint32_t oflags; 123 uint32_t nflags; 124 int dowait; 125 126 for (;;) { 127 oflags = pmp->trans.flags; 128 cpu_ccfence(); 129 dowait = 0; 130 131 if (flags & HAMMER2_TRANS_ISFLUSH) { 132 /* 133 * Interlock against other flush transactions. 134 */ 135 if (oflags & HAMMER2_TRANS_ISFLUSH) { 136 nflags = oflags | HAMMER2_TRANS_WAITING; 137 dowait = 1; 138 } else { 139 nflags = (oflags | flags) + 1; 140 } 141 } else if (flags & HAMMER2_TRANS_BUFCACHE) { 142 /* 143 * Requesting strategy transaction from buffer-cache, 144 * or a VM getpages/putpages through the buffer cache. 145 * We must allow such transactions in all situations 146 * to avoid deadlocks. 147 */ 148 nflags = (oflags | flags) + 1; 149 } else { 150 /* 151 * Normal transaction. We do not interlock against 152 * BUFCACHE or ISFLUSH. 153 * 154 * Note that vnode locks may be held going into 155 * this call. 156 * 157 * NOTE: Remember that non-modifying operations 158 * such as read, stat, readdir, etc, do 159 * not use transactions. 160 */ 161 nflags = (oflags | flags) + 1; 162 } 163 if (dowait) 164 tsleep_interlock(&pmp->trans.sync_wait, 0); 165 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) { 166 if (dowait == 0) 167 break; 168 tsleep(&pmp->trans.sync_wait, PINTERLOCKED, 169 "h2trans", hz); 170 /* retry */ 171 } else { 172 cpu_pause(); 173 /* retry */ 174 } 175 /* retry */ 176 } 177 178 #if 0 179 /* 180 * When entering a FLUSH transaction with COPYQ set, wait for the 181 * transaction count to drop to 1 (our flush transaction only) 182 * before proceeding. 183 * 184 * This waits for all non-flush transactions to complete and blocks 185 * new non-flush transactions from starting until COPYQ is cleared. 186 * (the flush will then proceed after clearing COPYQ). This should 187 * be a very short stall on modifying operations. 188 */ 189 while ((flags & HAMMER2_TRANS_ISFLUSH) && 190 (flags & HAMMER2_TRANS_COPYQ)) { 191 oflags = pmp->trans.flags; 192 cpu_ccfence(); 193 if ((oflags & HAMMER2_TRANS_MASK) == 1) 194 break; 195 nflags = oflags | HAMMER2_TRANS_WAITING; 196 tsleep_interlock(&pmp->trans.sync_wait, 0); 197 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) { 198 tsleep(&pmp->trans.sync_wait, PINTERLOCKED, 199 "h2trans2", hz); 200 } 201 } 202 #endif 203 } 204 205 /* 206 * Start a sub-transaction, there is no 'subdone' function. This will 207 * issue a new modify_tid (mtid) for the current transaction, which is a 208 * CLC (cluster level change) id and not a per-node id. 209 * 210 * This function must be called for each XOP when multiple XOPs are run in 211 * sequence within a transaction. 212 * 213 * Callers typically update the inode with the transaction mtid manually 214 * to enforce sequencing. 215 */ 216 hammer2_tid_t 217 hammer2_trans_sub(hammer2_pfs_t *pmp) 218 { 219 hammer2_tid_t mtid; 220 221 mtid = atomic_fetchadd_64(&pmp->modify_tid, 1); 222 223 return (mtid); 224 } 225 226 void 227 hammer2_trans_setflags(hammer2_pfs_t *pmp, uint32_t flags) 228 { 229 atomic_set_int(&pmp->trans.flags, flags); 230 } 231 232 /* 233 * Typically used to clear trans flags asynchronously. If TRANS_WAITING 234 * is in the mask, and was previously set, this function will wake up 235 * any waiters. 236 */ 237 void 238 hammer2_trans_clearflags(hammer2_pfs_t *pmp, uint32_t flags) 239 { 240 uint32_t oflags; 241 uint32_t nflags; 242 243 for (;;) { 244 oflags = pmp->trans.flags; 245 cpu_ccfence(); 246 nflags = oflags & ~flags; 247 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) { 248 if ((oflags ^ nflags) & HAMMER2_TRANS_WAITING) 249 wakeup(&pmp->trans.sync_wait); 250 break; 251 } 252 cpu_pause(); 253 /* retry */ 254 } 255 } 256 257 void 258 hammer2_trans_done(hammer2_pfs_t *pmp, uint32_t flags) 259 { 260 uint32_t oflags; 261 uint32_t nflags; 262 263 #if 0 264 /* 265 * Modifying ops on the front-end can cause dirty inodes to 266 * build up in the sideq. We don't flush these on inactive/reclaim 267 * due to potential deadlocks, so we have to deal with them from 268 * inside other nominal modifying front-end transactions. 269 */ 270 if ((flags & HAMMER2_TRANS_SIDEQ) && 271 pmp->sideq_count > hammer2_limit_dirty_inodes / 2 && 272 pmp->sideq_count > (pmp->inum_count >> 3) && 273 pmp->mp) { 274 speedup_syncer(pmp->mp); 275 } 276 #endif 277 278 /* 279 * Clean-up the transaction. Wakeup any waiters when finishing 280 * a flush transaction or transitioning the non-flush transaction 281 * count from 2->1 while a flush transaction is pending. 282 */ 283 for (;;) { 284 oflags = pmp->trans.flags; 285 cpu_ccfence(); 286 KKASSERT(oflags & HAMMER2_TRANS_MASK); 287 288 nflags = (oflags - 1) & ~flags; 289 if (flags & HAMMER2_TRANS_ISFLUSH) { 290 nflags &= ~HAMMER2_TRANS_WAITING; 291 } 292 if ((oflags & (HAMMER2_TRANS_ISFLUSH|HAMMER2_TRANS_MASK)) == 293 (HAMMER2_TRANS_ISFLUSH|2)) { 294 nflags &= ~HAMMER2_TRANS_WAITING; 295 } 296 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) { 297 if ((oflags ^ nflags) & HAMMER2_TRANS_WAITING) 298 wakeup(&pmp->trans.sync_wait); 299 break; 300 } 301 cpu_pause(); 302 /* retry */ 303 } 304 } 305 306 /* 307 * Obtain new, unique inode number (not serialized by caller). 308 */ 309 hammer2_tid_t 310 hammer2_trans_newinum(hammer2_pfs_t *pmp) 311 { 312 hammer2_tid_t tid; 313 314 tid = atomic_fetchadd_64(&pmp->inode_tid, 1); 315 316 return tid; 317 } 318 319 /* 320 * Assert that a strategy call is ok here. Currently we allow strategy 321 * calls in all situations, including during flushes. Previously: 322 * (old) (1) In a normal transaction. 323 * (old) (2) In a flush transaction only if PREFLUSH is also set. 324 */ 325 void 326 hammer2_trans_assert_strategy(hammer2_pfs_t *pmp) 327 { 328 #if 0 329 KKASSERT((pmp->trans.flags & HAMMER2_TRANS_ISFLUSH) == 0 || 330 (pmp->trans.flags & HAMMER2_TRANS_PREFLUSH)); 331 #endif 332 } 333 334 /* 335 * Flush the chain and all modified sub-chains through the specified 336 * synchronization point, propagating blockref updates back up. As 337 * part of this propagation, mirror_tid and inode/data usage statistics 338 * propagates back upward. 339 * 340 * Returns a HAMMER2 error code, 0 if no error. Note that I/O errors from 341 * buffers dirtied during the flush operation can occur later. 342 * 343 * modify_tid (clc - cluster level change) is not propagated. 344 * 345 * update_tid (clc) is used for validation and is not propagated by this 346 * function. 347 * 348 * This routine can be called from several places but the most important 349 * is from VFS_SYNC (frontend) via hammer2_xop_inode_flush (backend). 350 * 351 * chain is locked on call and will remain locked on return. The chain's 352 * UPDATE flag indicates that its parent's block table (which is not yet 353 * part of the flush) should be updated. 354 * 355 * flags: 356 * HAMMER2_FLUSH_TOP Indicates that this is the top of the flush. 357 * Is cleared for the recursion. 358 * 359 * HAMMER2_FLUSH_ALL Recurse everything 360 * 361 * HAMMER2_FLUSH_INODE_STOP 362 * Stop at PFS inode or normal inode boundary 363 */ 364 int 365 hammer2_flush(hammer2_chain_t *chain, int flags) 366 { 367 hammer2_flush_info_t info; 368 hammer2_dev_t *hmp; 369 int loops; 370 371 /* 372 * Execute the recursive flush and handle deferrals. 373 * 374 * Chains can be ridiculously long (thousands deep), so to 375 * avoid blowing out the kernel stack the recursive flush has a 376 * depth limit. Elements at the limit are placed on a list 377 * for re-execution after the stack has been popped. 378 */ 379 bzero(&info, sizeof(info)); 380 info.flags = flags & ~HAMMER2_FLUSH_TOP; 381 382 /* 383 * Calculate parent (can be NULL), if not NULL the flush core 384 * expects the parent to be referenced so it can easily lock/unlock 385 * it without it getting ripped up. 386 */ 387 if ((info.parent = chain->parent) != NULL) 388 hammer2_chain_ref(info.parent); 389 390 /* 391 * Extra ref needed because flush_core expects it when replacing 392 * chain. 393 */ 394 hammer2_chain_ref(chain); 395 hmp = chain->hmp; 396 loops = 0; 397 398 for (;;) { 399 /* 400 * [re]flush chain as the deep recursion may have generated 401 * additional modifications. 402 */ 403 if (info.parent != chain->parent) { 404 if (hammer2_debug & 0x0040) { 405 kprintf("LOST CHILD4 %p->%p " 406 "(actual parent %p)\n", 407 info.parent, chain, chain->parent); 408 } 409 hammer2_chain_drop(info.parent); 410 info.parent = chain->parent; 411 hammer2_chain_ref(info.parent); 412 } 413 if (hammer2_flush_core(&info, chain, flags) == 0) 414 break; 415 416 if (++loops % 1000 == 0) { 417 kprintf("hammer2_flush: excessive loops on %p\n", 418 chain); 419 if (hammer2_debug & 0x100000) 420 Debugger("hell4"); 421 } 422 } 423 #ifdef HAMMER2_SCAN_DEBUG 424 if (info.scan_count >= 10) 425 kprintf("hammer2_flush: scan_count %ld (%ld,%ld,%ld,%ld) " 426 "bt(%ld,%ld,%ld,%ld,%ld,%ld)\n", 427 info.scan_count, 428 info.scan_mod_count, 429 info.scan_upd_count, 430 info.scan_onf_count, 431 info.scan_del_count, 432 info.scan_btype[1], 433 info.scan_btype[2], 434 info.scan_btype[3], 435 info.scan_btype[4], 436 info.scan_btype[5], 437 info.scan_btype[6]); 438 #endif 439 hammer2_chain_drop(chain); 440 if (info.parent) 441 hammer2_chain_drop(info.parent); 442 return (info.error); 443 } 444 445 /* 446 * This is the core of the chain flushing code. The chain is locked by the 447 * caller and must also have an extra ref on it by the caller, and remains 448 * locked and will have an extra ref on return. info.parent is referenced 449 * but not locked. 450 * 451 * Upon return, the caller can test the UPDATE bit on the chain to determine 452 * if the parent needs updating. 453 * 454 * If non-zero is returned, the chain's parent changed during the flush and 455 * the caller must retry the operation. 456 * 457 * (1) Determine if this node is a candidate for the flush, return if it is 458 * not. fchain and vchain are always candidates for the flush. 459 * 460 * (2) If we recurse too deep the chain is entered onto the deferral list and 461 * the current flush stack is aborted until after the deferral list is 462 * run. 463 * 464 * (3) Recursively flush live children (rbtree). This can create deferrals. 465 * A successful flush clears the MODIFIED and UPDATE bits on the children 466 * and typically causes the parent to be marked MODIFIED as the children 467 * update the parent's block table. A parent might already be marked 468 * MODIFIED due to a deletion (whos blocktable update in the parent is 469 * handled by the frontend), or if the parent itself is modified by the 470 * frontend for other reasons. 471 * 472 * (4) Permanently disconnected sub-trees are cleaned up by the front-end. 473 * Deleted-but-open inodes can still be individually flushed via the 474 * filesystem syncer. 475 * 476 * (5) Delete parents on the way back up if they are normal indirect blocks 477 * and have no children. 478 * 479 * (6) Note that an unmodified child may still need the block table in its 480 * parent updated (e.g. rename/move). The child will have UPDATE set 481 * in this case. 482 * 483 * WARNING ON BREF MODIFY_TID/MIRROR_TID 484 * 485 * blockref.modify_tid is consistent only within a PFS, and will not be 486 * consistent during synchronization. mirror_tid is consistent across the 487 * block device regardless of the PFS. 488 */ 489 static int 490 hammer2_flush_core(hammer2_flush_info_t *info, hammer2_chain_t *chain, 491 int flags) 492 { 493 hammer2_chain_t *parent; 494 hammer2_dev_t *hmp; 495 int save_error; 496 int retry; 497 498 retry = 0; 499 500 /* 501 * (1) Optimize downward recursion to locate nodes needing action. 502 * Nothing to do if none of these flags are set. 503 */ 504 if ((chain->flags & HAMMER2_CHAIN_FLUSH_MASK) == 0) { 505 if (hammer2_debug & 0x200) { 506 if (info->debug == NULL) 507 info->debug = chain; 508 } else { 509 return 0; 510 } 511 } 512 513 hmp = chain->hmp; 514 515 /* 516 * NOTE: parent can be NULL, usually due to destroy races. 517 */ 518 parent = info->parent; 519 KKASSERT(chain->parent == parent); 520 521 /* 522 * Downward search recursion 523 * 524 * We must be careful on cold stops, which often occur on inode 525 * boundaries due to the way hammer2_vfs_sync() sequences the flush. 526 * Be sure to issue an appropriate chain_setflush() 527 */ 528 if ((chain->flags & HAMMER2_CHAIN_PFSBOUNDARY) && 529 (flags & HAMMER2_FLUSH_ALL) == 0 && 530 (flags & HAMMER2_FLUSH_TOP) == 0 && 531 chain->pmp && chain->pmp->mp) { 532 /* 533 * If FLUSH_ALL is not specified the caller does not want 534 * to recurse through PFS roots that have been mounted. 535 * 536 * (If the PFS has not been mounted there may not be 537 * anything monitoring its chains and its up to us 538 * to flush it). 539 * 540 * The typical sequence is to flush dirty PFS's starting at 541 * their root downward, then flush the device root (vchain). 542 * It is this second flush that typically leaves out the 543 * ALL flag. 544 * 545 * However we must still process the PFSROOT chains for block 546 * table updates in their parent (which IS part of our flush). 547 * 548 * NOTE: The volume root, vchain, does not set PFSBOUNDARY. 549 * 550 * NOTE: We must re-set ONFLUSH in the parent to retain if 551 * this chain (that we are skipping) requires work. 552 */ 553 if (chain->flags & (HAMMER2_CHAIN_ONFLUSH | 554 HAMMER2_CHAIN_DESTROY | 555 HAMMER2_CHAIN_MODIFIED)) { 556 hammer2_chain_setflush(parent); 557 } 558 goto done; 559 } else if (chain->bref.type == HAMMER2_BREF_TYPE_INODE && 560 (flags & HAMMER2_FLUSH_INODE_STOP) && 561 (flags & HAMMER2_FLUSH_ALL) == 0 && 562 (flags & HAMMER2_FLUSH_TOP) == 0 && 563 chain->pmp && chain->pmp->mp) { 564 /* 565 * When FLUSH_INODE_STOP is specified we are being asked not 566 * to include any inode changes for inodes we encounter, 567 * with the exception of the inode that the flush began with. 568 * So: INODE, INODE_STOP, and TOP==0 basically. 569 * 570 * Dirty inodes are flushed based on the hammer2_inode 571 * in-memory structure, issuing a chain_setflush() here 572 * will only cause unnecessary traversals of the topology. 573 */ 574 goto done; 575 #if 0 576 /* 577 * If FLUSH_INODE_STOP is specified and both ALL and TOP 578 * are clear, we must not flush the chain. The chain should 579 * have already been flushed and any further ONFLUSH/UPDATE 580 * setting will be related to the next flush. 581 * 582 * This features allows us to flush inodes independently of 583 * each other and meta-data above the inodes separately. 584 */ 585 if (chain->flags & (HAMMER2_CHAIN_ONFLUSH | 586 HAMMER2_CHAIN_DESTROY | 587 HAMMER2_CHAIN_MODIFIED)) { 588 if (parent) 589 hammer2_chain_setflush(parent); 590 } 591 #endif 592 } else if (info->depth == HAMMER2_FLUSH_DEPTH_LIMIT) { 593 /* 594 * Recursion depth reached. 595 */ 596 panic("hammer2: flush depth limit"); 597 } else if (chain->flags & (HAMMER2_CHAIN_ONFLUSH | 598 HAMMER2_CHAIN_DESTROY)) { 599 /* 600 * Downward recursion search (actual flush occurs bottom-up). 601 * pre-clear ONFLUSH. It can get set again due to races or 602 * flush errors, which we want so the scan finds us again in 603 * the next flush. 604 * 605 * We must also recurse if DESTROY is set so we can finally 606 * get rid of the related children, otherwise the node will 607 * just get re-flushed on lastdrop. 608 * 609 * WARNING! The recursion will unlock/relock info->parent 610 * (which is 'chain'), potentially allowing it 611 * to be ripped up. 612 */ 613 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH); 614 save_error = info->error; 615 info->error = 0; 616 info->parent = chain; 617 618 /* 619 * We may have to do this twice to catch any indirect 620 * block maintenance that occurs. 621 */ 622 hammer2_spin_ex(&chain->core.spin); 623 RB_SCAN(hammer2_chain_tree, &chain->core.rbtree, 624 NULL, hammer2_flush_recurse, info); 625 if (chain->flags & HAMMER2_CHAIN_ONFLUSH) { 626 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH); 627 RB_SCAN(hammer2_chain_tree, &chain->core.rbtree, 628 NULL, hammer2_flush_recurse, info); 629 } 630 hammer2_spin_unex(&chain->core.spin); 631 info->parent = parent; 632 633 /* 634 * Re-set the flush bits if the flush was incomplete or 635 * an error occurred. If an error occurs it is typically 636 * an allocation error. Errors do not cause deferrals. 637 */ 638 if (info->error) 639 hammer2_chain_setflush(chain); 640 info->error |= save_error; 641 642 /* 643 * If we lost the parent->chain association we have to 644 * stop processing this chain because it is no longer 645 * in this recursion. If it moved, it will be handled 646 * by the ONFLUSH flag elsewhere. 647 */ 648 if (chain->parent != parent) { 649 kprintf("LOST CHILD2 %p->%p (actual parent %p)\n", 650 parent, chain, chain->parent); 651 goto done; 652 } 653 } 654 655 /* 656 * Now we are in the bottom-up part of the recursion. 657 * 658 * We continue to try to update the chain on lower-level errors, but 659 * the flush code may decide not to flush the volume root. 660 * 661 * XXX should we continue to try to update the chain if an error 662 * occurred? 663 */ 664 665 /* 666 * Both parent and chain must be locked in order to flush chain, 667 * in order to properly update the parent under certain conditions. 668 * 669 * In addition, we can't safely unlock/relock the chain once we 670 * start flushing the chain itself, which we would have to do later 671 * on in order to lock the parent if we didn't do that now. 672 */ 673 hammer2_chain_ref_hold(chain); 674 hammer2_chain_unlock(chain); 675 if (parent) 676 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS); 677 hammer2_chain_lock(chain, HAMMER2_RESOLVE_MAYBE); 678 hammer2_chain_drop_unhold(chain); 679 680 /* 681 * Can't process if we can't access their content. 682 */ 683 if ((parent && parent->error) || chain->error) { 684 kprintf("hammer2: chain error during flush\n"); 685 info->error |= chain->error; 686 if (parent) { 687 info->error |= parent->error; 688 hammer2_chain_unlock(parent); 689 } 690 goto done; 691 } 692 693 if (chain->parent != parent) { 694 if (hammer2_debug & 0x0040) { 695 kprintf("LOST CHILD3 %p->%p (actual parent %p)\n", 696 parent, chain, chain->parent); 697 } 698 KKASSERT(parent != NULL); 699 hammer2_chain_unlock(parent); 700 retry = 1; 701 goto done; 702 } 703 704 /* 705 * Propagate the DESTROY flag downwards. This dummies up the flush 706 * code and tries to invalidate related buffer cache buffers to 707 * avoid the disk write. 708 */ 709 if (parent && (parent->flags & HAMMER2_CHAIN_DESTROY)) 710 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DESTROY); 711 712 /* 713 * Dispose of the modified bit. 714 * 715 * If parent is present, the UPDATE bit should already be set. 716 * UPDATE should already be set. 717 * bref.mirror_tid should already be set. 718 */ 719 if (chain->flags & HAMMER2_CHAIN_MODIFIED) { 720 KKASSERT((chain->flags & HAMMER2_CHAIN_UPDATE) || 721 chain->parent == NULL); 722 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_MODIFIED); 723 atomic_add_long(&hammer2_count_modified_chains, -1); 724 725 /* 726 * Manage threads waiting for excessive dirty memory to 727 * be retired. 728 */ 729 if (chain->pmp) 730 hammer2_pfs_memory_wakeup(chain->pmp); 731 732 #if 0 733 if ((chain->flags & HAMMER2_CHAIN_UPDATE) == 0 && 734 chain != &hmp->vchain && 735 chain != &hmp->fchain) { 736 /* 737 * Set UPDATE bit indicating that the parent block 738 * table requires updating. 739 */ 740 atomic_set_int(&chain->flags, HAMMER2_CHAIN_UPDATE); 741 } 742 #endif 743 744 /* 745 * Issue the flush. This is indirect via the DIO. 746 * 747 * NOTE: A DELETED node that reaches this point must be 748 * flushed for synchronization point consistency. 749 * 750 * NOTE: Even though MODIFIED was already set, the related DIO 751 * might not be dirty due to a system buffer cache 752 * flush and must be set dirty if we are going to make 753 * further modifications to the buffer. Chains with 754 * embedded data don't need this. 755 */ 756 if (hammer2_debug & 0x1000) { 757 kprintf("Flush %p.%d %016jx/%d data=%016jx\n", 758 chain, chain->bref.type, 759 (uintmax_t)chain->bref.key, 760 chain->bref.keybits, 761 (uintmax_t)chain->bref.data_off); 762 } 763 if (hammer2_debug & 0x2000) { 764 Debugger("Flush hell"); 765 } 766 767 /* 768 * Update chain CRCs for flush. 769 * 770 * NOTE: Volume headers are NOT flushed here as they require 771 * special processing. 772 */ 773 switch(chain->bref.type) { 774 case HAMMER2_BREF_TYPE_FREEMAP: 775 /* 776 * Update the volume header's freemap_tid to the 777 * freemap's flushing mirror_tid. 778 * 779 * (note: embedded data, do not call setdirty) 780 */ 781 KKASSERT(hmp->vchain.flags & HAMMER2_CHAIN_MODIFIED); 782 KKASSERT(chain == &hmp->fchain); 783 hmp->voldata.freemap_tid = chain->bref.mirror_tid; 784 if (hammer2_debug & 0x8000) { 785 /* debug only, avoid syslogd loop */ 786 kprintf("sync freemap mirror_tid %08jx\n", 787 (intmax_t)chain->bref.mirror_tid); 788 } 789 790 /* 791 * The freemap can be flushed independently of the 792 * main topology, but for the case where it is 793 * flushed in the same transaction, and flushed 794 * before vchain (a case we want to allow for 795 * performance reasons), make sure modifications 796 * made during the flush under vchain use a new 797 * transaction id. 798 * 799 * Otherwise the mount recovery code will get confused. 800 */ 801 ++hmp->voldata.mirror_tid; 802 break; 803 case HAMMER2_BREF_TYPE_VOLUME: 804 /* 805 * The free block table is flushed by 806 * hammer2_vfs_sync() before it flushes vchain. 807 * We must still hold fchain locked while copying 808 * voldata to volsync, however. 809 * 810 * These do not error per-say since their data does 811 * not need to be re-read from media on lock. 812 * 813 * (note: embedded data, do not call setdirty) 814 */ 815 hammer2_chain_lock(&hmp->fchain, 816 HAMMER2_RESOLVE_ALWAYS); 817 hammer2_voldata_lock(hmp); 818 if (hammer2_debug & 0x8000) { 819 /* debug only, avoid syslogd loop */ 820 kprintf("sync volume mirror_tid %08jx\n", 821 (intmax_t)chain->bref.mirror_tid); 822 } 823 824 /* 825 * Update the volume header's mirror_tid to the 826 * main topology's flushing mirror_tid. It is 827 * possible that voldata.mirror_tid is already 828 * beyond bref.mirror_tid due to the bump we made 829 * above in BREF_TYPE_FREEMAP. 830 */ 831 if (hmp->voldata.mirror_tid < chain->bref.mirror_tid) { 832 hmp->voldata.mirror_tid = 833 chain->bref.mirror_tid; 834 } 835 836 /* 837 * The volume header is flushed manually by the 838 * syncer, not here. All we do here is adjust the 839 * crc's. 840 */ 841 KKASSERT(chain->data != NULL); 842 KKASSERT(chain->dio == NULL); 843 844 hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT1]= 845 hammer2_icrc32( 846 (char *)&hmp->voldata + 847 HAMMER2_VOLUME_ICRC1_OFF, 848 HAMMER2_VOLUME_ICRC1_SIZE); 849 hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT0]= 850 hammer2_icrc32( 851 (char *)&hmp->voldata + 852 HAMMER2_VOLUME_ICRC0_OFF, 853 HAMMER2_VOLUME_ICRC0_SIZE); 854 hmp->voldata.icrc_volheader = 855 hammer2_icrc32( 856 (char *)&hmp->voldata + 857 HAMMER2_VOLUME_ICRCVH_OFF, 858 HAMMER2_VOLUME_ICRCVH_SIZE); 859 860 if (hammer2_debug & 0x8000) { 861 /* debug only, avoid syslogd loop */ 862 kprintf("syncvolhdr %016jx %016jx\n", 863 hmp->voldata.mirror_tid, 864 hmp->vchain.bref.mirror_tid); 865 } 866 hmp->volsync = hmp->voldata; 867 atomic_set_int(&chain->flags, HAMMER2_CHAIN_VOLUMESYNC); 868 hammer2_voldata_unlock(hmp); 869 hammer2_chain_unlock(&hmp->fchain); 870 break; 871 case HAMMER2_BREF_TYPE_DATA: 872 /* 873 * Data elements have already been flushed via the 874 * logical file buffer cache. Their hash was set in 875 * the bref by the vop_write code. Do not re-dirty. 876 * 877 * Make sure any device buffer(s) have been flushed 878 * out here (there aren't usually any to flush) XXX. 879 */ 880 break; 881 case HAMMER2_BREF_TYPE_INDIRECT: 882 case HAMMER2_BREF_TYPE_FREEMAP_NODE: 883 case HAMMER2_BREF_TYPE_FREEMAP_LEAF: 884 /* 885 * Buffer I/O will be cleaned up when the volume is 886 * flushed (but the kernel is free to flush it before 887 * then, as well). 888 */ 889 KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0); 890 hammer2_chain_setcheck(chain, chain->data); 891 break; 892 case HAMMER2_BREF_TYPE_DIRENT: 893 /* 894 * A directory entry can use the check area to store 895 * the filename for filenames <= 64 bytes, don't blow 896 * it up! 897 */ 898 KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0); 899 if (chain->bytes) 900 hammer2_chain_setcheck(chain, chain->data); 901 break; 902 case HAMMER2_BREF_TYPE_INODE: 903 /* 904 * NOTE: We must call io_setdirty() to make any late 905 * changes to the inode data, the system might 906 * have already flushed the buffer. 907 */ 908 if (chain->data->ipdata.meta.op_flags & 909 HAMMER2_OPFLAG_PFSROOT) { 910 /* 911 * non-NULL pmp if mounted as a PFS. We must 912 * sync fields cached in the pmp? XXX 913 */ 914 hammer2_inode_data_t *ipdata; 915 916 hammer2_io_setdirty(chain->dio); 917 ipdata = &chain->data->ipdata; 918 if (chain->pmp) { 919 ipdata->meta.pfs_inum = 920 chain->pmp->inode_tid; 921 } 922 } else { 923 /* can't be mounted as a PFS */ 924 } 925 926 KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0); 927 hammer2_chain_setcheck(chain, chain->data); 928 break; 929 default: 930 KKASSERT(chain->flags & HAMMER2_CHAIN_EMBEDDED); 931 panic("hammer2_flush_core: unsupported " 932 "embedded bref %d", 933 chain->bref.type); 934 /* NOT REACHED */ 935 } 936 937 /* 938 * If the chain was destroyed try to avoid unnecessary I/O 939 * that might not have yet occurred. Remove the data range 940 * from dedup candidacy and attempt to invalidation that 941 * potentially dirty portion of the I/O buffer. 942 */ 943 if (chain->flags & HAMMER2_CHAIN_DESTROY) { 944 hammer2_io_dedup_delete(hmp, 945 chain->bref.type, 946 chain->bref.data_off, 947 chain->bytes); 948 #if 0 949 hammer2_io_t *dio; 950 if (chain->dio) { 951 hammer2_io_inval(chain->dio, 952 chain->bref.data_off, 953 chain->bytes); 954 } else if ((dio = hammer2_io_getquick(hmp, 955 chain->bref.data_off, 956 chain->bytes, 957 1)) != NULL) { 958 hammer2_io_inval(dio, 959 chain->bref.data_off, 960 chain->bytes); 961 hammer2_io_putblk(&dio); 962 } 963 #endif 964 } 965 } 966 967 /* 968 * If UPDATE is set the parent block table may need to be updated. 969 * This can fail if the hammer2_chain_modify() fails. 970 * 971 * NOTE: UPDATE may be set on vchain or fchain in which case 972 * parent could be NULL, or on an inode that has not yet 973 * been inserted into the radix tree. It's easiest to allow 974 * the case and test for NULL. parent can also wind up being 975 * NULL due to a deletion so we need to handle the case anyway. 976 * 977 * NOTE: UPDATE can be set when chains are renamed into or out of 978 * an indirect block, without the chain itself being flagged 979 * MODIFIED. 980 * 981 * If no parent exists we can just clear the UPDATE bit. If the 982 * chain gets reattached later on the bit will simply get set 983 * again. 984 */ 985 if ((chain->flags & HAMMER2_CHAIN_UPDATE) && parent == NULL) 986 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE); 987 988 /* 989 * When flushing an inode outside of a FLUSH_FSSYNC we must NOT 990 * update the parent block table to point at the flushed inode. 991 * The block table should only ever be updated by the filesystem 992 * sync code. If we do, inode<->inode dependencies (such as 993 * directory entries vs inode nlink count) can wind up not being 994 * flushed together and result in a broken topology if a crash/reboot 995 * occurs at the wrong time. 996 */ 997 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE && 998 (flags & HAMMER2_FLUSH_INODE_STOP) && 999 (flags & HAMMER2_FLUSH_FSSYNC) == 0 && 1000 (flags & HAMMER2_FLUSH_ALL) == 0 && 1001 chain->pmp && chain->pmp->mp) { 1002 #ifdef HAMMER2_DEBUG_SYNC 1003 kprintf("inum %ld do not update parent, non-fssync\n", 1004 (long)chain->bref.key); 1005 #endif 1006 goto skipupdate; 1007 } 1008 #ifdef HAMMER2_DEBUG_SYNC 1009 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) 1010 kprintf("inum %ld update parent\n", (long)chain->bref.key); 1011 #endif 1012 1013 /* 1014 * The chain may need its blockrefs updated in the parent, normal 1015 * path. 1016 */ 1017 if (chain->flags & HAMMER2_CHAIN_UPDATE) { 1018 hammer2_blockref_t *base; 1019 int count; 1020 1021 /* 1022 * Clear UPDATE flag, mark parent modified, update its 1023 * modify_tid if necessary, and adjust the parent blockmap. 1024 */ 1025 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE); 1026 1027 /* 1028 * (optional code) 1029 * 1030 * Avoid actually modifying and updating the parent if it 1031 * was flagged for destruction. This can greatly reduce 1032 * disk I/O in large tree removals because the 1033 * hammer2_io_setinval() call in the upward recursion 1034 * (see MODIFIED code above) can only handle a few cases. 1035 */ 1036 if (parent->flags & HAMMER2_CHAIN_DESTROY) { 1037 if (parent->bref.modify_tid < chain->bref.modify_tid) { 1038 parent->bref.modify_tid = 1039 chain->bref.modify_tid; 1040 } 1041 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_BMAPPED | 1042 HAMMER2_CHAIN_BMAPUPD); 1043 goto skipupdate; 1044 } 1045 1046 /* 1047 * The flusher is responsible for deleting empty indirect 1048 * blocks at this point. If we don't do this, no major harm 1049 * will be done but the empty indirect blocks will stay in 1050 * the topology and make it a messy and inefficient. 1051 * 1052 * The flusher is also responsible for collapsing the 1053 * content of an indirect block into its parent whenever 1054 * possible (with some hysteresis). Not doing this will also 1055 * not harm the topology, but would make it messy and 1056 * inefficient. 1057 */ 1058 if (chain->bref.type == HAMMER2_BREF_TYPE_INDIRECT) { 1059 if (hammer2_chain_indirect_maintenance(parent, chain)) 1060 goto skipupdate; 1061 } 1062 1063 /* 1064 * We are updating the parent's blockmap, the parent must 1065 * be set modified. If this fails we re-set the UPDATE flag 1066 * in the child. 1067 * 1068 * NOTE! A modification error can be ENOSPC. We still want 1069 * to flush modified chains recursively, not break out, 1070 * so we just skip the update in this situation and 1071 * continue. That is, we still need to try to clean 1072 * out dirty chains and buffers. 1073 * 1074 * This may not help bulkfree though. XXX 1075 */ 1076 save_error = hammer2_chain_modify(parent, 0, 0, 0); 1077 if (save_error) { 1078 info->error |= save_error; 1079 kprintf("hammer2_flush: %016jx.%02x error=%08x\n", 1080 parent->bref.data_off, parent->bref.type, 1081 save_error); 1082 atomic_set_int(&chain->flags, HAMMER2_CHAIN_UPDATE); 1083 goto skipupdate; 1084 } 1085 if (parent->bref.modify_tid < chain->bref.modify_tid) 1086 parent->bref.modify_tid = chain->bref.modify_tid; 1087 1088 /* 1089 * Calculate blockmap pointer 1090 */ 1091 switch(parent->bref.type) { 1092 case HAMMER2_BREF_TYPE_INODE: 1093 /* 1094 * Access the inode's block array. However, there is 1095 * no block array if the inode is flagged DIRECTDATA. 1096 */ 1097 if (parent->data && 1098 (parent->data->ipdata.meta.op_flags & 1099 HAMMER2_OPFLAG_DIRECTDATA) == 0) { 1100 base = &parent->data-> 1101 ipdata.u.blockset.blockref[0]; 1102 } else { 1103 base = NULL; 1104 } 1105 count = HAMMER2_SET_COUNT; 1106 break; 1107 case HAMMER2_BREF_TYPE_INDIRECT: 1108 case HAMMER2_BREF_TYPE_FREEMAP_NODE: 1109 if (parent->data) 1110 base = &parent->data->npdata[0]; 1111 else 1112 base = NULL; 1113 count = parent->bytes / sizeof(hammer2_blockref_t); 1114 break; 1115 case HAMMER2_BREF_TYPE_VOLUME: 1116 base = &chain->hmp->voldata.sroot_blockset.blockref[0]; 1117 count = HAMMER2_SET_COUNT; 1118 break; 1119 case HAMMER2_BREF_TYPE_FREEMAP: 1120 base = &parent->data->npdata[0]; 1121 count = HAMMER2_SET_COUNT; 1122 break; 1123 default: 1124 base = NULL; 1125 count = 0; 1126 panic("hammer2_flush_core: " 1127 "unrecognized blockref type: %d", 1128 parent->bref.type); 1129 } 1130 1131 /* 1132 * Blocktable updates 1133 * 1134 * We synchronize pending statistics at this time. Delta 1135 * adjustments designated for the current and upper level 1136 * are synchronized. 1137 */ 1138 if (base && (chain->flags & HAMMER2_CHAIN_BMAPUPD)) { 1139 if (chain->flags & HAMMER2_CHAIN_BMAPPED) { 1140 hammer2_spin_ex(&parent->core.spin); 1141 hammer2_base_delete(parent, base, count, chain, 1142 NULL); 1143 hammer2_spin_unex(&parent->core.spin); 1144 /* base_delete clears both bits */ 1145 } else { 1146 atomic_clear_int(&chain->flags, 1147 HAMMER2_CHAIN_BMAPUPD); 1148 } 1149 } 1150 if (base && (chain->flags & HAMMER2_CHAIN_BMAPPED) == 0) { 1151 hammer2_spin_ex(&parent->core.spin); 1152 hammer2_base_insert(parent, base, count, 1153 chain, &chain->bref); 1154 hammer2_spin_unex(&parent->core.spin); 1155 /* base_insert sets BMAPPED */ 1156 } 1157 } 1158 skipupdate: 1159 if (parent) 1160 hammer2_chain_unlock(parent); 1161 1162 /* 1163 * Final cleanup after flush 1164 */ 1165 done: 1166 KKASSERT(chain->refs > 0); 1167 if (hammer2_debug & 0x200) { 1168 if (info->debug == chain) 1169 info->debug = NULL; 1170 } 1171 return retry; 1172 } 1173 1174 /* 1175 * Flush recursion helper, called from flush_core, calls flush_core. 1176 * 1177 * Flushes the children of the caller's chain (info->parent), restricted 1178 * by sync_tid. 1179 * 1180 * This function may set info->error as a side effect. 1181 * 1182 * WARNING! If we do not call hammer2_flush_core() we must update 1183 * bref.mirror_tid ourselves to indicate that the flush has 1184 * processed the child. 1185 * 1186 * WARNING! parent->core spinlock is held on entry and return. 1187 */ 1188 static int 1189 hammer2_flush_recurse(hammer2_chain_t *child, void *data) 1190 { 1191 hammer2_flush_info_t *info = data; 1192 hammer2_chain_t *parent = info->parent; 1193 1194 #ifdef HAMMER2_SCAN_DEBUG 1195 ++info->scan_count; 1196 if (child->flags & HAMMER2_CHAIN_MODIFIED) 1197 ++info->scan_mod_count; 1198 if (child->flags & HAMMER2_CHAIN_UPDATE) 1199 ++info->scan_upd_count; 1200 if (child->flags & HAMMER2_CHAIN_ONFLUSH) 1201 ++info->scan_onf_count; 1202 #endif 1203 1204 /* 1205 * (child can never be fchain or vchain so a special check isn't 1206 * needed). 1207 * 1208 * We must ref the child before unlocking the spinlock. 1209 * 1210 * The caller has added a ref to the parent so we can temporarily 1211 * unlock it in order to lock the child. However, if it no longer 1212 * winds up being the child of the parent we must skip this child. 1213 * 1214 * NOTE! chain locking errors are fatal. They are never out-of-space 1215 * errors. 1216 */ 1217 hammer2_chain_ref(child); 1218 hammer2_spin_unex(&parent->core.spin); 1219 1220 hammer2_chain_ref_hold(parent); 1221 hammer2_chain_unlock(parent); 1222 hammer2_chain_lock(child, HAMMER2_RESOLVE_MAYBE); 1223 if (child->parent != parent) { 1224 kprintf("LOST CHILD1 %p->%p (actual parent %p)\n", 1225 parent, child, child->parent); 1226 goto done; 1227 } 1228 if (child->error) { 1229 kprintf("CHILD ERROR DURING FLUSH LOCK %p->%p\n", 1230 parent, child); 1231 info->error |= child->error; 1232 goto done; 1233 } 1234 1235 /* 1236 * Must propagate the DESTROY flag downwards, otherwise the 1237 * parent could end up never being removed because it will 1238 * be requeued to the flusher if it survives this run due to 1239 * the flag. 1240 */ 1241 if (parent && (parent->flags & HAMMER2_CHAIN_DESTROY)) 1242 atomic_set_int(&child->flags, HAMMER2_CHAIN_DESTROY); 1243 #ifdef HAMMER2_SCAN_DEBUG 1244 if (child->flags & HAMMER2_CHAIN_DESTROY) 1245 ++info->scan_del_count; 1246 #endif 1247 /* 1248 * Special handling of the root inode. Because the root inode 1249 * contains an index of all the inodes in the PFS in addition to 1250 * its normal directory entries, any flush that is not part of a 1251 * filesystem sync must only flush the directory entries, and not 1252 * anything else. 1253 * 1254 * The child might be an indirect block, but H2 guarantees that 1255 * the key-range will fully partition the inode index from the 1256 * directory entries so the case just works naturally. 1257 */ 1258 if ((parent->bref.flags & HAMMER2_BREF_FLAG_PFSROOT) && 1259 (child->flags & HAMMER2_CHAIN_DESTROY) == 0 && 1260 parent->bref.type == HAMMER2_BREF_TYPE_INODE && 1261 (info->flags & HAMMER2_FLUSH_FSSYNC) == 0) { 1262 if ((child->bref.key & HAMMER2_DIRHASH_VISIBLE) == 0) { 1263 if (child->flags & HAMMER2_CHAIN_FLUSH_MASK) { 1264 hammer2_chain_setflush(parent); 1265 } 1266 goto done; 1267 } 1268 } 1269 1270 /* 1271 * Recurse and collect deferral data. We're in the media flush, 1272 * this can cross PFS boundaries. 1273 */ 1274 if (child->flags & HAMMER2_CHAIN_FLUSH_MASK) { 1275 #ifdef HAMMER2_SCAN_DEBUG 1276 if (child->bref.type < 7) 1277 ++info->scan_btype[child->bref.type]; 1278 #endif 1279 ++info->depth; 1280 hammer2_flush_core(info, child, info->flags); 1281 --info->depth; 1282 } else if (hammer2_debug & 0x200) { 1283 if (info->debug == NULL) 1284 info->debug = child; 1285 ++info->depth; 1286 hammer2_flush_core(info, child, info->flags); 1287 --info->depth; 1288 if (info->debug == child) 1289 info->debug = NULL; 1290 } 1291 1292 done: 1293 /* 1294 * Relock to continue the loop. 1295 */ 1296 hammer2_chain_unlock(child); 1297 hammer2_chain_lock(parent, HAMMER2_RESOLVE_MAYBE); 1298 hammer2_chain_drop_unhold(parent); 1299 if (parent->error) { 1300 kprintf("PARENT ERROR DURING FLUSH LOCK %p->%p\n", 1301 parent, child); 1302 info->error |= parent->error; 1303 } 1304 hammer2_chain_drop(child); 1305 KKASSERT(info->parent == parent); 1306 hammer2_spin_ex(&parent->core.spin); 1307 1308 return (0); 1309 } 1310 1311 /* 1312 * flush helper (backend threaded) 1313 * 1314 * Flushes chain topology for the specified inode. 1315 * 1316 * HAMMER2_XOP_INODE_STOP The flush recursion stops at inode boundaries. 1317 * Inodes belonging to the same flush are flushed 1318 * separately. 1319 * 1320 * chain->parent can be NULL, usually due to destroy races or detached inodes. 1321 * 1322 * Primarily called from vfs_sync(). 1323 */ 1324 void 1325 hammer2_xop_inode_flush(hammer2_xop_t *arg, void *scratch __unused, int clindex) 1326 { 1327 hammer2_xop_flush_t *xop = &arg->xop_flush; 1328 hammer2_chain_t *chain; 1329 hammer2_inode_t *ip; 1330 hammer2_dev_t *hmp; 1331 hammer2_pfs_t *pmp; 1332 int flush_error = 0; 1333 int fsync_error = 0; 1334 int total_error = 0; 1335 int j; 1336 int xflags; 1337 int ispfsroot = 0; 1338 1339 xflags = HAMMER2_FLUSH_TOP; 1340 if (xop->head.flags & HAMMER2_XOP_INODE_STOP) 1341 xflags |= HAMMER2_FLUSH_INODE_STOP; 1342 if (xop->head.flags & HAMMER2_XOP_FSSYNC) 1343 xflags |= HAMMER2_FLUSH_FSSYNC; 1344 1345 /* 1346 * Flush core chains 1347 */ 1348 ip = xop->head.ip1; 1349 pmp = ip->pmp; 1350 chain = hammer2_inode_chain(ip, clindex, HAMMER2_RESOLVE_ALWAYS); 1351 if (chain) { 1352 hmp = chain->hmp; 1353 if (chain->flags & HAMMER2_CHAIN_FLUSH_MASK) { 1354 /* 1355 * Due to flush partitioning the chain topology 1356 * above the inode's chain may no longer be flagged. 1357 * When asked to flush an inode, remark the topology 1358 * leading to that inode. 1359 */ 1360 if (chain->parent) 1361 hammer2_chain_setflush(chain->parent); 1362 hammer2_flush(chain, xflags); 1363 1364 /* XXX cluster */ 1365 if (ip == pmp->iroot && pmp != hmp->spmp) { 1366 hammer2_spin_ex(&pmp->inum_spin); 1367 pmp->pfs_iroot_blocksets[clindex] = 1368 chain->data->ipdata.u.blockset; 1369 hammer2_spin_unex(&pmp->inum_spin); 1370 } 1371 1372 #if 0 1373 /* 1374 * Propogate upwards but only cross an inode boundary 1375 * for inodes associated with the current filesystem 1376 * sync. 1377 */ 1378 if ((xop->head.flags & HAMMER2_XOP_PARENTONFLUSH) || 1379 chain->bref.type != HAMMER2_BREF_TYPE_INODE) { 1380 parent = chain->parent; 1381 if (parent) 1382 hammer2_chain_setflush(parent); 1383 } 1384 #endif 1385 } 1386 if (chain->flags & HAMMER2_CHAIN_PFSBOUNDARY) 1387 ispfsroot = 1; 1388 hammer2_chain_unlock(chain); 1389 hammer2_chain_drop(chain); 1390 chain = NULL; 1391 } else { 1392 hmp = NULL; 1393 } 1394 1395 /* 1396 * Only flush the volume header if asked to, plus the inode must also 1397 * be the PFS root. 1398 */ 1399 if ((xop->head.flags & HAMMER2_XOP_VOLHDR) == 0) 1400 goto skip; 1401 if (ispfsroot == 0) 1402 goto skip; 1403 1404 /* 1405 * Flush volume roots. Avoid replication, we only want to 1406 * flush each hammer2_dev (hmp) once. 1407 */ 1408 for (j = clindex - 1; j >= 0; --j) { 1409 if ((chain = ip->cluster.array[j].chain) != NULL) { 1410 if (chain->hmp == hmp) { 1411 chain = NULL; /* safety */ 1412 goto skip; 1413 } 1414 } 1415 } 1416 chain = NULL; /* safety */ 1417 1418 /* 1419 * spmp transaction. The super-root is never directly mounted so 1420 * there shouldn't be any vnodes, let alone any dirty vnodes 1421 * associated with it, so we shouldn't have to mess around with any 1422 * vnode flushes here. 1423 */ 1424 hammer2_trans_init(hmp->spmp, HAMMER2_TRANS_ISFLUSH); 1425 1426 /* 1427 * We must flush the superroot down to the PFS iroot. Remember 1428 * that hammer2_chain_setflush() stops at inode boundaries, so 1429 * the pmp->iroot has been flushed and flagged down to the superroot, 1430 * but the volume root (vchain) probably has not yet been flagged. 1431 */ 1432 if (hmp->spmp->iroot) { 1433 chain = hmp->spmp->iroot->cluster.array[0].chain; 1434 if (chain) { 1435 hammer2_chain_ref(chain); 1436 hammer2_chain_lock(chain, HAMMER2_RESOLVE_ALWAYS); 1437 flush_error |= 1438 hammer2_flush(chain, 1439 HAMMER2_FLUSH_TOP | 1440 HAMMER2_FLUSH_INODE_STOP | 1441 HAMMER2_FLUSH_FSSYNC); 1442 hammer2_chain_unlock(chain); 1443 hammer2_chain_drop(chain); 1444 } 1445 } 1446 1447 /* 1448 * Media mounts have two 'roots', vchain for the topology 1449 * and fchain for the free block table. Flush both. 1450 * 1451 * Note that the topology and free block table are handled 1452 * independently, so the free block table can wind up being 1453 * ahead of the topology. We depend on the bulk free scan 1454 * code to deal with any loose ends. 1455 * 1456 * vchain and fchain do not error on-lock since their data does 1457 * not have to be re-read from media. 1458 */ 1459 hammer2_chain_ref(&hmp->vchain); 1460 hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS); 1461 hammer2_chain_ref(&hmp->fchain); 1462 hammer2_chain_lock(&hmp->fchain, HAMMER2_RESOLVE_ALWAYS); 1463 if (hmp->fchain.flags & HAMMER2_CHAIN_FLUSH_MASK) { 1464 /* 1465 * This will also modify vchain as a side effect, 1466 * mark vchain as modified now. 1467 */ 1468 hammer2_voldata_modify(hmp); 1469 chain = &hmp->fchain; 1470 flush_error |= hammer2_flush(chain, HAMMER2_FLUSH_TOP); 1471 KKASSERT(chain == &hmp->fchain); 1472 } 1473 hammer2_chain_unlock(&hmp->fchain); 1474 hammer2_chain_unlock(&hmp->vchain); 1475 hammer2_chain_drop(&hmp->fchain); 1476 /* vchain dropped down below */ 1477 1478 hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS); 1479 if (hmp->vchain.flags & HAMMER2_CHAIN_FLUSH_MASK) { 1480 chain = &hmp->vchain; 1481 flush_error |= hammer2_flush(chain, HAMMER2_FLUSH_TOP); 1482 KKASSERT(chain == &hmp->vchain); 1483 } 1484 hammer2_chain_unlock(&hmp->vchain); 1485 hammer2_chain_drop(&hmp->vchain); 1486 1487 /* 1488 * We can't safely flush the volume header until we have 1489 * flushed any device buffers which have built up. 1490 * 1491 * XXX this isn't being incremental 1492 */ 1493 vn_lock(hmp->devvp, LK_EXCLUSIVE | LK_RETRY); 1494 fsync_error = VOP_FSYNC(hmp->devvp, MNT_WAIT, 0); 1495 vn_unlock(hmp->devvp); 1496 if (fsync_error || flush_error) { 1497 kprintf("hammer2: sync error fsync=%d h2flush=0x%04x dev=%s\n", 1498 fsync_error, flush_error, hmp->devrepname); 1499 } 1500 1501 /* 1502 * The flush code sets CHAIN_VOLUMESYNC to indicate that the 1503 * volume header needs synchronization via hmp->volsync. 1504 * 1505 * XXX synchronize the flag & data with only this flush XXX 1506 */ 1507 if (fsync_error == 0 && flush_error == 0 && 1508 (hmp->vchain.flags & HAMMER2_CHAIN_VOLUMESYNC)) { 1509 struct buf *bp; 1510 int vol_error = 0; 1511 1512 /* 1513 * Synchronize the disk before flushing the volume 1514 * header. 1515 */ 1516 bp = getpbuf(NULL); 1517 bp->b_bio1.bio_offset = 0; 1518 bp->b_bufsize = 0; 1519 bp->b_bcount = 0; 1520 bp->b_cmd = BUF_CMD_FLUSH; 1521 bp->b_bio1.bio_done = biodone_sync; 1522 bp->b_bio1.bio_flags |= BIO_SYNC; 1523 vn_strategy(hmp->devvp, &bp->b_bio1); 1524 fsync_error = biowait(&bp->b_bio1, "h2vol"); 1525 relpbuf(bp, NULL); 1526 1527 /* 1528 * Then we can safely flush the version of the 1529 * volume header synchronized by the flush code. 1530 */ 1531 j = hmp->volhdrno + 1; 1532 if (j < 0) 1533 j = 0; 1534 if (j >= HAMMER2_NUM_VOLHDRS) 1535 j = 0; 1536 if (j * HAMMER2_ZONE_BYTES64 + HAMMER2_SEGSIZE > 1537 hmp->volsync.volu_size) { 1538 j = 0; 1539 } 1540 if (hammer2_debug & 0x8000) { 1541 /* debug only, avoid syslogd loop */ 1542 kprintf("sync volhdr %d %jd\n", 1543 j, (intmax_t)hmp->volsync.volu_size); 1544 } 1545 bp = getblk(hmp->devvp, j * HAMMER2_ZONE_BYTES64, 1546 HAMMER2_PBUFSIZE, GETBLK_KVABIO, 0); 1547 atomic_clear_int(&hmp->vchain.flags, 1548 HAMMER2_CHAIN_VOLUMESYNC); 1549 bkvasync(bp); 1550 bcopy(&hmp->volsync, bp->b_data, HAMMER2_PBUFSIZE); 1551 vol_error = bwrite(bp); 1552 hmp->volhdrno = j; 1553 if (vol_error) 1554 fsync_error = vol_error; 1555 } 1556 if (flush_error) 1557 total_error = flush_error; 1558 if (fsync_error) 1559 total_error = hammer2_errno_to_error(fsync_error); 1560 1561 /* spmp trans */ 1562 hammer2_trans_done(hmp->spmp, HAMMER2_TRANS_ISFLUSH); 1563 skip: 1564 hammer2_xop_feed(&xop->head, NULL, clindex, total_error); 1565 } 1566