1 /* 2 * Copyright (c) 2011-2018 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@dragonflybsd.org> 6 * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in 16 * the documentation and/or other materials provided with the 17 * distribution. 18 * 3. Neither the name of The DragonFly Project nor the names of its 19 * contributors may be used to endorse or promote products derived 20 * from this software without specific, prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 25 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 26 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 28 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 29 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 30 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 31 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 32 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 */ 35 #include <sys/cdefs.h> 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/types.h> 39 #include <sys/lock.h> 40 #include <sys/uuid.h> 41 #include <sys/vnode.h> 42 43 #include "hammer2.h" 44 45 #define INODE_DEBUG 0 46 47 RB_GENERATE2(hammer2_inode_tree, hammer2_inode, rbnode, hammer2_inode_cmp, 48 hammer2_tid_t, meta.inum); 49 50 int 51 hammer2_inode_cmp(hammer2_inode_t *ip1, hammer2_inode_t *ip2) 52 { 53 if (ip1->meta.inum < ip2->meta.inum) 54 return(-1); 55 if (ip1->meta.inum > ip2->meta.inum) 56 return(1); 57 return(0); 58 } 59 60 /* 61 * Caller holds pmp->list_spin and the inode should be locked. Merge ip 62 * with the specified depend. 63 * 64 * If the ip is on SYNCQ it stays there and (void *)-1 is returned, indicating 65 * that successive calls must ensure the ip is on a pass2 depend (or they are 66 * all SYNCQ). If the passed-in depend is not NULL and not (void *)-1 then 67 * we can set pass2 on it and return. 68 * 69 * If the ip is not on SYNCQ it is merged with the passed-in depend, creating 70 * a self-depend if necessary, and depend->pass2 is set according 71 * to the PASS2 flag. SIDEQ is set. 72 */ 73 static __noinline 74 hammer2_depend_t * 75 hammer2_inode_setdepend_locked(hammer2_inode_t *ip, hammer2_depend_t *depend) 76 { 77 hammer2_pfs_t *pmp = ip->pmp; 78 hammer2_depend_t *dtmp; 79 hammer2_inode_t *iptmp; 80 81 /* 82 * If ip is SYNCQ its entry is used for the syncq list and it will 83 * no longer be associated with a dependency. Merging this status 84 * with a passed-in depend implies PASS2. 85 */ 86 if (ip->flags & HAMMER2_INODE_SYNCQ) { 87 if (depend == (void *)-1 || 88 depend == NULL) { 89 return ((void *)-1); 90 } 91 depend->pass2 = 1; 92 hammer2_trans_setflags(pmp, HAMMER2_TRANS_RESCAN); 93 94 return depend; 95 } 96 97 /* 98 * If ip is already SIDEQ, merge ip->depend into the passed-in depend. 99 * If it is not, associate the ip with the passed-in depend, creating 100 * a single-entry dependency using depend_static if necessary. 101 * 102 * NOTE: The use of ip->depend_static always requires that the 103 * specific ip containing the structure is part of that 104 * particular depend_static's dependency group. 105 */ 106 if (ip->flags & HAMMER2_INODE_SIDEQ) { 107 /* 108 * Merge ip->depend with the passed-in depend. If the 109 * passed-in depend is not a special case, all ips associated 110 * with ip->depend (including the original ip) must be moved 111 * to the passed-in depend. 112 */ 113 if (depend == NULL) { 114 depend = ip->depend; 115 } else if (depend == (void *)-1) { 116 depend = ip->depend; 117 depend->pass2 = 1; 118 } else if (depend != ip->depend) { 119 #ifdef INVARIANTS 120 int sanitychk = 0; 121 #endif 122 dtmp = ip->depend; 123 while ((iptmp = TAILQ_FIRST(&dtmp->sideq)) != NULL) { 124 #ifdef INVARIANTS 125 if (iptmp == ip) 126 sanitychk = 1; 127 #endif 128 TAILQ_REMOVE(&dtmp->sideq, iptmp, entry); 129 TAILQ_INSERT_TAIL(&depend->sideq, iptmp, entry); 130 iptmp->depend = depend; 131 } 132 KKASSERT(sanitychk == 1); 133 depend->count += dtmp->count; 134 depend->pass2 |= dtmp->pass2; 135 TAILQ_REMOVE(&pmp->depq, dtmp, entry); 136 dtmp->count = 0; 137 dtmp->pass2 = 0; 138 } 139 } else { 140 /* 141 * Add ip to the sideq, creating a self-dependency if 142 * necessary. 143 */ 144 hammer2_inode_ref(ip); 145 atomic_set_int(&ip->flags, HAMMER2_INODE_SIDEQ); 146 if (depend == NULL) { 147 depend = &ip->depend_static; 148 TAILQ_INSERT_TAIL(&pmp->depq, depend, entry); 149 } else if (depend == (void *)-1) { 150 depend = &ip->depend_static; 151 depend->pass2 = 1; 152 TAILQ_INSERT_TAIL(&pmp->depq, depend, entry); 153 } /* else add ip to passed-in depend */ 154 TAILQ_INSERT_TAIL(&depend->sideq, ip, entry); 155 ip->depend = depend; 156 ++depend->count; 157 ++pmp->sideq_count; 158 } 159 160 if (ip->flags & HAMMER2_INODE_SYNCQ_PASS2) 161 depend->pass2 = 1; 162 if (depend->pass2) 163 hammer2_trans_setflags(pmp, HAMMER2_TRANS_RESCAN); 164 165 return depend; 166 } 167 168 /* 169 * Put a solo inode on the SIDEQ (meaning that its dirty). This can also 170 * occur from inode_lock4() and inode_depend(). 171 * 172 * Caller must pass-in a locked inode. 173 */ 174 void 175 hammer2_inode_delayed_sideq(hammer2_inode_t *ip) 176 { 177 hammer2_pfs_t *pmp = ip->pmp; 178 179 /* 180 * Optimize case to avoid pmp spinlock. 181 */ 182 if ((ip->flags & (HAMMER2_INODE_SYNCQ | HAMMER2_INODE_SIDEQ)) == 0) { 183 hammer2_spin_ex(&pmp->list_spin); 184 hammer2_inode_setdepend_locked(ip, NULL); 185 hammer2_spin_unex(&pmp->list_spin); 186 } 187 } 188 189 /* 190 * Lock an inode, with SYNCQ semantics. 191 * 192 * HAMMER2 offers shared and exclusive locks on inodes. Pass a mask of 193 * flags for options: 194 * 195 * - pass HAMMER2_RESOLVE_SHARED if a shared lock is desired. 196 * shared locks are not subject to SYNCQ semantics, exclusive locks 197 * are. 198 * 199 * - pass HAMMER2_RESOLVE_ALWAYS if you need the inode's meta-data. 200 * Most front-end inode locks do. 201 * 202 * - pass HAMMER2_RESOLVE_NEVER if you do not want to require that 203 * the inode data be resolved. This is used by the syncthr because 204 * it can run on an unresolved/out-of-sync cluster, and also by the 205 * vnode reclamation code to avoid unnecessary I/O (particularly when 206 * disposing of hundreds of thousands of cached vnodes). 207 * 208 * This function, along with lock4, has SYNCQ semantics. If the inode being 209 * locked is on the SYNCQ, that is it has been staged by the syncer, we must 210 * block until the operation is complete (even if we can lock the inode). In 211 * order to reduce the stall time, we re-order the inode to the front of the 212 * pmp->syncq prior to blocking. This reordering VERY significantly improves 213 * performance. 214 * 215 * The inode locking function locks the inode itself, resolves any stale 216 * chains in the inode's cluster, and allocates a fresh copy of the 217 * cluster with 1 ref and all the underlying chains locked. 218 * 219 * ip->cluster will be stable while the inode is locked. 220 * 221 * NOTE: We don't combine the inode/chain lock because putting away an 222 * inode would otherwise confuse multiple lock holders of the inode. 223 */ 224 void 225 hammer2_inode_lock(hammer2_inode_t *ip, int how) 226 { 227 hammer2_pfs_t *pmp; 228 229 hammer2_inode_ref(ip); 230 pmp = ip->pmp; 231 232 /* 233 * Inode structure mutex - Shared lock 234 */ 235 if (how & HAMMER2_RESOLVE_SHARED) { 236 hammer2_mtx_sh(&ip->lock); 237 return; 238 } 239 240 /* 241 * Inode structure mutex - Exclusive lock 242 * 243 * An exclusive lock (if not recursive) must wait for inodes on 244 * SYNCQ to flush first, to ensure that meta-data dependencies such 245 * as the nlink count and related directory entries are not split 246 * across flushes. 247 * 248 * If the vnode is locked by the current thread it must be unlocked 249 * across the tsleep() to avoid a deadlock. 250 */ 251 hammer2_mtx_ex(&ip->lock); 252 if (hammer2_mtx_refs(&ip->lock) > 1) 253 return; 254 while ((ip->flags & HAMMER2_INODE_SYNCQ) && pmp) { 255 hammer2_spin_ex(&pmp->list_spin); 256 if (ip->flags & HAMMER2_INODE_SYNCQ) { 257 tsleep_interlock(&ip->flags, 0); 258 atomic_set_int(&ip->flags, HAMMER2_INODE_SYNCQ_WAKEUP); 259 TAILQ_REMOVE(&pmp->syncq, ip, entry); 260 TAILQ_INSERT_HEAD(&pmp->syncq, ip, entry); 261 hammer2_spin_unex(&pmp->list_spin); 262 hammer2_mtx_unlock(&ip->lock); 263 tsleep(&ip->flags, PINTERLOCKED, "h2sync", 0); 264 hammer2_mtx_ex(&ip->lock); 265 continue; 266 } 267 hammer2_spin_unex(&pmp->list_spin); 268 break; 269 } 270 } 271 272 /* 273 * Exclusively lock up to four inodes, in order, with SYNCQ semantics. 274 * ip1 and ip2 must not be NULL. ip3 and ip4 may be NULL, but if ip3 is 275 * NULL then ip4 must also be NULL. 276 * 277 * This creates a dependency between up to four inodes. 278 */ 279 void 280 hammer2_inode_lock4(hammer2_inode_t *ip1, hammer2_inode_t *ip2, 281 hammer2_inode_t *ip3, hammer2_inode_t *ip4) 282 { 283 hammer2_inode_t *ips[4]; 284 hammer2_inode_t *iptmp; 285 hammer2_inode_t *ipslp; 286 hammer2_depend_t *depend; 287 hammer2_pfs_t *pmp; 288 size_t count; 289 size_t i; 290 291 pmp = ip1->pmp; /* may be NULL */ 292 KKASSERT(pmp == ip2->pmp); 293 294 ips[0] = ip1; 295 ips[1] = ip2; 296 if (ip3 == NULL) { 297 count = 2; 298 } else if (ip4 == NULL) { 299 count = 3; 300 ips[2] = ip3; 301 KKASSERT(pmp == ip3->pmp); 302 } else { 303 count = 4; 304 ips[2] = ip3; 305 ips[3] = ip4; 306 KKASSERT(pmp == ip3->pmp); 307 KKASSERT(pmp == ip4->pmp); 308 } 309 310 for (i = 0; i < count; ++i) 311 hammer2_inode_ref(ips[i]); 312 313 restart: 314 /* 315 * Lock the inodes in order 316 */ 317 for (i = 0; i < count; ++i) { 318 hammer2_mtx_ex(&ips[i]->lock); 319 } 320 321 /* 322 * Associate dependencies, record the first inode found on SYNCQ 323 * (operation is allowed to proceed for inodes on PASS2) for our 324 * sleep operation, this inode is theoretically the last one sync'd 325 * in the sequence. 326 * 327 * All inodes found on SYNCQ are moved to the head of the syncq 328 * to reduce stalls. 329 */ 330 hammer2_spin_ex(&pmp->list_spin); 331 depend = NULL; 332 ipslp = NULL; 333 for (i = 0; i < count; ++i) { 334 iptmp = ips[i]; 335 depend = hammer2_inode_setdepend_locked(iptmp, depend); 336 if (iptmp->flags & HAMMER2_INODE_SYNCQ) { 337 TAILQ_REMOVE(&pmp->syncq, iptmp, entry); 338 TAILQ_INSERT_HEAD(&pmp->syncq, iptmp, entry); 339 if (ipslp == NULL) 340 ipslp = iptmp; 341 } 342 } 343 hammer2_spin_unex(&pmp->list_spin); 344 345 /* 346 * Block and retry if any of the inodes are on SYNCQ. It is 347 * important that we allow the operation to proceed in the 348 * PASS2 case, to avoid deadlocking against the vnode. 349 */ 350 if (ipslp) { 351 for (i = 0; i < count; ++i) 352 hammer2_mtx_unlock(&ips[i]->lock); 353 tsleep(&ipslp->flags, 0, "h2sync", 2); 354 goto restart; 355 } 356 } 357 358 /* 359 * Release an inode lock. If another thread is blocked on SYNCQ_WAKEUP 360 * we wake them up. 361 */ 362 void 363 hammer2_inode_unlock(hammer2_inode_t *ip) 364 { 365 if (ip->flags & HAMMER2_INODE_SYNCQ_WAKEUP) { 366 atomic_clear_int(&ip->flags, HAMMER2_INODE_SYNCQ_WAKEUP); 367 hammer2_mtx_unlock(&ip->lock); 368 wakeup(&ip->flags); 369 } else { 370 hammer2_mtx_unlock(&ip->lock); 371 } 372 hammer2_inode_drop(ip); 373 } 374 375 /* 376 * If either ip1 or ip2 have been tapped by the syncer, make sure that both 377 * are. This ensure that dependencies (e.g. dirent-v-inode) are synced 378 * together. For dirent-v-inode depends, pass the dirent as ip1. 379 * 380 * If neither ip1 or ip2 have been tapped by the syncer, merge them into a 381 * single dependency. Dependencies are entered into pmp->depq. This 382 * effectively flags the inodes SIDEQ. 383 * 384 * Both ip1 and ip2 must be locked by the caller. This also ensures 385 * that we can't race the end of the syncer's queue run. 386 */ 387 void 388 hammer2_inode_depend(hammer2_inode_t *ip1, hammer2_inode_t *ip2) 389 { 390 hammer2_pfs_t *pmp; 391 hammer2_depend_t *depend; 392 393 pmp = ip1->pmp; 394 hammer2_spin_ex(&pmp->list_spin); 395 depend = hammer2_inode_setdepend_locked(ip1, NULL); 396 depend = hammer2_inode_setdepend_locked(ip2, depend); 397 hammer2_spin_unex(&pmp->list_spin); 398 } 399 400 /* 401 * Select a chain out of an inode's cluster and lock it. 402 * 403 * The inode does not have to be locked. 404 */ 405 hammer2_chain_t * 406 hammer2_inode_chain(hammer2_inode_t *ip, int clindex, int how) 407 { 408 hammer2_chain_t *chain; 409 hammer2_cluster_t *cluster; 410 411 hammer2_spin_sh(&ip->cluster_spin); 412 cluster = &ip->cluster; 413 if (clindex >= cluster->nchains) 414 chain = NULL; 415 else 416 chain = cluster->array[clindex].chain; 417 if (chain) { 418 hammer2_chain_ref(chain); 419 hammer2_spin_unsh(&ip->cluster_spin); 420 hammer2_chain_lock(chain, how); 421 } else { 422 hammer2_spin_unsh(&ip->cluster_spin); 423 } 424 return chain; 425 } 426 427 hammer2_chain_t * 428 hammer2_inode_chain_and_parent(hammer2_inode_t *ip, int clindex, 429 hammer2_chain_t **parentp, int how) 430 { 431 hammer2_chain_t *chain; 432 hammer2_chain_t *parent; 433 434 for (;;) { 435 hammer2_spin_sh(&ip->cluster_spin); 436 if (clindex >= ip->cluster.nchains) 437 chain = NULL; 438 else 439 chain = ip->cluster.array[clindex].chain; 440 if (chain) { 441 hammer2_chain_ref(chain); 442 hammer2_spin_unsh(&ip->cluster_spin); 443 hammer2_chain_lock(chain, how); 444 } else { 445 hammer2_spin_unsh(&ip->cluster_spin); 446 } 447 448 /* 449 * Get parent, lock order must be (parent, chain). 450 */ 451 parent = chain->parent; 452 if (parent) { 453 hammer2_chain_ref(parent); 454 hammer2_chain_unlock(chain); 455 hammer2_chain_lock(parent, how); 456 hammer2_chain_lock(chain, how); 457 } 458 if (ip->cluster.array[clindex].chain == chain && 459 chain->parent == parent) { 460 break; 461 } 462 463 /* 464 * Retry 465 */ 466 hammer2_chain_unlock(chain); 467 hammer2_chain_drop(chain); 468 if (parent) { 469 hammer2_chain_unlock(parent); 470 hammer2_chain_drop(parent); 471 } 472 } 473 *parentp = parent; 474 475 return chain; 476 } 477 478 /* 479 * Temporarily release a lock held shared or exclusive. Caller must 480 * hold the lock shared or exclusive on call and lock will be released 481 * on return. 482 * 483 * Restore a lock that was temporarily released. 484 */ 485 hammer2_mtx_state_t 486 hammer2_inode_lock_temp_release(hammer2_inode_t *ip) 487 { 488 return hammer2_mtx_temp_release(&ip->lock); 489 } 490 491 void 492 hammer2_inode_lock_temp_restore(hammer2_inode_t *ip, hammer2_mtx_state_t ostate) 493 { 494 hammer2_mtx_temp_restore(&ip->lock, ostate); 495 } 496 497 /* 498 * Upgrade a shared inode lock to exclusive and return. If the inode lock 499 * is already held exclusively this is a NOP. 500 * 501 * The caller MUST hold the inode lock either shared or exclusive on call 502 * and will own the lock exclusively on return. 503 * 504 * Returns non-zero if the lock was already exclusive prior to the upgrade. 505 */ 506 int 507 hammer2_inode_lock_upgrade(hammer2_inode_t *ip) 508 { 509 int wasexclusive; 510 511 if (mtx_islocked_ex(&ip->lock)) { 512 wasexclusive = 1; 513 } else { 514 hammer2_mtx_unlock(&ip->lock); 515 hammer2_mtx_ex(&ip->lock); 516 wasexclusive = 0; 517 } 518 return wasexclusive; 519 } 520 521 /* 522 * Downgrade an inode lock from exclusive to shared only if the inode 523 * lock was previously shared. If the inode lock was previously exclusive, 524 * this is a NOP. 525 */ 526 void 527 hammer2_inode_lock_downgrade(hammer2_inode_t *ip, int wasexclusive) 528 { 529 if (wasexclusive == 0) 530 hammer2_mtx_downgrade(&ip->lock); 531 } 532 533 /* 534 * Lookup an inode by inode number 535 */ 536 hammer2_inode_t * 537 hammer2_inode_lookup(hammer2_pfs_t *pmp, hammer2_tid_t inum) 538 { 539 hammer2_inode_t *ip; 540 541 KKASSERT(pmp); 542 if (pmp->spmp_hmp) { 543 ip = NULL; 544 } else { 545 hammer2_spin_ex(&pmp->inum_spin); 546 ip = RB_LOOKUP(hammer2_inode_tree, &pmp->inum_tree, inum); 547 if (ip) 548 hammer2_inode_ref(ip); 549 hammer2_spin_unex(&pmp->inum_spin); 550 } 551 return(ip); 552 } 553 554 /* 555 * Adding a ref to an inode is only legal if the inode already has at least 556 * one ref. 557 * 558 * (can be called with spinlock held) 559 */ 560 void 561 hammer2_inode_ref(hammer2_inode_t *ip) 562 { 563 atomic_add_int(&ip->refs, 1); 564 if (hammer2_debug & 0x80000) { 565 kprintf("INODE+1 %p (%d->%d)\n", ip, ip->refs - 1, ip->refs); 566 print_backtrace(8); 567 } 568 } 569 570 /* 571 * Drop an inode reference, freeing the inode when the last reference goes 572 * away. 573 */ 574 void 575 hammer2_inode_drop(hammer2_inode_t *ip) 576 { 577 hammer2_pfs_t *pmp; 578 u_int refs; 579 580 while (ip) { 581 if (hammer2_debug & 0x80000) { 582 kprintf("INODE-1 %p (%d->%d)\n", 583 ip, ip->refs, ip->refs - 1); 584 print_backtrace(8); 585 } 586 refs = ip->refs; 587 cpu_ccfence(); 588 if (refs == 1) { 589 /* 590 * Transition to zero, must interlock with 591 * the inode inumber lookup tree (if applicable). 592 * It should not be possible for anyone to race 593 * the transition to 0. 594 */ 595 pmp = ip->pmp; 596 KKASSERT(pmp); 597 hammer2_spin_ex(&pmp->inum_spin); 598 599 if (atomic_cmpset_int(&ip->refs, 1, 0)) { 600 KKASSERT(hammer2_mtx_refs(&ip->lock) == 0); 601 if (ip->flags & HAMMER2_INODE_ONRBTREE) { 602 atomic_clear_int(&ip->flags, 603 HAMMER2_INODE_ONRBTREE); 604 RB_REMOVE(hammer2_inode_tree, 605 &pmp->inum_tree, ip); 606 --pmp->inum_count; 607 } 608 hammer2_spin_unex(&pmp->inum_spin); 609 610 ip->pmp = NULL; 611 612 /* 613 * Cleaning out ip->cluster isn't entirely 614 * trivial. 615 */ 616 hammer2_inode_repoint(ip, NULL); 617 618 kfree_obj(ip, pmp->minode); 619 atomic_add_long(&pmp->inmem_inodes, -1); 620 ip = NULL; /* will terminate loop */ 621 } else { 622 hammer2_spin_unex(&ip->pmp->inum_spin); 623 } 624 } else { 625 /* 626 * Non zero transition 627 */ 628 if (atomic_cmpset_int(&ip->refs, refs, refs - 1)) 629 break; 630 } 631 } 632 } 633 634 /* 635 * Get the vnode associated with the given inode, allocating the vnode if 636 * necessary. The vnode will be returned exclusively locked. 637 * 638 * *errorp is set to a UNIX error, not a HAMMER2 error. 639 * 640 * The caller must lock the inode (shared or exclusive). 641 * 642 * Great care must be taken to avoid deadlocks and vnode acquisition/reclaim 643 * races. 644 */ 645 struct vnode * 646 hammer2_igetv(hammer2_inode_t *ip, int *errorp) 647 { 648 hammer2_pfs_t *pmp; 649 struct vnode *vp; 650 651 pmp = ip->pmp; 652 KKASSERT(pmp != NULL); 653 *errorp = 0; 654 655 for (;;) { 656 /* 657 * Attempt to reuse an existing vnode assignment. It is 658 * possible to race a reclaim so the vget() may fail. The 659 * inode must be unlocked during the vget() to avoid a 660 * deadlock against a reclaim. 661 */ 662 int wasexclusive; 663 664 vp = ip->vp; 665 if (vp) { 666 /* 667 * Inode must be unlocked during the vget() to avoid 668 * possible deadlocks, but leave the ip ref intact. 669 * 670 * vnode is held to prevent destruction during the 671 * vget(). The vget() can still fail if we lost 672 * a reclaim race on the vnode. 673 */ 674 hammer2_mtx_state_t ostate; 675 676 vhold(vp); 677 ostate = hammer2_inode_lock_temp_release(ip); 678 if (vget(vp, LK_EXCLUSIVE)) { 679 vdrop(vp); 680 hammer2_inode_lock_temp_restore(ip, ostate); 681 continue; 682 } 683 hammer2_inode_lock_temp_restore(ip, ostate); 684 vdrop(vp); 685 /* vp still locked and ref from vget */ 686 if (ip->vp != vp) { 687 kprintf("hammer2: igetv race %p/%p\n", 688 ip->vp, vp); 689 vput(vp); 690 continue; 691 } 692 *errorp = 0; 693 break; 694 } 695 696 /* 697 * No vnode exists, allocate a new vnode. Beware of 698 * allocation races. This function will return an 699 * exclusively locked and referenced vnode. 700 */ 701 *errorp = getnewvnode(VT_HAMMER2, pmp->mp, &vp, 0, 0); 702 if (*errorp) { 703 kprintf("hammer2: igetv getnewvnode failed %d\n", 704 *errorp); 705 vp = NULL; 706 break; 707 } 708 709 /* 710 * Lock the inode and check for an allocation race. 711 */ 712 wasexclusive = hammer2_inode_lock_upgrade(ip); 713 if (ip->vp != NULL) { 714 vp->v_type = VBAD; 715 vx_put(vp); 716 hammer2_inode_lock_downgrade(ip, wasexclusive); 717 continue; 718 } 719 720 switch (ip->meta.type) { 721 case HAMMER2_OBJTYPE_DIRECTORY: 722 vp->v_type = VDIR; 723 break; 724 case HAMMER2_OBJTYPE_REGFILE: 725 /* 726 * Regular file must use buffer cache I/O 727 * (VKVABIO cpu sync semantics supported) 728 */ 729 vp->v_type = VREG; 730 vsetflags(vp, VKVABIO); 731 vinitvmio(vp, ip->meta.size, 732 HAMMER2_LBUFSIZE, 733 (int)ip->meta.size & HAMMER2_LBUFMASK); 734 break; 735 case HAMMER2_OBJTYPE_SOFTLINK: 736 /* 737 * XXX for now we are using the generic file_read 738 * and file_write code so we need a buffer cache 739 * association. 740 * 741 * (VKVABIO cpu sync semantics supported) 742 */ 743 vp->v_type = VLNK; 744 vsetflags(vp, VKVABIO); 745 vinitvmio(vp, ip->meta.size, 746 HAMMER2_LBUFSIZE, 747 (int)ip->meta.size & HAMMER2_LBUFMASK); 748 break; 749 case HAMMER2_OBJTYPE_CDEV: 750 vp->v_type = VCHR; 751 /* fall through */ 752 case HAMMER2_OBJTYPE_BDEV: 753 vp->v_ops = &pmp->mp->mnt_vn_spec_ops; 754 if (ip->meta.type != HAMMER2_OBJTYPE_CDEV) 755 vp->v_type = VBLK; 756 addaliasu(vp, 757 ip->meta.rmajor, 758 ip->meta.rminor); 759 break; 760 case HAMMER2_OBJTYPE_FIFO: 761 vp->v_type = VFIFO; 762 vp->v_ops = &pmp->mp->mnt_vn_fifo_ops; 763 break; 764 case HAMMER2_OBJTYPE_SOCKET: 765 vp->v_type = VSOCK; 766 break; 767 default: 768 panic("hammer2: unhandled objtype %d", 769 ip->meta.type); 770 break; 771 } 772 773 if (ip == pmp->iroot) 774 vsetflags(vp, VROOT); 775 776 vp->v_data = ip; 777 ip->vp = vp; 778 hammer2_inode_ref(ip); /* vp association */ 779 hammer2_inode_lock_downgrade(ip, wasexclusive); 780 vx_downgrade(vp); 781 break; 782 } 783 784 /* 785 * Return non-NULL vp and *errorp == 0, or NULL vp and *errorp != 0. 786 */ 787 if (hammer2_debug & 0x0002) { 788 kprintf("igetv vp %p refs 0x%08x aux 0x%08x\n", 789 vp, vp->v_refcnt, vp->v_auxrefs); 790 } 791 return (vp); 792 } 793 794 /* 795 * XXX this API needs a rewrite. It needs to be split into a 796 * hammer2_inode_alloc() and hammer2_inode_build() to allow us to get 797 * rid of the inode/chain lock reversal fudge. 798 * 799 * Returns the inode associated with the passed-in cluster, allocating a new 800 * hammer2_inode structure if necessary, then synchronizing it to the passed 801 * xop cluster. When synchronizing, if idx >= 0, only cluster index (idx) 802 * is synchronized. Otherwise the whole cluster is synchronized. inum will 803 * be extracted from the passed-in xop and the inum argument will be ignored. 804 * 805 * If xop is passed as NULL then a new hammer2_inode is allocated with the 806 * specified inum, and returned. For normal inodes, the inode will be 807 * indexed in memory and if it already exists the existing ip will be 808 * returned instead of allocating a new one. The superroot and PFS inodes 809 * are not indexed in memory. 810 * 811 * The passed-in cluster must be locked and will remain locked on return. 812 * The returned inode will be locked and the caller may dispose of both 813 * via hammer2_inode_unlock() + hammer2_inode_drop(). However, if the caller 814 * needs to resolve a hardlink it must ref/unlock/relock/drop the inode. 815 * 816 * The hammer2_inode structure regulates the interface between the high level 817 * kernel VNOPS API and the filesystem backend (the chains). 818 * 819 * On return the inode is locked with the supplied cluster. 820 */ 821 hammer2_inode_t * 822 hammer2_inode_get(hammer2_pfs_t *pmp, hammer2_xop_head_t *xop, 823 hammer2_tid_t inum, int idx) 824 { 825 hammer2_inode_t *nip; 826 const hammer2_inode_data_t *iptmp; 827 const hammer2_inode_data_t *nipdata; 828 829 KKASSERT(xop == NULL || 830 hammer2_cluster_type(&xop->cluster) == 831 HAMMER2_BREF_TYPE_INODE); 832 KKASSERT(pmp); 833 834 /* 835 * Interlocked lookup/ref of the inode. This code is only needed 836 * when looking up inodes with nlinks != 0 (TODO: optimize out 837 * otherwise and test for duplicates). 838 * 839 * Cluster can be NULL during the initial pfs allocation. 840 */ 841 if (xop) { 842 iptmp = &hammer2_xop_gdata(xop)->ipdata; 843 inum = iptmp->meta.inum; 844 hammer2_xop_pdata(xop); 845 } 846 again: 847 nip = hammer2_inode_lookup(pmp, inum); 848 if (nip) { 849 /* 850 * We may have to unhold the cluster to avoid a deadlock 851 * against vnlru (and possibly other XOPs). 852 */ 853 if (xop) { 854 if (hammer2_mtx_ex_try(&nip->lock) != 0) { 855 hammer2_cluster_unhold(&xop->cluster); 856 hammer2_mtx_ex(&nip->lock); 857 hammer2_cluster_rehold(&xop->cluster); 858 } 859 } else { 860 hammer2_mtx_ex(&nip->lock); 861 } 862 863 /* 864 * Handle SMP race (not applicable to the super-root spmp 865 * which can't index inodes due to duplicative inode numbers). 866 */ 867 if (pmp->spmp_hmp == NULL && 868 (nip->flags & HAMMER2_INODE_ONRBTREE) == 0) { 869 hammer2_mtx_unlock(&nip->lock); 870 hammer2_inode_drop(nip); 871 goto again; 872 } 873 if (xop) { 874 if (idx >= 0) 875 hammer2_inode_repoint_one(nip, &xop->cluster, 876 idx); 877 else 878 hammer2_inode_repoint(nip, &xop->cluster); 879 } 880 return nip; 881 } 882 883 /* 884 * We couldn't find the inode number, create a new inode and try to 885 * insert it, handle insertion races. 886 */ 887 nip = kmalloc_obj(sizeof(*nip), pmp->minode, M_WAITOK | M_ZERO); 888 hammer2_spin_init(&nip->cluster_spin, "h2clspin"); 889 atomic_add_long(&pmp->inmem_inodes, 1); 890 891 /* 892 * Initialize nip's cluster. A cluster is provided for normal 893 * inodes but typically not for the super-root or PFS inodes. 894 */ 895 { 896 hammer2_inode_t *nnip = nip; 897 nip->ihash = (int)hammer2_icrc32(&nnip, sizeof(nnip)); 898 } 899 900 nip->cluster.refs = 1; 901 nip->cluster.pmp = pmp; 902 nip->cluster.flags |= HAMMER2_CLUSTER_INODE; 903 if (xop) { 904 nipdata = &hammer2_xop_gdata(xop)->ipdata; 905 nip->meta = nipdata->meta; 906 hammer2_xop_pdata(xop); 907 hammer2_inode_repoint(nip, &xop->cluster); 908 } else { 909 nip->meta.inum = inum; /* PFS inum is always 1 XXX */ 910 /* mtime will be updated when a cluster is available */ 911 } 912 913 nip->pmp = pmp; 914 915 /* 916 * ref and lock on nip gives it state compatible to after a 917 * hammer2_inode_lock() call. 918 */ 919 nip->refs = 1; 920 hammer2_mtx_init(&nip->lock, "h2inode"); 921 hammer2_mtx_init(&nip->truncate_lock, "h2trunc"); 922 hammer2_mtx_ex(&nip->lock); 923 TAILQ_INIT(&nip->depend_static.sideq); 924 /* combination of thread lock and chain lock == inode lock */ 925 926 /* 927 * Attempt to add the inode. If it fails we raced another inode 928 * get. Undo all the work and try again. 929 */ 930 if (pmp->spmp_hmp == NULL) { 931 hammer2_spin_ex(&pmp->inum_spin); 932 if (RB_INSERT(hammer2_inode_tree, &pmp->inum_tree, nip)) { 933 hammer2_spin_unex(&pmp->inum_spin); 934 hammer2_mtx_unlock(&nip->lock); 935 hammer2_inode_drop(nip); 936 goto again; 937 } 938 atomic_set_int(&nip->flags, HAMMER2_INODE_ONRBTREE); 939 ++pmp->inum_count; 940 hammer2_spin_unex(&pmp->inum_spin); 941 } 942 return (nip); 943 } 944 945 /* 946 * Create a PFS inode under the superroot. This function will create the 947 * inode, its media chains, and also insert it into the media. 948 * 949 * Caller must be in a flush transaction because we are inserting the inode 950 * onto the media. 951 */ 952 hammer2_inode_t * 953 hammer2_inode_create_pfs(hammer2_pfs_t *spmp, 954 const uint8_t *name, size_t name_len, 955 int *errorp) 956 { 957 hammer2_xop_create_t *xop; 958 hammer2_inode_t *pip; 959 hammer2_inode_t *nip; 960 int error; 961 uuid_t pip_uid; 962 uuid_t pip_gid; 963 uint32_t pip_mode; 964 uint8_t pip_comp_algo; 965 uint8_t pip_check_algo; 966 hammer2_tid_t pip_inum; 967 hammer2_key_t lhc; 968 969 pip = spmp->iroot; 970 nip = NULL; 971 972 lhc = hammer2_dirhash(name, name_len); 973 *errorp = 0; 974 975 /* 976 * Locate the inode or indirect block to create the new 977 * entry in. At the same time check for key collisions 978 * and iterate until we don't get one. 979 * 980 * Lock the directory exclusively for now to guarantee that 981 * we can find an unused lhc for the name. Due to collisions, 982 * two different creates can end up with the same lhc so we 983 * cannot depend on the OS to prevent the collision. 984 */ 985 hammer2_inode_lock(pip, 0); 986 987 pip_uid = pip->meta.uid; 988 pip_gid = pip->meta.gid; 989 pip_mode = pip->meta.mode; 990 pip_comp_algo = pip->meta.comp_algo; 991 pip_check_algo = pip->meta.check_algo; 992 pip_inum = (pip == pip->pmp->iroot) ? 1 : pip->meta.inum; 993 994 /* 995 * Locate an unused key in the collision space. 996 */ 997 { 998 hammer2_xop_scanlhc_t *sxop; 999 hammer2_key_t lhcbase; 1000 1001 lhcbase = lhc; 1002 sxop = hammer2_xop_alloc(pip, HAMMER2_XOP_MODIFYING); 1003 sxop->lhc = lhc; 1004 hammer2_xop_start(&sxop->head, &hammer2_scanlhc_desc); 1005 while ((error = hammer2_xop_collect(&sxop->head, 0)) == 0) { 1006 if (lhc != sxop->head.cluster.focus->bref.key) 1007 break; 1008 ++lhc; 1009 } 1010 hammer2_xop_retire(&sxop->head, HAMMER2_XOPMASK_VOP); 1011 1012 if (error) { 1013 if (error != HAMMER2_ERROR_ENOENT) 1014 goto done2; 1015 ++lhc; 1016 error = 0; 1017 } 1018 if ((lhcbase ^ lhc) & ~HAMMER2_DIRHASH_LOMASK) { 1019 error = HAMMER2_ERROR_ENOSPC; 1020 goto done2; 1021 } 1022 } 1023 1024 /* 1025 * Create the inode with the lhc as the key. 1026 */ 1027 xop = hammer2_xop_alloc(pip, HAMMER2_XOP_MODIFYING); 1028 xop->lhc = lhc; 1029 xop->flags = HAMMER2_INSERT_PFSROOT; 1030 bzero(&xop->meta, sizeof(xop->meta)); 1031 1032 xop->meta.type = HAMMER2_OBJTYPE_DIRECTORY; 1033 xop->meta.inum = 1; 1034 xop->meta.iparent = pip_inum; 1035 1036 /* Inherit parent's inode compression mode. */ 1037 xop->meta.comp_algo = pip_comp_algo; 1038 xop->meta.check_algo = pip_check_algo; 1039 xop->meta.version = HAMMER2_INODE_VERSION_ONE; 1040 hammer2_update_time(&xop->meta.ctime); 1041 xop->meta.mtime = xop->meta.ctime; 1042 xop->meta.mode = 0755; 1043 xop->meta.nlinks = 1; 1044 1045 /* 1046 * Regular files and softlinks allow a small amount of data to be 1047 * directly embedded in the inode. This flag will be cleared if 1048 * the size is extended past the embedded limit. 1049 */ 1050 if (xop->meta.type == HAMMER2_OBJTYPE_REGFILE || 1051 xop->meta.type == HAMMER2_OBJTYPE_SOFTLINK) { 1052 xop->meta.op_flags |= HAMMER2_OPFLAG_DIRECTDATA; 1053 } 1054 hammer2_xop_setname(&xop->head, name, name_len); 1055 xop->meta.name_len = name_len; 1056 xop->meta.name_key = lhc; 1057 KKASSERT(name_len < HAMMER2_INODE_MAXNAME); 1058 1059 hammer2_xop_start(&xop->head, &hammer2_inode_create_desc); 1060 1061 error = hammer2_xop_collect(&xop->head, 0); 1062 #if INODE_DEBUG 1063 kprintf("CREATE INODE %*.*s\n", 1064 (int)name_len, (int)name_len, name); 1065 #endif 1066 1067 if (error) { 1068 *errorp = error; 1069 goto done; 1070 } 1071 1072 /* 1073 * Set up the new inode if not a hardlink pointer. 1074 * 1075 * NOTE: *_get() integrates chain's lock into the inode lock. 1076 * 1077 * NOTE: Only one new inode can currently be created per 1078 * transaction. If the need arises we can adjust 1079 * hammer2_trans_init() to allow more. 1080 * 1081 * NOTE: nipdata will have chain's blockset data. 1082 */ 1083 nip = hammer2_inode_get(pip->pmp, &xop->head, -1, -1); 1084 nip->comp_heuristic = 0; 1085 done: 1086 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP); 1087 done2: 1088 hammer2_inode_unlock(pip); 1089 1090 return (nip); 1091 } 1092 1093 /* 1094 * Create a new, normal inode. This function will create the inode, 1095 * the media chains, but will not insert the chains onto the media topology 1096 * (doing so would require a flush transaction and cause long stalls). 1097 * 1098 * Caller must be in a normal transaction. 1099 */ 1100 hammer2_inode_t * 1101 hammer2_inode_create_normal(hammer2_inode_t *pip, 1102 struct vattr *vap, struct ucred *cred, 1103 hammer2_key_t inum, int *errorp) 1104 { 1105 hammer2_xop_create_t *xop; 1106 hammer2_inode_t *dip; 1107 hammer2_inode_t *nip; 1108 int error; 1109 uid_t xuid; 1110 uuid_t pip_uid; 1111 uuid_t pip_gid; 1112 uint32_t pip_mode; 1113 uint8_t pip_comp_algo; 1114 uint8_t pip_check_algo; 1115 hammer2_tid_t pip_inum; 1116 uint8_t type; 1117 1118 dip = pip->pmp->iroot; 1119 KKASSERT(dip != NULL); 1120 1121 *errorp = 0; 1122 1123 /*hammer2_inode_lock(dip, 0);*/ 1124 1125 pip_uid = pip->meta.uid; 1126 pip_gid = pip->meta.gid; 1127 pip_mode = pip->meta.mode; 1128 pip_comp_algo = pip->meta.comp_algo; 1129 pip_check_algo = pip->meta.check_algo; 1130 pip_inum = (pip == pip->pmp->iroot) ? 1 : pip->meta.inum; 1131 1132 /* 1133 * Create the in-memory hammer2_inode structure for the specified 1134 * inode. 1135 */ 1136 nip = hammer2_inode_get(dip->pmp, NULL, inum, -1); 1137 nip->comp_heuristic = 0; 1138 KKASSERT((nip->flags & HAMMER2_INODE_CREATING) == 0 && 1139 nip->cluster.nchains == 0); 1140 atomic_set_int(&nip->flags, HAMMER2_INODE_CREATING); 1141 1142 /* 1143 * Setup the inode meta-data 1144 */ 1145 nip->meta.type = hammer2_get_obj_type(vap->va_type); 1146 1147 switch (nip->meta.type) { 1148 case HAMMER2_OBJTYPE_CDEV: 1149 case HAMMER2_OBJTYPE_BDEV: 1150 nip->meta.rmajor = vap->va_rmajor; 1151 nip->meta.rminor = vap->va_rminor; 1152 break; 1153 default: 1154 break; 1155 } 1156 type = nip->meta.type; 1157 1158 KKASSERT(nip->meta.inum == inum); 1159 nip->meta.iparent = pip_inum; 1160 1161 /* Inherit parent's inode compression mode. */ 1162 nip->meta.comp_algo = pip_comp_algo; 1163 nip->meta.check_algo = pip_check_algo; 1164 nip->meta.version = HAMMER2_INODE_VERSION_ONE; 1165 hammer2_update_time(&nip->meta.ctime); 1166 nip->meta.mtime = nip->meta.ctime; 1167 nip->meta.mode = vap->va_mode; 1168 nip->meta.nlinks = 1; 1169 1170 xuid = hammer2_to_unix_xid(&pip_uid); 1171 xuid = vop_helper_create_uid(dip->pmp->mp, pip_mode, 1172 xuid, cred, 1173 &vap->va_mode); 1174 if (vap->va_vaflags & VA_UID_UUID_VALID) 1175 nip->meta.uid = vap->va_uid_uuid; 1176 else if (vap->va_uid != (uid_t)VNOVAL) 1177 hammer2_guid_to_uuid(&nip->meta.uid, vap->va_uid); 1178 else 1179 hammer2_guid_to_uuid(&nip->meta.uid, xuid); 1180 1181 if (vap->va_vaflags & VA_GID_UUID_VALID) 1182 nip->meta.gid = vap->va_gid_uuid; 1183 else if (vap->va_gid != (gid_t)VNOVAL) 1184 hammer2_guid_to_uuid(&nip->meta.gid, vap->va_gid); 1185 else 1186 nip->meta.gid = pip_gid; 1187 1188 /* 1189 * Regular files and softlinks allow a small amount of data to be 1190 * directly embedded in the inode. This flag will be cleared if 1191 * the size is extended past the embedded limit. 1192 */ 1193 if (nip->meta.type == HAMMER2_OBJTYPE_REGFILE || 1194 nip->meta.type == HAMMER2_OBJTYPE_SOFTLINK) { 1195 nip->meta.op_flags |= HAMMER2_OPFLAG_DIRECTDATA; 1196 } 1197 1198 /* 1199 * Create the inode using (inum) as the key. Pass pip for 1200 * method inheritance. 1201 */ 1202 xop = hammer2_xop_alloc(pip, HAMMER2_XOP_MODIFYING); 1203 xop->lhc = inum; 1204 xop->flags = 0; 1205 xop->meta = nip->meta; 1206 KKASSERT(vap); 1207 1208 xop->meta.name_len = hammer2_xop_setname_inum(&xop->head, inum); 1209 xop->meta.name_key = inum; 1210 nip->meta.name_len = xop->meta.name_len; 1211 nip->meta.name_key = xop->meta.name_key; 1212 hammer2_inode_modify(nip); 1213 1214 /* 1215 * Create the inode media chains but leave them detached. We are 1216 * not in a flush transaction so we can't mess with media topology 1217 * above normal inodes (i.e. the index of the inodes themselves). 1218 * 1219 * We've already set the INODE_CREATING flag. The inode's media 1220 * chains will be inserted onto the media topology on the next 1221 * filesystem sync. 1222 */ 1223 hammer2_xop_start(&xop->head, &hammer2_inode_create_det_desc); 1224 1225 error = hammer2_xop_collect(&xop->head, 0); 1226 #if INODE_DEBUG 1227 kprintf("create inode type %d error %d\n", nip->meta.type, error); 1228 #endif 1229 1230 if (error) { 1231 *errorp = error; 1232 goto done; 1233 } 1234 1235 /* 1236 * Associate the media chains created by the backend with the 1237 * frontend inode. 1238 */ 1239 hammer2_inode_repoint(nip, &xop->head.cluster); 1240 done: 1241 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP); 1242 /*hammer2_inode_unlock(dip);*/ 1243 1244 return (nip); 1245 } 1246 1247 /* 1248 * Create a directory entry under dip with the specified name, inode number, 1249 * and OBJTYPE (type). 1250 * 1251 * This returns a UNIX errno code, not a HAMMER2_ERROR_* code. 1252 * 1253 * Caller must hold dip locked. 1254 */ 1255 int 1256 hammer2_dirent_create(hammer2_inode_t *dip, const char *name, size_t name_len, 1257 hammer2_key_t inum, uint8_t type) 1258 { 1259 hammer2_xop_mkdirent_t *xop; 1260 hammer2_key_t lhc; 1261 int error; 1262 1263 lhc = 0; 1264 error = 0; 1265 1266 KKASSERT(name != NULL); 1267 lhc = hammer2_dirhash(name, name_len); 1268 1269 /* 1270 * Locate the inode or indirect block to create the new 1271 * entry in. At the same time check for key collisions 1272 * and iterate until we don't get one. 1273 * 1274 * Lock the directory exclusively for now to guarantee that 1275 * we can find an unused lhc for the name. Due to collisions, 1276 * two different creates can end up with the same lhc so we 1277 * cannot depend on the OS to prevent the collision. 1278 */ 1279 hammer2_inode_modify(dip); 1280 1281 /* 1282 * If name specified, locate an unused key in the collision space. 1283 * Otherwise use the passed-in lhc directly. 1284 */ 1285 { 1286 hammer2_xop_scanlhc_t *sxop; 1287 hammer2_key_t lhcbase; 1288 1289 lhcbase = lhc; 1290 sxop = hammer2_xop_alloc(dip, HAMMER2_XOP_MODIFYING); 1291 sxop->lhc = lhc; 1292 hammer2_xop_start(&sxop->head, &hammer2_scanlhc_desc); 1293 while ((error = hammer2_xop_collect(&sxop->head, 0)) == 0) { 1294 if (lhc != sxop->head.cluster.focus->bref.key) 1295 break; 1296 ++lhc; 1297 } 1298 hammer2_xop_retire(&sxop->head, HAMMER2_XOPMASK_VOP); 1299 1300 if (error) { 1301 if (error != HAMMER2_ERROR_ENOENT) 1302 goto done2; 1303 ++lhc; 1304 error = 0; 1305 } 1306 if ((lhcbase ^ lhc) & ~HAMMER2_DIRHASH_LOMASK) { 1307 error = HAMMER2_ERROR_ENOSPC; 1308 goto done2; 1309 } 1310 } 1311 1312 /* 1313 * Create the directory entry with the lhc as the key. 1314 */ 1315 xop = hammer2_xop_alloc(dip, HAMMER2_XOP_MODIFYING); 1316 xop->lhc = lhc; 1317 bzero(&xop->dirent, sizeof(xop->dirent)); 1318 xop->dirent.inum = inum; 1319 xop->dirent.type = type; 1320 xop->dirent.namlen = name_len; 1321 1322 KKASSERT(name_len < HAMMER2_INODE_MAXNAME); 1323 hammer2_xop_setname(&xop->head, name, name_len); 1324 1325 hammer2_xop_start(&xop->head, &hammer2_inode_mkdirent_desc); 1326 1327 error = hammer2_xop_collect(&xop->head, 0); 1328 1329 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP); 1330 done2: 1331 error = hammer2_error_to_errno(error); 1332 1333 return error; 1334 } 1335 1336 /* 1337 * Repoint ip->cluster's chains to cluster's chains and fixup the default 1338 * focus. All items, valid or invalid, are repointed. hammer2_xop_start() 1339 * filters out invalid or non-matching elements. 1340 * 1341 * Caller must hold the inode and cluster exclusive locked, if not NULL, 1342 * must also be locked. 1343 * 1344 * Cluster may be NULL to clean out any chains in ip->cluster. 1345 */ 1346 void 1347 hammer2_inode_repoint(hammer2_inode_t *ip, hammer2_cluster_t *cluster) 1348 { 1349 hammer2_chain_t *dropch[HAMMER2_MAXCLUSTER]; 1350 hammer2_chain_t *ochain; 1351 hammer2_chain_t *nchain; 1352 int i; 1353 1354 bzero(dropch, sizeof(dropch)); 1355 1356 /* 1357 * Replace chains in ip->cluster with chains from cluster and 1358 * adjust the focus if necessary. 1359 * 1360 * NOTE: nchain and/or ochain can be NULL due to gaps 1361 * in the cluster arrays. 1362 */ 1363 hammer2_spin_ex(&ip->cluster_spin); 1364 for (i = 0; cluster && i < cluster->nchains; ++i) { 1365 /* 1366 * Do not replace elements which are the same. Also handle 1367 * element count discrepancies. 1368 */ 1369 nchain = cluster->array[i].chain; 1370 if (i < ip->cluster.nchains) { 1371 ochain = ip->cluster.array[i].chain; 1372 if (ochain == nchain) 1373 continue; 1374 } else { 1375 ochain = NULL; 1376 } 1377 1378 /* 1379 * Make adjustments 1380 */ 1381 ip->cluster.array[i].chain = nchain; 1382 ip->cluster.array[i].flags &= ~HAMMER2_CITEM_INVALID; 1383 ip->cluster.array[i].flags |= cluster->array[i].flags & 1384 HAMMER2_CITEM_INVALID; 1385 if (nchain) 1386 hammer2_chain_ref(nchain); 1387 dropch[i] = ochain; 1388 } 1389 1390 /* 1391 * Release any left-over chains in ip->cluster. 1392 */ 1393 while (i < ip->cluster.nchains) { 1394 nchain = ip->cluster.array[i].chain; 1395 if (nchain) { 1396 ip->cluster.array[i].chain = NULL; 1397 ip->cluster.array[i].flags |= HAMMER2_CITEM_INVALID; 1398 } 1399 dropch[i] = nchain; 1400 ++i; 1401 } 1402 1403 /* 1404 * Fixup fields. Note that the inode-embedded cluster is never 1405 * directly locked. 1406 */ 1407 if (cluster) { 1408 ip->cluster.nchains = cluster->nchains; 1409 ip->cluster.focus = cluster->focus; 1410 ip->cluster.flags = cluster->flags & ~HAMMER2_CLUSTER_LOCKED; 1411 } else { 1412 ip->cluster.nchains = 0; 1413 ip->cluster.focus = NULL; 1414 ip->cluster.flags &= ~HAMMER2_CLUSTER_ZFLAGS; 1415 } 1416 1417 hammer2_spin_unex(&ip->cluster_spin); 1418 1419 /* 1420 * Cleanup outside of spinlock 1421 */ 1422 while (--i >= 0) { 1423 if (dropch[i]) 1424 hammer2_chain_drop(dropch[i]); 1425 } 1426 } 1427 1428 /* 1429 * Repoint a single element from the cluster to the ip. Used by the 1430 * synchronization threads to piecemeal update inodes. Does not change 1431 * focus and requires inode to be re-locked to clean-up flags (XXX). 1432 */ 1433 void 1434 hammer2_inode_repoint_one(hammer2_inode_t *ip, hammer2_cluster_t *cluster, 1435 int idx) 1436 { 1437 hammer2_chain_t *ochain; 1438 hammer2_chain_t *nchain; 1439 int i; 1440 1441 hammer2_spin_ex(&ip->cluster_spin); 1442 KKASSERT(idx < cluster->nchains); 1443 if (idx < ip->cluster.nchains) { 1444 ochain = ip->cluster.array[idx].chain; 1445 nchain = cluster->array[idx].chain; 1446 } else { 1447 ochain = NULL; 1448 nchain = cluster->array[idx].chain; 1449 for (i = ip->cluster.nchains; i <= idx; ++i) { 1450 bzero(&ip->cluster.array[i], 1451 sizeof(ip->cluster.array[i])); 1452 ip->cluster.array[i].flags |= HAMMER2_CITEM_INVALID; 1453 } 1454 ip->cluster.nchains = idx + 1; 1455 } 1456 if (ochain != nchain) { 1457 /* 1458 * Make adjustments. 1459 */ 1460 ip->cluster.array[idx].chain = nchain; 1461 ip->cluster.array[idx].flags &= ~HAMMER2_CITEM_INVALID; 1462 ip->cluster.array[idx].flags |= cluster->array[idx].flags & 1463 HAMMER2_CITEM_INVALID; 1464 } 1465 hammer2_spin_unex(&ip->cluster_spin); 1466 if (ochain != nchain) { 1467 if (nchain) 1468 hammer2_chain_ref(nchain); 1469 if (ochain) 1470 hammer2_chain_drop(ochain); 1471 } 1472 } 1473 1474 hammer2_key_t 1475 hammer2_inode_data_count(const hammer2_inode_t *ip) 1476 { 1477 hammer2_chain_t *chain; 1478 hammer2_key_t count = 0; 1479 int i; 1480 1481 for (i = 0; i < ip->cluster.nchains; ++i) { 1482 if ((chain = ip->cluster.array[i].chain) != NULL) { 1483 if (count < chain->bref.embed.stats.data_count) 1484 count = chain->bref.embed.stats.data_count; 1485 } 1486 } 1487 return count; 1488 } 1489 1490 hammer2_key_t 1491 hammer2_inode_inode_count(const hammer2_inode_t *ip) 1492 { 1493 hammer2_chain_t *chain; 1494 hammer2_key_t count = 0; 1495 int i; 1496 1497 for (i = 0; i < ip->cluster.nchains; ++i) { 1498 if ((chain = ip->cluster.array[i].chain) != NULL) { 1499 if (count < chain->bref.embed.stats.inode_count) 1500 count = chain->bref.embed.stats.inode_count; 1501 } 1502 } 1503 return count; 1504 } 1505 1506 /* 1507 * Called with a locked inode to finish unlinking an inode after xop_unlink 1508 * had been run. This function is responsible for decrementing nlinks. 1509 * 1510 * We don't bother decrementing nlinks if the file is not open and this was 1511 * the last link. 1512 * 1513 * If the inode is a hardlink target it's chain has not yet been deleted, 1514 * otherwise it's chain has been deleted. 1515 * 1516 * If isopen then any prior deletion was not permanent and the inode is 1517 * left intact with nlinks == 0; 1518 */ 1519 int 1520 hammer2_inode_unlink_finisher(hammer2_inode_t *ip, struct vnode **vprecyclep) 1521 { 1522 hammer2_pfs_t *pmp; 1523 struct vnode *vp; 1524 int error; 1525 1526 pmp = ip->pmp; 1527 error = 0; 1528 1529 /* 1530 * Decrement nlinks. Catch a bad nlinks count here too (e.g. 0 or 1531 * negative), and just assume a transition to 0. 1532 */ 1533 if ((int64_t)ip->meta.nlinks <= 1) { 1534 atomic_set_int(&ip->flags, HAMMER2_INODE_ISUNLINKED); 1535 1536 /* 1537 * Scrap the vnode as quickly as possible. The vp association 1538 * stays intact while we hold the inode locked. However, vp 1539 * can be NULL here. 1540 */ 1541 vp = ip->vp; 1542 cpu_ccfence(); 1543 1544 /* 1545 * If no vp is associated there is no high-level state to 1546 * deal with and we can scrap the inode immediately. 1547 */ 1548 if (vp == NULL) { 1549 if ((ip->flags & HAMMER2_INODE_DELETING) == 0) { 1550 atomic_set_int(&ip->flags, 1551 HAMMER2_INODE_DELETING); 1552 hammer2_inode_delayed_sideq(ip); 1553 } 1554 return 0; 1555 } 1556 1557 /* 1558 * Because INODE_ISUNLINKED is set with the inode lock 1559 * held, the vnode cannot be ripped up from under us. 1560 * There may still be refs so knote anyone waiting for 1561 * a delete notification. 1562 * 1563 * The vnode is not necessarily ref'd due to the unlinking 1564 * itself, so we have to defer handling to the end of the 1565 * VOP, which will then call hammer2_inode_vprecycle(). 1566 */ 1567 if (vprecyclep) { 1568 vhold(vp); 1569 *vprecyclep = vp; 1570 } 1571 } 1572 1573 /* 1574 * Adjust nlinks and retain the inode on the media for now 1575 */ 1576 hammer2_inode_modify(ip); 1577 if ((int64_t)ip->meta.nlinks > 1) 1578 --ip->meta.nlinks; 1579 else 1580 ip->meta.nlinks = 0; 1581 1582 return 0; 1583 } 1584 1585 /* 1586 * Called at the end of a VOP that removes a file with a vnode that 1587 * we want to try to dispose of quickly due to a file deletion. If 1588 * we don't do this, the vnode can hang around with 0 refs for a very 1589 * long time and prevent reclamation of the underlying file and inode 1590 * (inode remains on-media with nlinks == 0 until the vnode is recycled 1591 * due to random system activity or a umount). 1592 */ 1593 void 1594 hammer2_inode_vprecycle(struct vnode *vp) 1595 { 1596 if (vget(vp, LK_EXCLUSIVE) == 0) { 1597 vfinalize(vp); 1598 hammer2_knote(vp, NOTE_DELETE); 1599 vdrop(vp); 1600 vput(vp); 1601 } else { 1602 vdrop(vp); 1603 } 1604 } 1605 1606 1607 /* 1608 * Mark an inode as being modified, meaning that the caller will modify 1609 * ip->meta. 1610 * 1611 * If a vnode is present we set the vnode dirty and the nominal filesystem 1612 * sync will also handle synchronizing the inode meta-data. Unless NOSIDEQ 1613 * we must ensure that the inode is on pmp->sideq. 1614 * 1615 * NOTE: We must always queue the inode to the sideq. This allows H2 to 1616 * shortcut vsyncscan() and flush inodes and their related vnodes 1617 * in a two stages. H2 still calls vfsync() for each vnode. 1618 * 1619 * NOTE: No mtid (modify_tid) is passed into this routine. The caller is 1620 * only modifying the in-memory inode. A modify_tid is synchronized 1621 * later when the inode gets flushed. 1622 * 1623 * NOTE: As an exception to the general rule, the inode MAY be locked 1624 * shared for this particular call. 1625 */ 1626 void 1627 hammer2_inode_modify(hammer2_inode_t *ip) 1628 { 1629 atomic_set_int(&ip->flags, HAMMER2_INODE_MODIFIED); 1630 if (ip->vp) 1631 vsetisdirty(ip->vp); 1632 if (ip->pmp && (ip->flags & HAMMER2_INODE_NOSIDEQ) == 0) 1633 hammer2_inode_delayed_sideq(ip); 1634 } 1635 1636 /* 1637 * Synchronize the inode's frontend state with the chain state prior 1638 * to any explicit flush of the inode or any strategy write call. This 1639 * does not flush the inode's chain or its sub-topology to media (higher 1640 * level layers are responsible for doing that). 1641 * 1642 * Called with a locked inode inside a normal transaction. 1643 * 1644 * inode must be locked. 1645 */ 1646 int 1647 hammer2_inode_chain_sync(hammer2_inode_t *ip) 1648 { 1649 int error; 1650 1651 error = 0; 1652 if (ip->flags & (HAMMER2_INODE_RESIZED | HAMMER2_INODE_MODIFIED)) { 1653 hammer2_xop_fsync_t *xop; 1654 1655 xop = hammer2_xop_alloc(ip, HAMMER2_XOP_MODIFYING); 1656 xop->clear_directdata = 0; 1657 if (ip->flags & HAMMER2_INODE_RESIZED) { 1658 if ((ip->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA) && 1659 ip->meta.size > HAMMER2_EMBEDDED_BYTES) { 1660 ip->meta.op_flags &= ~HAMMER2_OPFLAG_DIRECTDATA; 1661 xop->clear_directdata = 1; 1662 } 1663 xop->osize = ip->osize; 1664 } else { 1665 xop->osize = ip->meta.size; /* safety */ 1666 } 1667 xop->ipflags = ip->flags; 1668 xop->meta = ip->meta; 1669 1670 atomic_clear_int(&ip->flags, HAMMER2_INODE_RESIZED | 1671 HAMMER2_INODE_MODIFIED); 1672 hammer2_xop_start(&xop->head, &hammer2_inode_chain_sync_desc); 1673 error = hammer2_xop_collect(&xop->head, 0); 1674 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP); 1675 if (error == HAMMER2_ERROR_ENOENT) 1676 error = 0; 1677 if (error) { 1678 kprintf("hammer2: unable to fsync inode %p\n", ip); 1679 /* 1680 atomic_set_int(&ip->flags, 1681 xop->ipflags & (HAMMER2_INODE_RESIZED | 1682 HAMMER2_INODE_MODIFIED)); 1683 */ 1684 /* XXX return error somehow? */ 1685 } 1686 } 1687 return error; 1688 } 1689 1690 /* 1691 * When an inode is flagged INODE_CREATING its chains have not actually 1692 * been inserting into the on-media tree yet. 1693 */ 1694 int 1695 hammer2_inode_chain_ins(hammer2_inode_t *ip) 1696 { 1697 int error; 1698 1699 error = 0; 1700 if (ip->flags & HAMMER2_INODE_CREATING) { 1701 hammer2_xop_create_t *xop; 1702 1703 atomic_clear_int(&ip->flags, HAMMER2_INODE_CREATING); 1704 xop = hammer2_xop_alloc(ip, HAMMER2_XOP_MODIFYING); 1705 xop->lhc = ip->meta.inum; 1706 xop->flags = 0; 1707 hammer2_xop_start(&xop->head, &hammer2_inode_create_ins_desc); 1708 error = hammer2_xop_collect(&xop->head, 0); 1709 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP); 1710 if (error == HAMMER2_ERROR_ENOENT) 1711 error = 0; 1712 if (error) { 1713 kprintf("hammer2: backend unable to " 1714 "insert inode %p %ld\n", ip, ip->meta.inum); 1715 /* XXX return error somehow? */ 1716 } 1717 } 1718 return error; 1719 } 1720 1721 /* 1722 * When an inode is flagged INODE_DELETING it has been deleted (no directory 1723 * entry or open refs are left, though as an optimization H2 might leave 1724 * nlinks == 1 to avoid unnecessary block updates). The backend flush then 1725 * needs to actually remove it from the topology. 1726 * 1727 * NOTE: backend flush must still sync and flush the deleted inode to clean 1728 * out related chains. 1729 * 1730 * NOTE: We must clear not only INODE_DELETING, but also INODE_ISUNLINKED 1731 * to prevent the vnode reclaim code from trying to delete it twice. 1732 */ 1733 int 1734 hammer2_inode_chain_des(hammer2_inode_t *ip) 1735 { 1736 int error; 1737 1738 error = 0; 1739 if (ip->flags & HAMMER2_INODE_DELETING) { 1740 hammer2_xop_destroy_t *xop; 1741 1742 atomic_clear_int(&ip->flags, HAMMER2_INODE_DELETING | 1743 HAMMER2_INODE_ISUNLINKED); 1744 xop = hammer2_xop_alloc(ip, HAMMER2_XOP_MODIFYING); 1745 hammer2_xop_start(&xop->head, &hammer2_inode_destroy_desc); 1746 error = hammer2_xop_collect(&xop->head, 0); 1747 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP); 1748 1749 if (error == HAMMER2_ERROR_ENOENT) 1750 error = 0; 1751 if (error) { 1752 kprintf("hammer2: backend unable to " 1753 "delete inode %p %ld\n", ip, ip->meta.inum); 1754 /* XXX return error somehow? */ 1755 } 1756 } 1757 return error; 1758 } 1759 1760 /* 1761 * Flushes the inode's chain and its sub-topology to media. Interlocks 1762 * HAMMER2_INODE_DIRTYDATA by clearing it prior to the flush. Any strategy 1763 * function creating or modifying a chain under this inode will re-set the 1764 * flag. 1765 * 1766 * inode must be locked. 1767 */ 1768 int 1769 hammer2_inode_chain_flush(hammer2_inode_t *ip, int flags) 1770 { 1771 hammer2_xop_fsync_t *xop; 1772 int error; 1773 1774 atomic_clear_int(&ip->flags, HAMMER2_INODE_DIRTYDATA); 1775 xop = hammer2_xop_alloc(ip, HAMMER2_XOP_MODIFYING | flags); 1776 hammer2_xop_start(&xop->head, &hammer2_inode_flush_desc); 1777 error = hammer2_xop_collect(&xop->head, HAMMER2_XOP_COLLECT_WAITALL); 1778 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP); 1779 if (error == HAMMER2_ERROR_ENOENT) 1780 error = 0; 1781 1782 return error; 1783 } 1784