1 /* 2 * Copyright (c) 2004,2013-2017 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * External lock/ref-related vnode functions 37 * 38 * vs_state transition locking requirements: 39 * 40 * INACTIVE -> CACHED|DYING vx_lock(excl) + vi->spin 41 * DYING -> CACHED vx_lock(excl) 42 * ACTIVE -> INACTIVE (none) + v_spin + vi->spin 43 * INACTIVE -> ACTIVE vn_lock(any) + v_spin + vi->spin 44 * CACHED -> ACTIVE vn_lock(any) + v_spin + vi->spin 45 * 46 * NOTE: Switching to/from ACTIVE/INACTIVE requires v_spin and vi->spin, 47 * 48 * Switching into ACTIVE also requires a vref and vnode lock, however 49 * the vnode lock is allowed to be SHARED. 50 * 51 * Switching into a CACHED or DYING state requires an exclusive vnode 52 * lock or vx_lock (which is almost the same thing). 53 */ 54 55 #include <sys/param.h> 56 #include <sys/systm.h> 57 #include <sys/kernel.h> 58 #include <sys/malloc.h> 59 #include <sys/mount.h> 60 #include <sys/proc.h> 61 #include <sys/vnode.h> 62 #include <sys/buf.h> 63 #include <sys/sysctl.h> 64 65 #include <machine/limits.h> 66 67 #include <vm/vm.h> 68 #include <vm/vm_object.h> 69 70 #include <sys/buf2.h> 71 #include <sys/thread2.h> 72 73 #define VACT_MAX 10 74 #define VACT_INC 2 75 76 static void vnode_terminate(struct vnode *vp); 77 78 static MALLOC_DEFINE(M_VNODE, "vnodes", "vnode structures"); 79 80 /* 81 * The vnode free list hold inactive vnodes. Aged inactive vnodes 82 * are inserted prior to the mid point, and otherwise inserted 83 * at the tail. 84 * 85 * The vnode code goes to great lengths to avoid moving vnodes between 86 * lists, but sometimes it is unavoidable. For this situation we try to 87 * avoid lock contention but we do not try very hard to avoid cache line 88 * congestion. A modestly sized hash table is used. 89 */ 90 #define VLIST_PRIME2 123462047LU 91 #define VLIST_XOR (uintptr_t)0xab4582fa8322fb71LLU 92 93 #define VLIST_HASH(vp) (((uintptr_t)vp ^ VLIST_XOR) % \ 94 VLIST_PRIME2 % (unsigned)ncpus) 95 96 static struct vnode_index *vnode_list_hash; 97 98 int activevnodes = 0; 99 SYSCTL_INT(_debug, OID_AUTO, activevnodes, CTLFLAG_RD, 100 &activevnodes, 0, "Number of active nodes"); 101 int cachedvnodes = 0; 102 SYSCTL_INT(_debug, OID_AUTO, cachedvnodes, CTLFLAG_RD, 103 &cachedvnodes, 0, "Number of total cached nodes"); 104 int inactivevnodes = 0; 105 SYSCTL_INT(_debug, OID_AUTO, inactivevnodes, CTLFLAG_RD, 106 &inactivevnodes, 0, "Number of inactive nodes"); 107 static int batchfreevnodes = 5; 108 SYSCTL_INT(_debug, OID_AUTO, batchfreevnodes, CTLFLAG_RW, 109 &batchfreevnodes, 0, "Number of vnodes to free at once"); 110 #ifdef TRACKVNODE 111 static u_long trackvnode; 112 SYSCTL_ULONG(_debug, OID_AUTO, trackvnode, CTLFLAG_RW, 113 &trackvnode, 0, ""); 114 #endif 115 116 /* 117 * Called from vfsinit() 118 */ 119 void 120 vfs_lock_init(void) 121 { 122 int i; 123 124 kmalloc_raise_limit(M_VNODE, 0); /* unlimited */ 125 vnode_list_hash = kmalloc(sizeof(*vnode_list_hash) * ncpus, 126 M_VNODE, M_ZERO | M_WAITOK); 127 for (i = 0; i < ncpus; ++i) { 128 struct vnode_index *vi = &vnode_list_hash[i]; 129 130 TAILQ_INIT(&vi->inactive_list); 131 TAILQ_INIT(&vi->active_list); 132 TAILQ_INSERT_TAIL(&vi->active_list, &vi->active_rover, v_list); 133 spin_init(&vi->spin, "vfslock"); 134 } 135 } 136 137 /* 138 * Misc functions 139 */ 140 static __inline 141 void 142 _vsetflags(struct vnode *vp, int flags) 143 { 144 atomic_set_int(&vp->v_flag, flags); 145 } 146 147 static __inline 148 void 149 _vclrflags(struct vnode *vp, int flags) 150 { 151 atomic_clear_int(&vp->v_flag, flags); 152 } 153 154 void 155 vsetflags(struct vnode *vp, int flags) 156 { 157 _vsetflags(vp, flags); 158 } 159 160 void 161 vclrflags(struct vnode *vp, int flags) 162 { 163 _vclrflags(vp, flags); 164 } 165 166 /* 167 * Place the vnode on the active list. 168 * 169 * Caller must hold vp->v_spin 170 */ 171 static __inline 172 void 173 _vactivate(struct vnode *vp) 174 { 175 struct vnode_index *vi = &vnode_list_hash[VLIST_HASH(vp)]; 176 177 #ifdef TRACKVNODE 178 if ((u_long)vp == trackvnode) 179 kprintf("_vactivate %p %08x\n", vp, vp->v_flag); 180 #endif 181 spin_lock(&vi->spin); 182 183 switch(vp->v_state) { 184 case VS_ACTIVE: 185 spin_unlock(&vi->spin); 186 panic("_vactivate: already active"); 187 /* NOT REACHED */ 188 return; 189 case VS_INACTIVE: 190 TAILQ_REMOVE(&vi->inactive_list, vp, v_list); 191 atomic_add_int(&mycpu->gd_inactivevnodes, -1); 192 break; 193 case VS_CACHED: 194 case VS_DYING: 195 break; 196 } 197 TAILQ_INSERT_TAIL(&vi->active_list, vp, v_list); 198 vp->v_state = VS_ACTIVE; 199 spin_unlock(&vi->spin); 200 atomic_add_int(&mycpu->gd_activevnodes, 1); 201 } 202 203 /* 204 * Put a vnode on the inactive list. 205 * 206 * Caller must hold v_spin 207 */ 208 static __inline 209 void 210 _vinactive(struct vnode *vp) 211 { 212 struct vnode_index *vi = &vnode_list_hash[VLIST_HASH(vp)]; 213 214 #ifdef TRACKVNODE 215 if ((u_long)vp == trackvnode) { 216 kprintf("_vinactive %p %08x\n", vp, vp->v_flag); 217 print_backtrace(-1); 218 } 219 #endif 220 spin_lock(&vi->spin); 221 222 /* 223 * Remove from active list if it is sitting on it 224 */ 225 switch(vp->v_state) { 226 case VS_ACTIVE: 227 TAILQ_REMOVE(&vi->active_list, vp, v_list); 228 atomic_add_int(&mycpu->gd_activevnodes, -1); 229 break; 230 case VS_INACTIVE: 231 spin_unlock(&vi->spin); 232 panic("_vinactive: already inactive"); 233 /* NOT REACHED */ 234 return; 235 case VS_CACHED: 236 case VS_DYING: 237 break; 238 } 239 240 /* 241 * Distinguish between basically dead vnodes, vnodes with cached 242 * data, and vnodes without cached data. A rover will shift the 243 * vnodes around as their cache status is lost. 244 */ 245 if (vp->v_flag & VRECLAIMED) { 246 TAILQ_INSERT_HEAD(&vi->inactive_list, vp, v_list); 247 } else { 248 TAILQ_INSERT_TAIL(&vi->inactive_list, vp, v_list); 249 } 250 vp->v_state = VS_INACTIVE; 251 spin_unlock(&vi->spin); 252 atomic_add_int(&mycpu->gd_inactivevnodes, 1); 253 } 254 255 static __inline 256 void 257 _vinactive_tail(struct vnode *vp) 258 { 259 struct vnode_index *vi = &vnode_list_hash[VLIST_HASH(vp)]; 260 261 spin_lock(&vi->spin); 262 263 /* 264 * Remove from active list if it is sitting on it 265 */ 266 switch(vp->v_state) { 267 case VS_ACTIVE: 268 TAILQ_REMOVE(&vi->active_list, vp, v_list); 269 atomic_add_int(&mycpu->gd_activevnodes, -1); 270 break; 271 case VS_INACTIVE: 272 spin_unlock(&vi->spin); 273 panic("_vinactive_tail: already inactive"); 274 /* NOT REACHED */ 275 return; 276 case VS_CACHED: 277 case VS_DYING: 278 break; 279 } 280 281 TAILQ_INSERT_TAIL(&vi->inactive_list, vp, v_list); 282 vp->v_state = VS_INACTIVE; 283 spin_unlock(&vi->spin); 284 atomic_add_int(&mycpu->gd_inactivevnodes, 1); 285 } 286 287 /* 288 * Add a ref to an active vnode. This function should never be called 289 * with an inactive vnode (use vget() instead), but might be called 290 * with other states. 291 */ 292 void 293 vref(struct vnode *vp) 294 { 295 KASSERT((VREFCNT(vp) > 0 && vp->v_state != VS_INACTIVE), 296 ("vref: bad refcnt %08x %d", vp->v_refcnt, vp->v_state)); 297 atomic_add_int(&vp->v_refcnt, 1); 298 } 299 300 void 301 synchronizevnodecount(void) 302 { 303 int nca = 0; 304 int act = 0; 305 int ina = 0; 306 int i; 307 308 for (i = 0; i < ncpus; ++i) { 309 globaldata_t gd = globaldata_find(i); 310 nca += gd->gd_cachedvnodes; 311 act += gd->gd_activevnodes; 312 ina += gd->gd_inactivevnodes; 313 } 314 cachedvnodes = nca; 315 activevnodes = act; 316 inactivevnodes = ina; 317 } 318 319 /* 320 * Count number of cached vnodes. This is middling expensive so be 321 * careful not to make this call in the critical path. Each cpu tracks 322 * its own accumulator. The individual accumulators must be summed 323 * together to get an accurate value. 324 */ 325 int 326 countcachedvnodes(void) 327 { 328 int i; 329 int n = 0; 330 331 for (i = 0; i < ncpus; ++i) { 332 globaldata_t gd = globaldata_find(i); 333 n += gd->gd_cachedvnodes; 334 } 335 return n; 336 } 337 338 int 339 countcachedandinactivevnodes(void) 340 { 341 int i; 342 int n = 0; 343 344 for (i = 0; i < ncpus; ++i) { 345 globaldata_t gd = globaldata_find(i); 346 n += gd->gd_cachedvnodes + gd->gd_inactivevnodes; 347 } 348 return n; 349 } 350 351 /* 352 * Release a ref on an active or inactive vnode. 353 * 354 * Caller has no other requirements. 355 * 356 * If VREF_FINALIZE is set this will deactivate the vnode on the 1->0 357 * transition, otherwise we leave the vnode in the active list and 358 * do a lockless transition to 0, which is very important for the 359 * critical path. 360 * 361 * (vrele() is not called when a vnode is being destroyed w/kfree) 362 */ 363 void 364 vrele(struct vnode *vp) 365 { 366 for (;;) { 367 int count = vp->v_refcnt; 368 cpu_ccfence(); 369 KKASSERT((count & VREF_MASK) > 0); 370 KKASSERT(vp->v_state == VS_ACTIVE || 371 vp->v_state == VS_INACTIVE); 372 373 /* 374 * 2+ case 375 */ 376 if ((count & VREF_MASK) > 1) { 377 if (atomic_cmpset_int(&vp->v_refcnt, count, count - 1)) 378 break; 379 continue; 380 } 381 382 /* 383 * 1->0 transition case must handle possible finalization. 384 * When finalizing we transition 1->0x40000000. Note that 385 * cachedvnodes is only adjusted on transitions to ->0. 386 * 387 * WARNING! VREF_TERMINATE can be cleared at any point 388 * when the refcnt is non-zero (by vget()) and 389 * the vnode has not been reclaimed. Thus 390 * transitions out of VREF_TERMINATE do not have 391 * to mess with cachedvnodes. 392 */ 393 if (count & VREF_FINALIZE) { 394 vx_lock(vp); 395 if (atomic_cmpset_int(&vp->v_refcnt, 396 count, VREF_TERMINATE)) { 397 vnode_terminate(vp); 398 break; 399 } 400 vx_unlock(vp); 401 } else { 402 if (atomic_cmpset_int(&vp->v_refcnt, count, 0)) { 403 atomic_add_int(&mycpu->gd_cachedvnodes, 1); 404 break; 405 } 406 } 407 /* retry */ 408 } 409 } 410 411 /* 412 * Add an auxiliary data structure reference to the vnode. Auxiliary 413 * references do not change the state of the vnode or prevent deactivation 414 * or reclamation of the vnode, but will prevent the vnode from being 415 * destroyed (kfree()'d). 416 * 417 * WARNING! vhold() must not acquire v_spin. The spinlock may or may not 418 * already be held by the caller. vdrop() will clean up the 419 * free list state. 420 */ 421 void 422 vhold(struct vnode *vp) 423 { 424 atomic_add_int(&vp->v_auxrefs, 1); 425 } 426 427 /* 428 * Remove an auxiliary reference from the vnode. 429 */ 430 void 431 vdrop(struct vnode *vp) 432 { 433 atomic_add_int(&vp->v_auxrefs, -1); 434 } 435 436 /* 437 * This function is called on the 1->0 transition (which is actually 438 * 1->VREF_TERMINATE) when VREF_FINALIZE is set, forcing deactivation 439 * of the vnode. 440 * 441 * Additional vrefs are allowed to race but will not result in a reentrant 442 * call to vnode_terminate() due to refcnt being VREF_TERMINATE. This 443 * prevents additional 1->0 transitions. 444 * 445 * ONLY A VGET() CAN REACTIVATE THE VNODE. 446 * 447 * Caller must hold the VX lock. 448 * 449 * NOTE: v_mount may be NULL due to assigmment to dead_vnode_vops 450 * 451 * NOTE: The vnode may be marked inactive with dirty buffers 452 * or dirty pages in its cached VM object still present. 453 * 454 * NOTE: VS_FREE should not be set on entry (the vnode was expected to 455 * previously be active). We lose control of the vnode the instant 456 * it is placed on the free list. 457 * 458 * The VX lock is required when transitioning to VS_CACHED but is 459 * not sufficient for the vshouldfree() interlocked test or when 460 * transitioning away from VS_CACHED. v_spin is also required for 461 * those cases. 462 */ 463 static 464 void 465 vnode_terminate(struct vnode *vp) 466 { 467 KKASSERT(vp->v_state == VS_ACTIVE); 468 469 if ((vp->v_flag & VINACTIVE) == 0) { 470 _vsetflags(vp, VINACTIVE); 471 if (vp->v_mount) 472 VOP_INACTIVE(vp); 473 } 474 spin_lock(&vp->v_spin); 475 _vinactive(vp); 476 spin_unlock(&vp->v_spin); 477 478 vx_unlock(vp); 479 } 480 481 /**************************************************************** 482 * VX LOCKING FUNCTIONS * 483 **************************************************************** 484 * 485 * These functions lock vnodes for reclamation and deactivation related 486 * activities. The caller must already be holding some sort of reference 487 * on the vnode. 488 */ 489 void 490 vx_lock(struct vnode *vp) 491 { 492 lockmgr(&vp->v_lock, LK_EXCLUSIVE); 493 } 494 495 void 496 vx_unlock(struct vnode *vp) 497 { 498 lockmgr(&vp->v_lock, LK_RELEASE); 499 } 500 501 /**************************************************************** 502 * VNODE ACQUISITION FUNCTIONS * 503 **************************************************************** 504 * 505 * These functions must be used when accessing a vnode that has no 506 * chance of being destroyed in a SMP race. That means the caller will 507 * usually either hold an auxiliary reference (such as the namecache) 508 * or hold some other lock that ensures that the vnode cannot be destroyed. 509 * 510 * These functions are MANDATORY for any code chain accessing a vnode 511 * whos activation state is not known. 512 * 513 * vget() can be called with LK_NOWAIT and will return EBUSY if the 514 * lock cannot be immediately acquired. 515 * 516 * vget()/vput() are used when reactivation is desired. 517 * 518 * vx_get() and vx_put() are used when reactivation is not desired. 519 */ 520 int 521 vget(struct vnode *vp, int flags) 522 { 523 int error; 524 525 /* 526 * A lock type must be passed 527 */ 528 if ((flags & LK_TYPE_MASK) == 0) { 529 panic("vget() called with no lock specified!"); 530 /* NOT REACHED */ 531 } 532 533 /* 534 * Reference the structure and then acquire the lock. 535 * 536 * NOTE: The requested lock might be a shared lock and does 537 * not protect our access to the refcnt or other fields. 538 */ 539 if ((atomic_fetchadd_int(&vp->v_refcnt, 1) & VREF_MASK) == 0) 540 atomic_add_int(&mycpu->gd_cachedvnodes, -1); 541 542 if ((error = vn_lock(vp, flags | LK_FAILRECLAIM)) != 0) { 543 /* 544 * The lock failed, undo and return an error. This will not 545 * normally trigger a termination. 546 */ 547 vrele(vp); 548 } else if (vp->v_flag & VRECLAIMED) { 549 /* 550 * The node is being reclaimed and cannot be reactivated 551 * any more, undo and return ENOENT. 552 */ 553 vn_unlock(vp); 554 vrele(vp); 555 error = ENOENT; 556 } else if (vp->v_state == VS_ACTIVE) { 557 /* 558 * A VS_ACTIVE vnode coupled with the fact that we have 559 * a vnode lock (even if shared) prevents v_state from 560 * changing. Since the vnode is not in a VRECLAIMED state, 561 * we can safely clear VINACTIVE. 562 * 563 * NOTE! Multiple threads may clear VINACTIVE if this is 564 * shared lock. This race is allowed. 565 */ 566 _vclrflags(vp, VINACTIVE); /* SMP race ok */ 567 vp->v_act += VACT_INC; 568 if (vp->v_act > VACT_MAX) /* SMP race ok */ 569 vp->v_act = VACT_MAX; 570 error = 0; 571 } else { 572 /* 573 * If the vnode is not VS_ACTIVE it must be reactivated 574 * in addition to clearing VINACTIVE. An exclusive spin_lock 575 * is needed to manipulate the vnode's list. 576 * 577 * Because the lockmgr lock might be shared, we might race 578 * another reactivation, which we handle. In this situation, 579 * however, the refcnt prevents other v_state races. 580 * 581 * As with above, clearing VINACTIVE is allowed to race other 582 * clearings of VINACTIVE. 583 * 584 * VREF_TERMINATE and VREF_FINALIZE can only be cleared when 585 * the refcnt is non-zero and the vnode has not been 586 * reclaimed. This also means that the transitions do 587 * not affect cachedvnodes. 588 */ 589 _vclrflags(vp, VINACTIVE); 590 vp->v_act += VACT_INC; 591 if (vp->v_act > VACT_MAX) /* SMP race ok */ 592 vp->v_act = VACT_MAX; 593 spin_lock(&vp->v_spin); 594 595 switch(vp->v_state) { 596 case VS_INACTIVE: 597 _vactivate(vp); 598 atomic_clear_int(&vp->v_refcnt, VREF_TERMINATE | 599 VREF_FINALIZE); 600 spin_unlock(&vp->v_spin); 601 break; 602 case VS_CACHED: 603 _vactivate(vp); 604 atomic_clear_int(&vp->v_refcnt, VREF_TERMINATE | 605 VREF_FINALIZE); 606 spin_unlock(&vp->v_spin); 607 break; 608 case VS_ACTIVE: 609 atomic_clear_int(&vp->v_refcnt, VREF_FINALIZE); 610 spin_unlock(&vp->v_spin); 611 break; 612 case VS_DYING: 613 spin_unlock(&vp->v_spin); 614 panic("Impossible VS_DYING state"); 615 break; 616 } 617 error = 0; 618 } 619 return(error); 620 } 621 622 #ifdef DEBUG_VPUT 623 624 void 625 debug_vput(struct vnode *vp, const char *filename, int line) 626 { 627 kprintf("vput(%p) %s:%d\n", vp, filename, line); 628 vn_unlock(vp); 629 vrele(vp); 630 } 631 632 #else 633 634 void 635 vput(struct vnode *vp) 636 { 637 vn_unlock(vp); 638 vrele(vp); 639 } 640 641 #endif 642 643 /* 644 * Acquire the vnode lock unguarded. 645 * 646 * The non-blocking version also uses a slightly different mechanic. 647 * This function will explicitly fail not only if it cannot acquire 648 * the lock normally, but also if the caller already holds a lock. 649 * 650 * The adjusted mechanic is used to close a loophole where complex 651 * VOP_RECLAIM code can circle around recursively and allocate the 652 * same vnode it is trying to destroy from the freelist. 653 * 654 * Any filesystem (aka UFS) which puts LK_CANRECURSE in lk_flags can 655 * cause the incorrect behavior to occur. If not for that lockmgr() 656 * would do the right thing. 657 * 658 * XXX The vx_*() locks should use auxrefs, not the main reference counter. 659 */ 660 void 661 vx_get(struct vnode *vp) 662 { 663 if ((atomic_fetchadd_int(&vp->v_refcnt, 1) & VREF_MASK) == 0) 664 atomic_add_int(&mycpu->gd_cachedvnodes, -1); 665 lockmgr(&vp->v_lock, LK_EXCLUSIVE); 666 } 667 668 int 669 vx_get_nonblock(struct vnode *vp) 670 { 671 int error; 672 673 if (lockinuse(&vp->v_lock)) 674 return(EBUSY); 675 error = lockmgr(&vp->v_lock, LK_EXCLUSIVE | LK_NOWAIT); 676 if (error == 0) { 677 if ((atomic_fetchadd_int(&vp->v_refcnt, 1) & VREF_MASK) == 0) 678 atomic_add_int(&mycpu->gd_cachedvnodes, -1); 679 } 680 return(error); 681 } 682 683 /* 684 * Release a VX lock that also held a ref on the vnode. vrele() will handle 685 * any needed state transitions. 686 * 687 * However, filesystems use this function to get rid of unwanted new vnodes 688 * so try to get the vnode on the correct queue in that case. 689 */ 690 void 691 vx_put(struct vnode *vp) 692 { 693 if (vp->v_type == VNON || vp->v_type == VBAD) 694 atomic_set_int(&vp->v_refcnt, VREF_FINALIZE); 695 lockmgr(&vp->v_lock, LK_RELEASE); 696 vrele(vp); 697 } 698 699 /* 700 * Try to reuse a vnode from the free list. This function is somewhat 701 * advisory in that NULL can be returned as a normal case, even if free 702 * vnodes are present. 703 * 704 * The scan is limited because it can result in excessive CPU use during 705 * periods of extreme vnode use. 706 * 707 * NOTE: The returned vnode is not completely initialized. 708 */ 709 static 710 struct vnode * 711 cleanfreevnode(int maxcount) 712 { 713 struct vnode_index *vi; 714 struct vnode *vp; 715 int count; 716 int trigger = (long)vmstats.v_page_count / (activevnodes * 2 + 1); 717 int ri; 718 int cpu_count; 719 720 /* 721 * Try to deactivate some vnodes cached on the active list. 722 */ 723 if (countcachedvnodes() < inactivevnodes) 724 goto skip; 725 726 ri = vnode_list_hash[mycpu->gd_cpuid].deac_rover + 1; 727 728 for (count = 0; count < maxcount * 2; ++count, ++ri) { 729 vi = &vnode_list_hash[((unsigned)ri >> 4) % ncpus]; 730 731 spin_lock(&vi->spin); 732 733 vp = TAILQ_NEXT(&vi->active_rover, v_list); 734 TAILQ_REMOVE(&vi->active_list, &vi->active_rover, v_list); 735 if (vp == NULL) { 736 TAILQ_INSERT_HEAD(&vi->active_list, 737 &vi->active_rover, v_list); 738 } else { 739 TAILQ_INSERT_AFTER(&vi->active_list, vp, 740 &vi->active_rover, v_list); 741 } 742 if (vp == NULL) { 743 spin_unlock(&vi->spin); 744 continue; 745 } 746 if ((vp->v_refcnt & VREF_MASK) != 0) { 747 spin_unlock(&vi->spin); 748 vp->v_act += VACT_INC; 749 if (vp->v_act > VACT_MAX) /* SMP race ok */ 750 vp->v_act = VACT_MAX; 751 continue; 752 } 753 754 /* 755 * decrement by less if the vnode's object has a lot of 756 * VM pages. XXX possible SMP races. 757 */ 758 if (vp->v_act > 0) { 759 vm_object_t obj; 760 if ((obj = vp->v_object) != NULL && 761 obj->resident_page_count >= trigger) { 762 vp->v_act -= 1; 763 } else { 764 vp->v_act -= VACT_INC; 765 } 766 if (vp->v_act < 0) 767 vp->v_act = 0; 768 spin_unlock(&vi->spin); 769 continue; 770 } 771 772 /* 773 * Try to deactivate the vnode. 774 */ 775 if ((atomic_fetchadd_int(&vp->v_refcnt, 1) & VREF_MASK) == 0) 776 atomic_add_int(&mycpu->gd_cachedvnodes, -1); 777 atomic_set_int(&vp->v_refcnt, VREF_FINALIZE); 778 779 spin_unlock(&vi->spin); 780 vrele(vp); 781 } 782 783 vnode_list_hash[mycpu->gd_cpuid].deac_rover = ri; 784 785 skip: 786 /* 787 * Loop trying to lock the first vnode on the free list. 788 * Cycle if we can't. 789 */ 790 cpu_count = ncpus; 791 ri = vnode_list_hash[mycpu->gd_cpuid].free_rover + 1; 792 793 for (count = 0; count < maxcount; ++count, ++ri) { 794 vi = &vnode_list_hash[((unsigned)ri >> 4) % ncpus]; 795 796 spin_lock(&vi->spin); 797 798 vp = TAILQ_FIRST(&vi->inactive_list); 799 if (vp == NULL) { 800 spin_unlock(&vi->spin); 801 if (--cpu_count == 0) 802 break; 803 ri = (ri + 16) & ~15; 804 --ri; 805 continue; 806 } 807 808 /* 809 * non-blocking vx_get will also ref the vnode on success. 810 */ 811 if (vx_get_nonblock(vp)) { 812 KKASSERT(vp->v_state == VS_INACTIVE); 813 TAILQ_REMOVE(&vi->inactive_list, vp, v_list); 814 TAILQ_INSERT_TAIL(&vi->inactive_list, vp, v_list); 815 spin_unlock(&vi->spin); 816 continue; 817 } 818 819 /* 820 * Because we are holding vfs_spin the vnode should currently 821 * be inactive and VREF_TERMINATE should still be set. 822 * 823 * Once vfs_spin is released the vnode's state should remain 824 * unmodified due to both the lock and ref on it. 825 */ 826 KKASSERT(vp->v_state == VS_INACTIVE); 827 spin_unlock(&vi->spin); 828 #ifdef TRACKVNODE 829 if ((u_long)vp == trackvnode) 830 kprintf("cleanfreevnode %p %08x\n", vp, vp->v_flag); 831 #endif 832 833 /* 834 * Do not reclaim/reuse a vnode while auxillary refs exists. 835 * This includes namecache refs due to a related ncp being 836 * locked or having children, a VM object association, or 837 * other hold users. 838 * 839 * Do not reclaim/reuse a vnode if someone else has a real 840 * ref on it. This can occur if a filesystem temporarily 841 * releases the vnode lock during VOP_RECLAIM. 842 */ 843 if (vp->v_auxrefs || 844 (vp->v_refcnt & ~VREF_FINALIZE) != VREF_TERMINATE + 1) { 845 failed: 846 if (vp->v_state == VS_INACTIVE) { 847 spin_lock(&vi->spin); 848 if (vp->v_state == VS_INACTIVE) { 849 TAILQ_REMOVE(&vi->inactive_list, 850 vp, v_list); 851 TAILQ_INSERT_TAIL(&vi->inactive_list, 852 vp, v_list); 853 } 854 spin_unlock(&vi->spin); 855 } 856 vx_put(vp); 857 continue; 858 } 859 860 /* 861 * VINACTIVE and VREF_TERMINATE are expected to both be set 862 * for vnodes pulled from the inactive list, and cannot be 863 * changed while we hold the vx lock. 864 * 865 * Try to reclaim the vnode. 866 */ 867 KKASSERT(vp->v_flag & VINACTIVE); 868 KKASSERT(vp->v_refcnt & VREF_TERMINATE); 869 870 if ((vp->v_flag & VRECLAIMED) == 0) { 871 if (cache_inval_vp_nonblock(vp)) 872 goto failed; 873 vgone_vxlocked(vp); 874 /* vnode is still VX locked */ 875 } 876 877 /* 878 * At this point if there are no other refs or auxrefs on 879 * the vnode with the inactive list locked, and we remove 880 * the vnode from the inactive list, it should not be 881 * possible for anyone else to access the vnode any more. 882 * 883 * Since the vnode is in a VRECLAIMED state, no new 884 * namecache associations could have been made and the 885 * vnode should have already been removed from its mountlist. 886 * 887 * Since we hold a VX lock on the vnode it cannot have been 888 * reactivated (moved out of the inactive list). 889 */ 890 KKASSERT(TAILQ_EMPTY(&vp->v_namecache)); 891 spin_lock(&vi->spin); 892 if (vp->v_auxrefs || 893 (vp->v_refcnt & ~VREF_FINALIZE) != VREF_TERMINATE + 1) { 894 spin_unlock(&vi->spin); 895 goto failed; 896 } 897 KKASSERT(vp->v_state == VS_INACTIVE); 898 TAILQ_REMOVE(&vi->inactive_list, vp, v_list); 899 atomic_add_int(&mycpu->gd_inactivevnodes, -1); 900 vp->v_state = VS_DYING; 901 spin_unlock(&vi->spin); 902 903 /* 904 * Nothing should have been able to access this vp. Only 905 * our ref should remain now. 906 */ 907 atomic_clear_int(&vp->v_refcnt, VREF_TERMINATE|VREF_FINALIZE); 908 KASSERT(vp->v_refcnt == 1, 909 ("vp %p badrefs %08x", vp, vp->v_refcnt)); 910 911 /* 912 * Return a VX locked vnode suitable for reuse. 913 */ 914 vnode_list_hash[mycpu->gd_cpuid].free_rover = ri; 915 return(vp); 916 } 917 vnode_list_hash[mycpu->gd_cpuid].free_rover = ri; 918 return(NULL); 919 } 920 921 /* 922 * Obtain a new vnode. The returned vnode is VX locked & vrefd. 923 * 924 * All new vnodes set the VAGE flags. An open() of the vnode will 925 * decrement the (2-bit) flags. Vnodes which are opened several times 926 * are thus retained in the cache over vnodes which are merely stat()d. 927 * 928 * We attempt to reuse an already-recycled vnode from our pcpu inactive 929 * queue first, and allocate otherwise. Attempting to recycle inactive 930 * vnodes here can lead to numerous deadlocks, particularly with 931 * softupdates. 932 */ 933 struct vnode * 934 allocvnode(int lktimeout, int lkflags) 935 { 936 struct vnode *vp; 937 struct vnode_index *vi; 938 939 /* 940 * lktimeout only applies when LK_TIMELOCK is used, and only 941 * the pageout daemon uses it. The timeout may not be zero 942 * or the pageout daemon can deadlock in low-VM situations. 943 */ 944 if (lktimeout == 0) 945 lktimeout = hz / 10; 946 947 /* 948 * Do not flag for synchronous recyclement unless there are enough 949 * freeable vnodes to recycle and the number of vnodes has 950 * significantly exceeded our target. We want the normal vnlru 951 * process to handle the cleaning (at 9/10's) before we are forced 952 * to flag it here at 11/10's for userexit path processing. 953 */ 954 if (numvnodes >= maxvnodes * 11 / 10 && 955 cachedvnodes + inactivevnodes >= maxvnodes * 5 / 10) { 956 struct thread *td = curthread; 957 if (td->td_lwp) 958 atomic_set_int(&td->td_lwp->lwp_mpflags, LWP_MP_VNLRU); 959 } 960 961 /* 962 * Try to trivially reuse a reclaimed vnode from the head of the 963 * inactive list for this cpu. Any vnode cycling which occurs 964 * which terminates the vnode will cause it to be returned to the 965 * same pcpu structure (e.g. unlink calls). 966 */ 967 vi = &vnode_list_hash[mycpuid]; 968 spin_lock(&vi->spin); 969 970 vp = TAILQ_FIRST(&vi->inactive_list); 971 if (vp && (vp->v_flag & VRECLAIMED)) { 972 /* 973 * non-blocking vx_get will also ref the vnode on success. 974 */ 975 if (vx_get_nonblock(vp)) { 976 KKASSERT(vp->v_state == VS_INACTIVE); 977 TAILQ_REMOVE(&vi->inactive_list, vp, v_list); 978 TAILQ_INSERT_TAIL(&vi->inactive_list, vp, v_list); 979 spin_unlock(&vi->spin); 980 goto slower; 981 } 982 983 /* 984 * Because we are holding vfs_spin the vnode should currently 985 * be inactive and VREF_TERMINATE should still be set. 986 * 987 * Once vfs_spin is released the vnode's state should remain 988 * unmodified due to both the lock and ref on it. 989 */ 990 KKASSERT(vp->v_state == VS_INACTIVE); 991 #ifdef TRACKVNODE 992 if ((u_long)vp == trackvnode) 993 kprintf("allocvnode %p %08x\n", vp, vp->v_flag); 994 #endif 995 996 /* 997 * Do not reclaim/reuse a vnode while auxillary refs exists. 998 * This includes namecache refs due to a related ncp being 999 * locked or having children, a VM object association, or 1000 * other hold users. 1001 * 1002 * Do not reclaim/reuse a vnode if someone else has a real 1003 * ref on it. This can occur if a filesystem temporarily 1004 * releases the vnode lock during VOP_RECLAIM. 1005 */ 1006 if (vp->v_auxrefs || 1007 (vp->v_refcnt & ~VREF_FINALIZE) != VREF_TERMINATE + 1) { 1008 if (vp->v_state == VS_INACTIVE) { 1009 if (vp->v_state == VS_INACTIVE) { 1010 TAILQ_REMOVE(&vi->inactive_list, 1011 vp, v_list); 1012 TAILQ_INSERT_TAIL(&vi->inactive_list, 1013 vp, v_list); 1014 } 1015 } 1016 spin_unlock(&vi->spin); 1017 vx_put(vp); 1018 goto slower; 1019 } 1020 1021 /* 1022 * VINACTIVE and VREF_TERMINATE are expected to both be set 1023 * for vnodes pulled from the inactive list, and cannot be 1024 * changed while we hold the vx lock. 1025 * 1026 * Try to reclaim the vnode. 1027 */ 1028 KKASSERT(vp->v_flag & VINACTIVE); 1029 KKASSERT(vp->v_refcnt & VREF_TERMINATE); 1030 1031 if ((vp->v_flag & VRECLAIMED) == 0) { 1032 spin_unlock(&vi->spin); 1033 vx_put(vp); 1034 goto slower; 1035 } 1036 1037 /* 1038 * At this point if there are no other refs or auxrefs on 1039 * the vnode with the inactive list locked, and we remove 1040 * the vnode from the inactive list, it should not be 1041 * possible for anyone else to access the vnode any more. 1042 * 1043 * Since the vnode is in a VRECLAIMED state, no new 1044 * namecache associations could have been made and the 1045 * vnode should have already been removed from its mountlist. 1046 * 1047 * Since we hold a VX lock on the vnode it cannot have been 1048 * reactivated (moved out of the inactive list). 1049 */ 1050 KKASSERT(TAILQ_EMPTY(&vp->v_namecache)); 1051 KKASSERT(vp->v_state == VS_INACTIVE); 1052 TAILQ_REMOVE(&vi->inactive_list, vp, v_list); 1053 atomic_add_int(&mycpu->gd_inactivevnodes, -1); 1054 vp->v_state = VS_DYING; 1055 spin_unlock(&vi->spin); 1056 1057 /* 1058 * Nothing should have been able to access this vp. Only 1059 * our ref should remain now. 1060 * 1061 * At this point we can kfree() the vnode if we want to. 1062 * Instead, we reuse it for the allocation. 1063 */ 1064 atomic_clear_int(&vp->v_refcnt, VREF_TERMINATE|VREF_FINALIZE); 1065 KASSERT(vp->v_refcnt == 1, 1066 ("vp %p badrefs %08x", vp, vp->v_refcnt)); 1067 bzero(vp, sizeof(*vp)); 1068 } else { 1069 spin_unlock(&vi->spin); 1070 slower: 1071 vp = kmalloc(sizeof(*vp), M_VNODE, M_ZERO | M_WAITOK); 1072 atomic_add_int(&numvnodes, 1); 1073 } 1074 1075 lwkt_token_init(&vp->v_token, "vnode"); 1076 lockinit(&vp->v_lock, "vnode", lktimeout, lkflags); 1077 TAILQ_INIT(&vp->v_namecache); 1078 RB_INIT(&vp->v_rbclean_tree); 1079 RB_INIT(&vp->v_rbdirty_tree); 1080 RB_INIT(&vp->v_rbhash_tree); 1081 spin_init(&vp->v_spin, "allocvnode"); 1082 1083 lockmgr(&vp->v_lock, LK_EXCLUSIVE); 1084 vp->v_refcnt = 1; 1085 vp->v_flag = VAGE0 | VAGE1; 1086 vp->v_pbuf_count = nswbuf_kva / NSWBUF_SPLIT; 1087 1088 KKASSERT(TAILQ_EMPTY(&vp->v_namecache)); 1089 /* exclusive lock still held */ 1090 1091 vp->v_filesize = NOOFFSET; 1092 vp->v_type = VNON; 1093 vp->v_tag = 0; 1094 vp->v_state = VS_CACHED; 1095 _vactivate(vp); 1096 1097 return (vp); 1098 } 1099 1100 /* 1101 * Called after a process has allocated a vnode via allocvnode() 1102 * and we detected that too many vnodes were present. 1103 * 1104 * This function is called just prior to a return to userland if the 1105 * process at some point had to allocate a new vnode during the last 1106 * system call and the vnode count was found to be excessive. 1107 * 1108 * This is a synchronous path that we do not normally want to execute. 1109 * 1110 * Flagged at >= 11/10's, runs if >= 10/10, vnlru runs at 9/10. 1111 * 1112 * WARNING: Sometimes numvnodes can blow out due to children being 1113 * present under directory vnodes in the namecache. For the 1114 * moment use an if() instead of a while() and note that if 1115 * we were to use a while() we would still have to break out 1116 * if freesomevnodes() returned 0. vnlru will also be trying 1117 * hard to free vnodes at the same time (with a lower trigger 1118 * pointer). 1119 */ 1120 void 1121 allocvnode_gc(void) 1122 { 1123 if (numvnodes >= maxvnodes && 1124 countcachedandinactivevnodes() >= maxvnodes * 5 / 10) { 1125 freesomevnodes(batchfreevnodes); 1126 } 1127 } 1128 1129 int 1130 freesomevnodes(int n) 1131 { 1132 struct vnode *vp; 1133 int count = 0; 1134 1135 while (n) { 1136 if ((vp = cleanfreevnode(n)) == NULL) 1137 break; 1138 vx_unlock(vp); 1139 --n; 1140 ++count; 1141 kfree(vp, M_VNODE); 1142 atomic_add_int(&numvnodes, -1); 1143 } 1144 return(count); 1145 } 1146