1 /* 2 * Copyright (c) 2005 Jeffrey M. Hsu. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Jeffrey M. Hsu. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of The DragonFly Project nor the names of its 16 * contributors may be used to endorse or promote products derived 17 * from this software without specific, prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 22 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 23 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 25 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 27 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 29 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * $DragonFly: src/sys/kern/kern_objcache.c,v 1.18 2007/04/30 07:18:53 dillon Exp $ 33 */ 34 35 #include <sys/param.h> 36 #include <sys/kernel.h> 37 #include <sys/systm.h> 38 #include <sys/callout.h> 39 #include <sys/globaldata.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/objcache.h> 43 #include <sys/spinlock.h> 44 #include <sys/thread.h> 45 #include <sys/thread2.h> 46 #include <sys/spinlock2.h> 47 48 static MALLOC_DEFINE(M_OBJCACHE, "objcache", "Object Cache"); 49 static MALLOC_DEFINE(M_OBJMAG, "objcache magazine", "Object Cache Magazine"); 50 51 #define INITIAL_MAG_CAPACITY 256 52 53 struct magazine { 54 int rounds; 55 int capacity; 56 int cleaning; 57 SLIST_ENTRY(magazine) nextmagazine; 58 void *objects[]; 59 }; 60 61 SLIST_HEAD(magazinelist, magazine); 62 63 /* 64 * per-cluster cache of magazines 65 * 66 * All fields in this structure are protected by the spinlock. 67 */ 68 struct magazinedepot { 69 /* 70 * The per-cpu object caches only exchanges completely full or 71 * completely empty magazines with the depot layer, so only have 72 * to cache these two types of magazines. 73 */ 74 struct magazinelist fullmagazines; 75 struct magazinelist emptymagazines; 76 int magcapacity; 77 78 /* protect this structure */ 79 struct spinlock spin; 80 81 /* magazines not yet allocated towards limit */ 82 int unallocated_objects; 83 84 /* infrequently used fields */ 85 int waiting; /* waiting for another cpu to 86 * return a full magazine to 87 * the depot */ 88 int contested; /* depot contention count */ 89 }; 90 91 /* 92 * per-cpu object cache 93 * All fields in this structure are protected by crit_enter(). 94 */ 95 struct percpu_objcache { 96 struct magazine *loaded_magazine; /* active magazine */ 97 struct magazine *previous_magazine; /* backup magazine */ 98 99 /* statistics */ 100 int gets_cumulative; /* total calls to get */ 101 int gets_null; /* objcache_get returned NULL */ 102 int puts_cumulative; /* total calls to put */ 103 int puts_othercluster; /* returned to other cluster */ 104 105 /* infrequently used fields */ 106 int waiting; /* waiting for a thread on this cpu to 107 * return an obj to the per-cpu cache */ 108 }; 109 110 /* only until we have NUMA cluster topology information XXX */ 111 #define MAXCLUSTERS 1 112 #define myclusterid 0 113 #define CLUSTER_OF(obj) 0 114 115 /* 116 * Two-level object cache consisting of NUMA cluster-level depots of 117 * fully loaded or completely empty magazines and cpu-level caches of 118 * individual objects. 119 */ 120 struct objcache { 121 char *name; 122 123 /* object constructor and destructor from blank storage */ 124 objcache_ctor_fn *ctor; 125 objcache_dtor_fn *dtor; 126 void *private; 127 128 /* interface to underlying allocator */ 129 objcache_alloc_fn *alloc; 130 objcache_free_fn *free; 131 void *allocator_args; 132 133 SLIST_ENTRY(objcache) oc_next; 134 135 /* NUMA-cluster level caches */ 136 struct magazinedepot depot[MAXCLUSTERS]; 137 138 struct percpu_objcache cache_percpu[]; /* per-cpu caches */ 139 }; 140 141 static struct spinlock objcachelist_spin; 142 static SLIST_HEAD(objcachelist, objcache) allobjcaches; 143 144 static struct magazine * 145 mag_alloc(int capacity) 146 { 147 struct magazine *mag; 148 149 mag = kmalloc(__offsetof(struct magazine, objects[capacity]), 150 M_OBJMAG, M_INTWAIT | M_ZERO); 151 mag->capacity = capacity; 152 mag->rounds = 0; 153 mag->cleaning = 0; 154 return (mag); 155 } 156 157 /* 158 * Utility routine for objects that don't require any de-construction. 159 */ 160 161 static void 162 null_dtor(void *obj, void *private) 163 { 164 /* do nothing */ 165 } 166 167 static boolean_t 168 null_ctor(void *obj, void *private, int ocflags) 169 { 170 return TRUE; 171 } 172 173 /* 174 * Create an object cache. 175 */ 176 struct objcache * 177 objcache_create(const char *name, int cluster_limit, int mag_capacity, 178 objcache_ctor_fn *ctor, objcache_dtor_fn *dtor, void *private, 179 objcache_alloc_fn *alloc, objcache_free_fn *free, 180 void *allocator_args) 181 { 182 struct objcache *oc; 183 struct magazinedepot *depot; 184 int cpuid; 185 186 /* allocate object cache structure */ 187 oc = kmalloc(__offsetof(struct objcache, cache_percpu[ncpus]), 188 M_OBJCACHE, M_WAITOK | M_ZERO); 189 oc->name = kstrdup(name, M_TEMP); 190 oc->ctor = ctor ? ctor : null_ctor; 191 oc->dtor = dtor ? dtor : null_dtor; 192 oc->private = private; 193 oc->free = free; 194 oc->allocator_args = allocator_args; 195 196 /* initialize depots */ 197 depot = &oc->depot[0]; 198 199 spin_init(&depot->spin); 200 SLIST_INIT(&depot->fullmagazines); 201 SLIST_INIT(&depot->emptymagazines); 202 203 if (mag_capacity == 0) 204 mag_capacity = INITIAL_MAG_CAPACITY; 205 depot->magcapacity = mag_capacity; 206 207 /* 208 * The cluster_limit must be sufficient to have three magazines per 209 * cpu. 210 */ 211 if (cluster_limit == 0) { 212 depot->unallocated_objects = -1; 213 } else { 214 if (cluster_limit < mag_capacity * ncpus * 3) 215 cluster_limit = mag_capacity * ncpus * 3; 216 depot->unallocated_objects = cluster_limit; 217 } 218 oc->alloc = alloc; 219 220 /* initialize per-cpu caches */ 221 for (cpuid = 0; cpuid < ncpus; cpuid++) { 222 struct percpu_objcache *cache_percpu = &oc->cache_percpu[cpuid]; 223 224 cache_percpu->loaded_magazine = mag_alloc(mag_capacity); 225 cache_percpu->previous_magazine = mag_alloc(mag_capacity); 226 } 227 spin_lock_wr(&objcachelist_spin); 228 SLIST_INSERT_HEAD(&allobjcaches, oc, oc_next); 229 spin_unlock_wr(&objcachelist_spin); 230 231 return (oc); 232 } 233 234 struct objcache * 235 objcache_create_simple(malloc_type_t mtype, size_t objsize) 236 { 237 struct objcache_malloc_args *margs; 238 struct objcache *oc; 239 240 margs = kmalloc(sizeof(*margs), M_OBJCACHE, M_WAITOK|M_ZERO); 241 margs->objsize = objsize; 242 margs->mtype = mtype; 243 oc = objcache_create(mtype->ks_shortdesc, 0, 0, 244 NULL, NULL, NULL, 245 objcache_malloc_alloc, objcache_malloc_free, 246 margs); 247 return (oc); 248 } 249 250 struct objcache * 251 objcache_create_mbacked(malloc_type_t mtype, size_t objsize, 252 int cluster_limit, int mag_capacity, 253 objcache_ctor_fn *ctor, objcache_dtor_fn *dtor, 254 void *private) 255 { 256 struct objcache_malloc_args *margs; 257 struct objcache *oc; 258 259 margs = kmalloc(sizeof(*margs), M_OBJCACHE, M_WAITOK|M_ZERO); 260 margs->objsize = objsize; 261 margs->mtype = mtype; 262 oc = objcache_create(mtype->ks_shortdesc, 263 cluster_limit, mag_capacity, 264 ctor, dtor, private, 265 objcache_malloc_alloc, objcache_malloc_free, 266 margs); 267 return(oc); 268 } 269 270 271 #define MAGAZINE_EMPTY(mag) (mag->rounds == 0) 272 #define MAGAZINE_NOTEMPTY(mag) (mag->rounds != 0) 273 #define MAGAZINE_FULL(mag) (mag->rounds == mag->capacity) 274 275 #define swap(x, y) ({ struct magazine *t = x; x = y; y = t; }) 276 277 /* 278 * Get an object from the object cache. 279 * 280 * WARNING! ocflags are only used when we have to go to the underlying 281 * allocator, so we cannot depend on flags such as M_ZERO. 282 */ 283 void * 284 objcache_get(struct objcache *oc, int ocflags) 285 { 286 struct percpu_objcache *cpucache = &oc->cache_percpu[mycpuid]; 287 struct magazine *loadedmag; 288 struct magazine *emptymag; 289 void *obj; 290 struct magazinedepot *depot; 291 292 KKASSERT((ocflags & M_ZERO) == 0); 293 crit_enter(); 294 ++cpucache->gets_cumulative; 295 296 retry: 297 /* 298 * Loaded magazine has an object. This is the hot path. 299 * It is lock-free and uses a critical section to block 300 * out interrupt handlers on the same processor. 301 */ 302 loadedmag = cpucache->loaded_magazine; 303 if (MAGAZINE_NOTEMPTY(loadedmag)) { 304 obj = loadedmag->objects[--loadedmag->rounds]; 305 crit_exit(); 306 return (obj); 307 } 308 309 /* Previous magazine has an object. */ 310 if (MAGAZINE_NOTEMPTY(cpucache->previous_magazine)) { 311 KKASSERT(cpucache->previous_magazine->cleaning + 312 cpucache->loaded_magazine->cleaning == 0); 313 swap(cpucache->loaded_magazine, cpucache->previous_magazine); 314 loadedmag = cpucache->loaded_magazine; 315 obj = loadedmag->objects[--loadedmag->rounds]; 316 crit_exit(); 317 return (obj); 318 } 319 320 /* 321 * Both magazines empty. Get a full magazine from the depot and 322 * move one of the empty ones to the depot. 323 * 324 * Obtain the depot spinlock. 325 * 326 * NOTE: Beyond this point, M_* flags are handled via oc->alloc() 327 */ 328 depot = &oc->depot[myclusterid]; 329 spin_lock_wr(&depot->spin); 330 331 /* 332 * Recheck the cpucache after obtaining the depot spinlock. This 333 * shouldn't be necessary now but don't take any chances. 334 */ 335 if (MAGAZINE_NOTEMPTY(cpucache->loaded_magazine) || 336 MAGAZINE_NOTEMPTY(cpucache->previous_magazine) 337 ) { 338 spin_unlock_wr(&depot->spin); 339 goto retry; 340 } 341 342 /* Check if depot has a full magazine. */ 343 if (!SLIST_EMPTY(&depot->fullmagazines)) { 344 emptymag = cpucache->previous_magazine; 345 cpucache->previous_magazine = cpucache->loaded_magazine; 346 cpucache->loaded_magazine = SLIST_FIRST(&depot->fullmagazines); 347 SLIST_REMOVE_HEAD(&depot->fullmagazines, nextmagazine); 348 349 /* 350 * Return emptymag to the depot. 351 */ 352 KKASSERT(MAGAZINE_EMPTY(emptymag)); 353 SLIST_INSERT_HEAD(&depot->emptymagazines, 354 emptymag, nextmagazine); 355 spin_unlock_wr(&depot->spin); 356 goto retry; 357 } 358 359 /* 360 * The depot does not have any non-empty magazines. If we have 361 * not hit our object limit we can allocate a new object using 362 * the back-end allocator. 363 * 364 * note: unallocated_objects can be initialized to -1, which has 365 * the effect of removing any allocation limits. 366 */ 367 if (depot->unallocated_objects) { 368 --depot->unallocated_objects; 369 spin_unlock_wr(&depot->spin); 370 crit_exit(); 371 372 obj = oc->alloc(oc->allocator_args, ocflags); 373 if (obj) { 374 if (oc->ctor(obj, oc->private, ocflags)) 375 return (obj); 376 oc->free(obj, oc->allocator_args); 377 spin_lock_wr(&depot->spin); 378 ++depot->unallocated_objects; 379 spin_unlock_wr(&depot->spin); 380 if (depot->waiting) 381 wakeup(depot); 382 obj = NULL; 383 } 384 if (obj == NULL) { 385 crit_enter(); 386 /* 387 * makes debugging easier when gets_cumulative does 388 * not include gets_null. 389 */ 390 ++cpucache->gets_null; 391 --cpucache->gets_cumulative; 392 crit_exit(); 393 } 394 return(obj); 395 } 396 397 /* 398 * Otherwise block if allowed to. 399 */ 400 if ((ocflags & (M_WAITOK|M_NULLOK)) == M_WAITOK) { 401 ++cpucache->waiting; 402 ++depot->waiting; 403 msleep(depot, &depot->spin, 0, "objcache_get", 0); 404 --cpucache->waiting; 405 --depot->waiting; 406 spin_unlock_wr(&depot->spin); 407 goto retry; 408 } 409 410 /* 411 * Otherwise fail 412 */ 413 ++cpucache->gets_null; 414 --cpucache->gets_cumulative; 415 crit_exit(); 416 spin_unlock_wr(&depot->spin); 417 return (NULL); 418 } 419 420 /* 421 * Wrapper for malloc allocation routines. 422 */ 423 void * 424 objcache_malloc_alloc(void *allocator_args, int ocflags) 425 { 426 struct objcache_malloc_args *alloc_args = allocator_args; 427 428 return (kmalloc(alloc_args->objsize, alloc_args->mtype, 429 ocflags & OC_MFLAGS)); 430 } 431 432 void 433 objcache_malloc_free(void *obj, void *allocator_args) 434 { 435 struct objcache_malloc_args *alloc_args = allocator_args; 436 437 kfree(obj, alloc_args->mtype); 438 } 439 440 /* 441 * Wrapper for allocation policies that pre-allocate at initialization time 442 * and don't do run-time allocation. 443 */ 444 void * 445 objcache_nop_alloc(void *allocator_args, int ocflags) 446 { 447 return (NULL); 448 } 449 450 void 451 objcache_nop_free(void *obj, void *allocator_args) 452 { 453 } 454 455 /* 456 * Return an object to the object cache. 457 */ 458 void 459 objcache_put(struct objcache *oc, void *obj) 460 { 461 struct percpu_objcache *cpucache = &oc->cache_percpu[mycpuid]; 462 struct magazine *loadedmag; 463 struct magazinedepot *depot; 464 465 crit_enter(); 466 ++cpucache->puts_cumulative; 467 468 if (CLUSTER_OF(obj) != myclusterid) { 469 #ifdef notyet 470 /* use lazy IPI to send object to owning cluster XXX todo */ 471 ++cpucache->puts_othercluster; 472 crit_exit(); 473 return; 474 #endif 475 } 476 477 retry: 478 /* 479 * Free slot available in loaded magazine. This is the hot path. 480 * It is lock-free and uses a critical section to block out interrupt 481 * handlers on the same processor. 482 */ 483 loadedmag = cpucache->loaded_magazine; 484 if (!MAGAZINE_FULL(loadedmag)) { 485 loadedmag->objects[loadedmag->rounds++] = obj; 486 if (cpucache->waiting) 487 wakeup_mycpu(&oc->depot[myclusterid]); 488 crit_exit(); 489 return; 490 } 491 492 /* 493 * Current magazine full, but previous magazine has room. XXX 494 */ 495 if (!MAGAZINE_FULL(cpucache->previous_magazine)) { 496 KKASSERT(cpucache->previous_magazine->cleaning + 497 cpucache->loaded_magazine->cleaning == 0); 498 swap(cpucache->loaded_magazine, cpucache->previous_magazine); 499 loadedmag = cpucache->loaded_magazine; 500 loadedmag->objects[loadedmag->rounds++] = obj; 501 if (cpucache->waiting) 502 wakeup_mycpu(&oc->depot[myclusterid]); 503 crit_exit(); 504 return; 505 } 506 507 /* 508 * Both magazines full. Get an empty magazine from the depot and 509 * move a full loaded magazine to the depot. Even though the 510 * magazine may wind up with space available after we block on 511 * the spinlock, we still cycle it through to avoid the non-optimal 512 * corner-case. 513 * 514 * Obtain the depot spinlock. 515 */ 516 depot = &oc->depot[myclusterid]; 517 spin_lock_wr(&depot->spin); 518 519 /* 520 * If an empty magazine is available in the depot, cycle it 521 * through and retry. 522 */ 523 if (!SLIST_EMPTY(&depot->emptymagazines)) { 524 KKASSERT(cpucache->previous_magazine->cleaning + 525 cpucache->loaded_magazine->cleaning == 0); 526 loadedmag = cpucache->previous_magazine; 527 cpucache->previous_magazine = cpucache->loaded_magazine; 528 cpucache->loaded_magazine = SLIST_FIRST(&depot->emptymagazines); 529 SLIST_REMOVE_HEAD(&depot->emptymagazines, nextmagazine); 530 531 /* 532 * Return loadedmag to the depot. Due to blocking it may 533 * not be entirely full and could even be empty. 534 */ 535 if (MAGAZINE_EMPTY(loadedmag)) { 536 SLIST_INSERT_HEAD(&depot->emptymagazines, 537 loadedmag, nextmagazine); 538 spin_unlock_wr(&depot->spin); 539 } else { 540 SLIST_INSERT_HEAD(&depot->fullmagazines, 541 loadedmag, nextmagazine); 542 spin_unlock_wr(&depot->spin); 543 if (depot->waiting) 544 wakeup(depot); 545 } 546 goto retry; 547 } 548 549 /* 550 * An empty mag is not available. This is a corner case which can 551 * occur due to cpus holding partially full magazines. Do not try 552 * to allocate a mag, just free the object. 553 */ 554 ++depot->unallocated_objects; 555 spin_unlock_wr(&depot->spin); 556 if (depot->waiting) 557 wakeup(depot); 558 crit_exit(); 559 oc->dtor(obj, oc->private); 560 oc->free(obj, oc->allocator_args); 561 } 562 563 /* 564 * The object is being put back into the cache, but the caller has 565 * indicated that the object is not in any shape to be reused and should 566 * be dtor'd immediately. 567 */ 568 void 569 objcache_dtor(struct objcache *oc, void *obj) 570 { 571 struct magazinedepot *depot; 572 573 depot = &oc->depot[myclusterid]; 574 spin_lock_wr(&depot->spin); 575 ++depot->unallocated_objects; 576 spin_unlock_wr(&depot->spin); 577 if (depot->waiting) 578 wakeup(depot); 579 oc->dtor(obj, oc->private); 580 oc->free(obj, oc->allocator_args); 581 } 582 583 /* 584 * Deallocate all objects in a magazine and free the magazine if requested. 585 * The magazine must already be disassociated from the depot. 586 * 587 * Must be called with a critical section held when called with a per-cpu 588 * magazine. The magazine may be indirectly modified during the loop. 589 * 590 * The number of objects freed is returned. 591 */ 592 static int 593 mag_purge(struct objcache *oc, struct magazine *mag, int freeit) 594 { 595 int count; 596 void *obj; 597 598 count = 0; 599 ++mag->cleaning; 600 while (mag->rounds) { 601 obj = mag->objects[--mag->rounds]; 602 oc->dtor(obj, oc->private); /* MAY BLOCK */ 603 oc->free(obj, oc->allocator_args); /* MAY BLOCK */ 604 ++count; 605 606 /* 607 * Cycle for interrupts 608 */ 609 if ((count & 15) == 0) { 610 crit_exit(); 611 crit_enter(); 612 } 613 } 614 --mag->cleaning; 615 if (freeit) 616 kfree(mag, M_OBJMAG); 617 return(count); 618 } 619 620 /* 621 * Disassociate zero or more magazines from a magazine list associated with 622 * the depot, update the depot, and move the magazines to a temporary 623 * list. 624 * 625 * The caller must check the depot for waiters and wake it up, typically 626 * after disposing of the magazines this function loads onto the temporary 627 * list. 628 */ 629 static void 630 maglist_disassociate(struct magazinedepot *depot, struct magazinelist *maglist, 631 struct magazinelist *tmplist, boolean_t purgeall) 632 { 633 struct magazine *mag; 634 635 while ((mag = SLIST_FIRST(maglist)) != NULL) { 636 SLIST_REMOVE_HEAD(maglist, nextmagazine); 637 SLIST_INSERT_HEAD(tmplist, mag, nextmagazine); 638 depot->unallocated_objects += mag->rounds; 639 } 640 } 641 642 /* 643 * Deallocate all magazines and their contents from the passed temporary 644 * list. The magazines have already been accounted for by their depots. 645 * 646 * The total number of rounds freed is returned. This number is typically 647 * only used to determine whether a wakeup on the depot is needed or not. 648 */ 649 static int 650 maglist_purge(struct objcache *oc, struct magazinelist *maglist) 651 { 652 struct magazine *mag; 653 int count = 0; 654 655 /* 656 * can't use SLIST_FOREACH because blocking releases the depot 657 * spinlock 658 */ 659 while ((mag = SLIST_FIRST(maglist)) != NULL) { 660 SLIST_REMOVE_HEAD(maglist, nextmagazine); 661 count += mag_purge(oc, mag, TRUE); 662 } 663 return(count); 664 } 665 666 /* 667 * De-allocates all magazines on the full and empty magazine lists. 668 * 669 * Because this routine is called with a spinlock held, the magazines 670 * can only be disassociated and moved to a temporary list, not freed. 671 * 672 * The caller is responsible for freeing the magazines. 673 */ 674 static void 675 depot_disassociate(struct magazinedepot *depot, struct magazinelist *tmplist) 676 { 677 maglist_disassociate(depot, &depot->fullmagazines, tmplist, TRUE); 678 maglist_disassociate(depot, &depot->emptymagazines, tmplist, TRUE); 679 } 680 681 #ifdef notneeded 682 void 683 objcache_reclaim(struct objcache *oc) 684 { 685 struct percpu_objcache *cache_percpu = &oc->cache_percpu[myclusterid]; 686 struct magazinedepot *depot = &oc->depot[myclusterid]; 687 struct magazinelist tmplist; 688 int count; 689 690 SLIST_INIT(&tmplist); 691 crit_enter(); 692 count = mag_purge(oc, cache_percpu->loaded_magazine, FALSE); 693 count += mag_purge(oc, cache_percpu->previous_magazine, FALSE); 694 crit_exit(); 695 696 spin_lock_wr(&depot->spin); 697 depot->unallocated_objects += count; 698 depot_disassociate(depot, &tmplist); 699 spin_unlock_wr(&depot->spin); 700 count += maglist_purge(oc, &tmplist); 701 if (count && depot->waiting) 702 wakeup(depot); 703 } 704 #endif 705 706 /* 707 * Try to free up some memory. Return as soon as some free memory is found. 708 * For each object cache on the reclaim list, first try the current per-cpu 709 * cache, then the full magazine depot. 710 */ 711 boolean_t 712 objcache_reclaimlist(struct objcache *oclist[], int nlist, int ocflags) 713 { 714 struct objcache *oc; 715 struct percpu_objcache *cpucache; 716 struct magazinedepot *depot; 717 struct magazinelist tmplist; 718 int i, count; 719 720 SLIST_INIT(&tmplist); 721 722 for (i = 0; i < nlist; i++) { 723 oc = oclist[i]; 724 cpucache = &oc->cache_percpu[mycpuid]; 725 depot = &oc->depot[myclusterid]; 726 727 crit_enter(); 728 count = mag_purge(oc, cpucache->loaded_magazine, FALSE); 729 if (count == 0) 730 count += mag_purge(oc, cpucache->previous_magazine, FALSE); 731 crit_exit(); 732 if (count > 0) { 733 spin_lock_wr(&depot->spin); 734 depot->unallocated_objects += count; 735 spin_unlock_wr(&depot->spin); 736 if (depot->waiting) 737 wakeup(depot); 738 return (TRUE); 739 } 740 spin_lock_wr(&depot->spin); 741 maglist_disassociate(depot, &depot->fullmagazines, 742 &tmplist, FALSE); 743 spin_unlock_wr(&depot->spin); 744 count = maglist_purge(oc, &tmplist); 745 if (count > 0) { 746 if (depot->waiting) 747 wakeup(depot); 748 return (TRUE); 749 } 750 } 751 return (FALSE); 752 } 753 754 /* 755 * Destroy an object cache. Must have no existing references. 756 */ 757 void 758 objcache_destroy(struct objcache *oc) 759 { 760 struct percpu_objcache *cache_percpu; 761 struct magazinedepot *depot; 762 int clusterid, cpuid; 763 struct magazinelist tmplist; 764 765 SLIST_INIT(&tmplist); 766 for (clusterid = 0; clusterid < MAXCLUSTERS; clusterid++) { 767 depot = &oc->depot[clusterid]; 768 spin_lock_wr(&depot->spin); 769 depot_disassociate(depot, &tmplist); 770 spin_unlock_wr(&depot->spin); 771 } 772 maglist_purge(oc, &tmplist); 773 774 for (cpuid = 0; cpuid < ncpus; cpuid++) { 775 cache_percpu = &oc->cache_percpu[cpuid]; 776 777 mag_purge(oc, cache_percpu->loaded_magazine, TRUE); 778 mag_purge(oc, cache_percpu->previous_magazine, TRUE); 779 cache_percpu->loaded_magazine = NULL; 780 cache_percpu->previous_magazine = NULL; 781 /* don't bother adjusting depot->unallocated_objects */ 782 } 783 784 kfree(oc->name, M_TEMP); 785 kfree(oc, M_OBJCACHE); 786 } 787 788 #if 0 789 /* 790 * Populate the per-cluster depot with elements from a linear block 791 * of memory. Must be called for individually for each cluster. 792 * Populated depots should not be destroyed. 793 */ 794 void 795 objcache_populate_linear(struct objcache *oc, void *base, int nelts, int size) 796 { 797 char *p = base; 798 char *end = (char *)base + (nelts * size); 799 struct magazinedepot *depot = &oc->depot[myclusterid]; 800 struct magazine *emptymag = mag_alloc(depot->magcapcity); 801 802 while (p < end) { 803 emptymag->objects[emptymag->rounds++] = p; 804 if (MAGAZINE_FULL(emptymag)) { 805 spin_lock_wr(&depot->spin); 806 SLIST_INSERT_HEAD(&depot->fullmagazines, emptymag, 807 nextmagazine); 808 depot->unallocated_objects += emptymag->rounds; 809 spin_unlock_wr(&depot->spin); 810 if (depot->waiting) 811 wakeup(depot); 812 emptymag = mag_alloc(depot->magcapacity); 813 } 814 p += size; 815 } 816 if (MAGAZINE_EMPTY(emptymag)) { 817 mag_purge(oc, emptymag, TRUE); 818 } else { 819 spin_lock_wr(&depot->spin); 820 SLIST_INSERT_HEAD(&depot->fullmagazines, emptymag, 821 nextmagazine); 822 depot->unallocated_objects += emptymag->rounds; 823 spin_unlock_wr(&depot->spin); 824 if (depot->waiting) 825 wakeup(depot); 826 emptymag = mag_alloc(depot->magcapacity); 827 } 828 } 829 #endif 830 831 #if 0 832 /* 833 * Check depot contention once a minute. 834 * 2 contested locks per second allowed. 835 */ 836 static int objcache_rebalance_period; 837 static const int objcache_contention_rate = 120; 838 static struct callout objcache_callout; 839 840 #define MAXMAGSIZE 512 841 842 /* 843 * Check depot contention and increase magazine size if necessary. 844 */ 845 static void 846 objcache_timer(void *dummy) 847 { 848 struct objcache *oc; 849 struct magazinedepot *depot; 850 struct magazinelist tmplist; 851 852 XXX we need to detect when an objcache is destroyed out from under 853 us XXX 854 855 SLIST_INIT(&tmplist); 856 857 spin_lock_wr(&objcachelist_spin); 858 SLIST_FOREACH(oc, &allobjcaches, oc_next) { 859 depot = &oc->depot[myclusterid]; 860 if (depot->magcapacity < MAXMAGSIZE) { 861 if (depot->contested > objcache_contention_rate) { 862 spin_lock_wr(&depot->spin); 863 depot_disassociate(depot, &tmplist); 864 depot->magcapacity *= 2; 865 spin_unlock_wr(&depot->spin); 866 kprintf("objcache_timer: increasing cache %s" 867 " magsize to %d, contested %d times\n", 868 oc->name, depot->magcapacity, 869 depot->contested); 870 } 871 depot->contested = 0; 872 } 873 spin_unlock_wr(&objcachelist_spin); 874 if (maglist_purge(oc, &tmplist) > 0 && depot->waiting) 875 wakeup(depot); 876 spin_lock_wr(&objcachelist_spin); 877 } 878 spin_unlock_wr(&objcachelist_spin); 879 880 callout_reset(&objcache_callout, objcache_rebalance_period, 881 objcache_timer, NULL); 882 } 883 884 #endif 885 886 static void 887 objcache_init(void) 888 { 889 spin_init(&objcachelist_spin); 890 #if 0 891 callout_init(&objcache_callout); 892 objcache_rebalance_period = 60 * hz; 893 callout_reset(&objcache_callout, objcache_rebalance_period, 894 objcache_timer, NULL); 895 #endif 896 } 897 SYSINIT(objcache, SI_BOOT2_OBJCACHE, SI_ORDER_FIRST, objcache_init, 0); 898