Lines Matching refs:keg
225 uma_keg_t keg; member
298 static void keg_drain(uma_keg_t keg, int domain);
324 static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
349 static inline struct noslabbits *slab_dbg_bits(uma_slab_t slab, uma_keg_t keg);
351 static bool uma_dbg_kskip(uma_keg_t keg, void *mem);
589 kasan_mark_slab_valid(uma_keg_t keg, void *mem) in kasan_mark_slab_valid() argument
593 if ((keg->uk_flags & UMA_ZONE_NOKASAN) == 0) { in kasan_mark_slab_valid()
594 sz = keg->uk_ppera * PAGE_SIZE; in kasan_mark_slab_valid()
600 kasan_mark_slab_invalid(uma_keg_t keg, void *mem) in kasan_mark_slab_invalid() argument
604 if ((keg->uk_flags & UMA_ZONE_NOKASAN) == 0) { in kasan_mark_slab_invalid()
605 if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0) in kasan_mark_slab_invalid()
606 sz = keg->uk_ppera * PAGE_SIZE; in kasan_mark_slab_invalid()
608 sz = keg->uk_pgoff; in kasan_mark_slab_invalid()
624 kasan_mark_slab_valid(uma_keg_t keg __unused, void *mem __unused) in kasan_mark_slab_valid()
629 kasan_mark_slab_invalid(uma_keg_t keg __unused, void *mem __unused) in kasan_mark_slab_invalid()
1173 uma_keg_t keg; in zone_timeout() local
1179 keg = zone->uz_keg; in zone_timeout()
1185 KEG_LOCK(keg, 0); in zone_timeout()
1186 pages = keg->uk_domain[0].ud_pages; in zone_timeout()
1195 if ((slabs = pages / keg->uk_ppera) > keg->uk_hash.uh_hashsize) { in zone_timeout()
1206 KEG_UNLOCK(keg, 0); in zone_timeout()
1208 KEG_LOCK(keg, 0); in zone_timeout()
1210 if (hash_expand(&keg->uk_hash, &newhash)) { in zone_timeout()
1211 oldhash = keg->uk_hash; in zone_timeout()
1212 keg->uk_hash = newhash; in zone_timeout()
1216 KEG_UNLOCK(keg, 0); in zone_timeout()
1221 KEG_UNLOCK(keg, 0); in zone_timeout()
1593 keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start) in keg_free_slab() argument
1601 keg->uk_name, keg, slab, PAGE_SIZE * keg->uk_ppera); in keg_free_slab()
1603 mem = slab_data(slab, keg); in keg_free_slab()
1604 size = PAGE_SIZE * keg->uk_ppera; in keg_free_slab()
1606 kasan_mark_slab_valid(keg, mem); in keg_free_slab()
1607 if (keg->uk_fini != NULL) { in keg_free_slab()
1618 if (!uma_dbg_kskip(keg, slab_item(slab, keg, i)) || in keg_free_slab()
1619 keg->uk_fini != trash_fini) in keg_free_slab()
1621 keg->uk_fini(slab_item(slab, keg, i), keg->uk_size); in keg_free_slab()
1624 if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) { in keg_free_slab()
1625 zone_free_item(slabzone(keg->uk_ipers), slab_tohashslab(slab), in keg_free_slab()
1628 keg->uk_freef(mem, size, flags); in keg_free_slab()
1633 keg_drain_domain(uma_keg_t keg, int domain) in keg_drain_domain() argument
1640 dom = &keg->uk_domain[domain]; in keg_drain_domain()
1644 keg->uk_name, keg, domain, dom->ud_free_items); in keg_drain_domain()
1646 KEG_LOCK(keg, domain); in keg_drain_domain()
1653 partial = dom->ud_free_items - dom->ud_free_slabs * keg->uk_ipers; in keg_drain_domain()
1654 if (partial < keg->uk_reserve) { in keg_drain_domain()
1656 howmany(keg->uk_reserve - partial, keg->uk_ipers)); in keg_drain_domain()
1676 if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) { in keg_drain_domain()
1678 UMA_HASH_REMOVE(&keg->uk_hash, slab); in keg_drain_domain()
1680 dom->ud_free_items -= stofree * keg->uk_ipers; in keg_drain_domain()
1682 dom->ud_pages -= stofree * keg->uk_ppera; in keg_drain_domain()
1683 KEG_UNLOCK(keg, domain); in keg_drain_domain()
1686 keg_free_slab(keg, slab, keg->uk_ipers); in keg_drain_domain()
1696 keg_drain(uma_keg_t keg, int domain) in keg_drain() argument
1700 if ((keg->uk_flags & UMA_ZONE_NOFREE) != 0) in keg_drain()
1703 keg_drain_domain(keg, domain); in keg_drain()
1706 keg_drain_domain(keg, i); in keg_drain()
1753 keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, in keg_alloc_slab() argument
1770 if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) { in keg_alloc_slab()
1772 hslab = zone_alloc_item(slabzone(keg->uk_ipers), NULL, in keg_alloc_slab()
1786 if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) in keg_alloc_slab()
1791 if (keg->uk_flags & UMA_ZONE_NODUMP) in keg_alloc_slab()
1795 size = keg->uk_ppera * PAGE_SIZE; in keg_alloc_slab()
1796 mem = keg->uk_allocf(zone, size, domain, &sflags, aflags); in keg_alloc_slab()
1798 if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) in keg_alloc_slab()
1799 zone_free_item(slabzone(keg->uk_ipers), in keg_alloc_slab()
1806 if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) in keg_alloc_slab()
1813 if (!(keg->uk_flags & UMA_ZFLAG_OFFPAGE)) in keg_alloc_slab()
1814 slab = (uma_slab_t)(mem + keg->uk_pgoff); in keg_alloc_slab()
1818 if (keg->uk_flags & UMA_ZFLAG_VTOSLAB) in keg_alloc_slab()
1819 for (i = 0; i < keg->uk_ppera; i++) in keg_alloc_slab()
1823 slab->us_freecount = keg->uk_ipers; in keg_alloc_slab()
1827 BIT_FILL(keg->uk_ipers, &slab->us_free); in keg_alloc_slab()
1829 BIT_ZERO(keg->uk_ipers, slab_dbg_bits(slab, keg)); in keg_alloc_slab()
1832 if (keg->uk_init != NULL) { in keg_alloc_slab()
1833 for (i = 0; i < keg->uk_ipers; i++) in keg_alloc_slab()
1834 if (keg->uk_init(slab_item(slab, keg, i), in keg_alloc_slab()
1835 keg->uk_size, flags) != 0) in keg_alloc_slab()
1837 if (i != keg->uk_ipers) { in keg_alloc_slab()
1838 keg_free_slab(keg, slab, i); in keg_alloc_slab()
1842 kasan_mark_slab_invalid(keg, mem); in keg_alloc_slab()
1843 KEG_LOCK(keg, domain); in keg_alloc_slab()
1846 slab, keg->uk_name, keg); in keg_alloc_slab()
1848 if (keg->uk_flags & UMA_ZFLAG_HASH) in keg_alloc_slab()
1849 UMA_HASH_INSERT(&keg->uk_hash, slab, mem); in keg_alloc_slab()
1856 dom = &keg->uk_domain[domain]; in keg_alloc_slab()
1858 dom->ud_pages += keg->uk_ppera; in keg_alloc_slab()
1859 dom->ud_free_items += keg->uk_ipers; in keg_alloc_slab()
2026 uma_keg_t keg; in noobj_alloc() local
2030 keg = zone->uz_keg; in noobj_alloc()
2058 zkva = keg->uk_kva + in noobj_alloc()
2059 atomic_fetchadd_long(&keg->uk_offset, round_page(bytes)); in noobj_alloc()
2158 slab_dbg_bits(uma_slab_t slab, uma_keg_t keg) in slab_dbg_bits() argument
2161 return ((void *)((char *)&slab->us_free + BITSET_SIZE(keg->uk_ipers))); in slab_dbg_bits()
2225 keg_layout_one(uma_keg_t keg, u_int rsize, u_int slabsize, u_int fmt, in keg_layout_one() argument
2239 kl->ipers = slab_ipers_hdr(keg->uk_size, rsize, kl->slabsize, in keg_layout_one()
2261 keg_layout(uma_keg_t keg) in keg_layout() argument
2272 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || in keg_layout()
2273 (keg->uk_size <= UMA_PCPU_ALLOC_SIZE && in keg_layout()
2274 (keg->uk_flags & UMA_ZONE_CACHESPREAD) == 0), in keg_layout()
2276 __func__, keg->uk_name, keg->uk_size, keg->uk_flags, in keg_layout()
2278 KASSERT((keg->uk_flags & (UMA_ZFLAG_INTERNAL | UMA_ZONE_VM)) == 0 || in keg_layout()
2279 (keg->uk_flags & (UMA_ZONE_NOTOUCH | UMA_ZONE_PCPU)) == 0, in keg_layout()
2280 ("%s: incompatible flags 0x%b", __func__, keg->uk_flags, in keg_layout()
2283 alignsize = keg->uk_align + 1; in keg_layout()
2298 rsize = MAX(keg->uk_size, UMA_SMALLEST_UNIT); in keg_layout()
2301 if ((keg->uk_flags & UMA_ZONE_CACHESPREAD) != 0) { in keg_layout()
2320 slabsize = round_page(keg->uk_size); in keg_layout()
2327 if ((keg->uk_flags & (UMA_ZONE_NOTOUCH | UMA_ZONE_PCPU)) == 0) in keg_layout()
2344 if ((keg->uk_flags & (UMA_ZFLAG_INTERNAL | UMA_ZONE_VM)) != 0) in keg_layout()
2357 i = (slabsize + rsize - keg->uk_size) / MAX(PAGE_SIZE, rsize); in keg_layout()
2359 keg->uk_name, keg, keg->uk_flags, PRINT_UMA_ZFLAGS, slabsize, in keg_layout()
2363 round_page(rsize * (i - 1) + keg->uk_size); in keg_layout()
2371 keg_layout_one(keg, rsize, slabsize, fmts[j], &kl_tmp); in keg_layout()
2379 keg->uk_name, kl.format, kl.ipers, rsize, in keg_layout()
2389 (keg->uk_flags & (UMA_ZONE_PCPU | UMA_ZONE_CONTIG)) != 0) in keg_layout()
2394 if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) in keg_layout()
2397 keg->uk_rsize = rsize; in keg_layout()
2398 keg->uk_ipers = kl.ipers; in keg_layout()
2399 keg->uk_ppera = pages; in keg_layout()
2400 keg->uk_flags |= kl.format; in keg_layout()
2407 if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0 || in keg_layout()
2408 (keg->uk_ipers - 1) * rsize >= PAGE_SIZE) { in keg_layout()
2409 if ((keg->uk_flags & UMA_ZONE_NOTPAGE) != 0) in keg_layout()
2410 keg->uk_flags |= UMA_ZFLAG_HASH; in keg_layout()
2412 keg->uk_flags |= UMA_ZFLAG_VTOSLAB; in keg_layout()
2416 __func__, keg->uk_name, keg->uk_flags, rsize, keg->uk_ipers, in keg_layout()
2418 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_MAX_SETSIZE, in keg_layout()
2420 keg->uk_name, keg->uk_flags, PRINT_UMA_ZFLAGS, rsize, in keg_layout()
2421 keg->uk_ipers, pages)); in keg_layout()
2435 uma_keg_t keg = mem; in keg_ctor() local
2439 bzero(keg, size); in keg_ctor()
2440 keg->uk_size = arg->size; in keg_ctor()
2441 keg->uk_init = arg->uminit; in keg_ctor()
2442 keg->uk_fini = arg->fini; in keg_ctor()
2443 keg->uk_align = arg->align; in keg_ctor()
2444 keg->uk_reserve = 0; in keg_ctor()
2445 keg->uk_flags = arg->flags; in keg_ctor()
2452 keg->uk_dr.dr_policy = DOMAINSET_RR(); in keg_ctor()
2453 keg->uk_dr.dr_iter = 0; in keg_ctor()
2459 keg->uk_name = zone->uz_name; in keg_ctor()
2462 keg->uk_init = zero_init; in keg_ctor()
2465 keg->uk_flags |= UMA_ZFLAG_VTOSLAB; in keg_ctor()
2468 keg->uk_flags &= ~UMA_ZONE_PCPU; in keg_ctor()
2471 keg_layout(keg); in keg_ctor()
2480 if ((keg->uk_flags & in keg_ctor()
2482 keg->uk_flags |= UMA_ZONE_FIRSTTOUCH; in keg_ctor()
2483 else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0) in keg_ctor()
2484 keg->uk_flags |= UMA_ZONE_ROUNDROBIN; in keg_ctor()
2492 if (keg->uk_ppera == 1) in keg_ctor()
2493 keg->uk_allocf = uma_small_alloc; in keg_ctor()
2497 keg->uk_allocf = startup_alloc; in keg_ctor()
2498 else if (keg->uk_flags & UMA_ZONE_PCPU) in keg_ctor()
2499 keg->uk_allocf = pcpu_page_alloc; in keg_ctor()
2500 else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && keg->uk_ppera > 1) in keg_ctor()
2501 keg->uk_allocf = contig_alloc; in keg_ctor()
2503 keg->uk_allocf = page_alloc; in keg_ctor()
2505 if (keg->uk_ppera == 1) in keg_ctor()
2506 keg->uk_freef = uma_small_free; in keg_ctor()
2509 if (keg->uk_flags & UMA_ZONE_PCPU) in keg_ctor()
2510 keg->uk_freef = pcpu_page_free; in keg_ctor()
2512 keg->uk_freef = page_free; in keg_ctor()
2518 KEG_LOCK_INIT(keg, i, (arg->flags & UMA_ZONE_MTXCLASS)); in keg_ctor()
2525 if (!(keg->uk_flags & UMA_ZFLAG_OFFPAGE)) { in keg_ctor()
2528 shsize = slab_sizeof(keg->uk_ipers); in keg_ctor()
2529 keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - shsize; in keg_ctor()
2537 KASSERT(keg->uk_pgoff + shsize <= PAGE_SIZE * keg->uk_ppera, in keg_ctor()
2539 zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size)); in keg_ctor()
2542 if (keg->uk_flags & UMA_ZFLAG_HASH) in keg_ctor()
2543 hash_alloc(&keg->uk_hash, 0); in keg_ctor()
2545 CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)", keg, zone->uz_name, zone); in keg_ctor()
2547 LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); in keg_ctor()
2550 LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); in keg_ctor()
2558 uma_keg_t keg; in zone_kva_available() local
2562 KEG_GET(zone, keg); in zone_kva_available()
2564 if (keg->uk_allocf == startup_alloc) { in zone_kva_available()
2566 if (keg->uk_flags & UMA_ZONE_PCPU) in zone_kva_available()
2567 keg->uk_allocf = pcpu_page_alloc; in zone_kva_available()
2568 else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && in zone_kva_available()
2569 keg->uk_ppera > 1) in zone_kva_available()
2570 keg->uk_allocf = contig_alloc; in zone_kva_available()
2572 keg->uk_allocf = page_alloc; in zone_kva_available()
2591 uma_keg_t keg; in zone_alloc_sysctl() local
2644 keg = zone->uz_keg; in zone_alloc_sysctl()
2647 "name", CTLFLAG_RD, keg->uk_name, "Keg name"); in zone_alloc_sysctl()
2649 "rsize", CTLFLAG_RD, &keg->uk_rsize, 0, in zone_alloc_sysctl()
2652 "ppera", CTLFLAG_RD, &keg->uk_ppera, 0, in zone_alloc_sysctl()
2655 "ipers", CTLFLAG_RD, &keg->uk_ipers, 0, in zone_alloc_sysctl()
2658 "align", CTLFLAG_RD, &keg->uk_align, 0, in zone_alloc_sysctl()
2661 "reserve", CTLFLAG_RD, &keg->uk_reserve, 0, in zone_alloc_sysctl()
2665 keg, 0, sysctl_handle_uma_slab_efficiency, "I", in zone_alloc_sysctl()
2670 dom = &keg->uk_domain[i]; in zone_alloc_sysctl()
2814 uma_keg_t keg; in zone_ctor() local
2886 keg = arg->keg; in zone_ctor()
2891 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg")); in zone_ctor()
2897 LIST_FOREACH(z, &keg->uk_zones, uz_link) { in zone_ctor()
2905 } else if (keg == NULL) { in zone_ctor()
2906 if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini, in zone_ctor()
2920 error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg, in zone_ctor()
2927 zone->uz_keg = keg; in zone_ctor()
2928 zone->uz_size = keg->uk_size; in zone_ctor()
2929 zone->uz_flags |= (keg->uk_flags & in zone_ctor()
2976 uma_keg_t keg; in keg_dtor() local
2980 keg = (uma_keg_t)arg; in keg_dtor()
2983 free += keg->uk_domain[i].ud_free_items; in keg_dtor()
2984 pages += keg->uk_domain[i].ud_pages; in keg_dtor()
2985 KEG_LOCK_FINI(keg, i); in keg_dtor()
2990 keg->uk_name ? keg->uk_name : "", in keg_dtor()
2991 pages / keg->uk_ppera * keg->uk_ipers - free, pages); in keg_dtor()
2993 hash_free(&keg->uk_hash); in keg_dtor()
3006 uma_keg_t keg; in zone_dtor() local
3020 keg = zone->uz_keg; in zone_dtor()
3021 keg->uk_reserve = 0; in zone_dtor()
3029 keg = zone->uz_keg; in zone_dtor()
3031 LIST_REMOVE(keg, uk_link); in zone_dtor()
3033 zone_free_item(kegs, keg, NULL, SKIP_NONE); in zone_dtor()
3048 uma_keg_t keg; in zone_foreach_unlocked() local
3051 LIST_FOREACH(keg, &uma_kegs, uk_link) { in zone_foreach_unlocked()
3052 LIST_FOREACH(zone, &keg->uk_zones, uz_link) in zone_foreach_unlocked()
3127 args.keg = primarykeg; in uma_startup1()
3138 args.keg = NULL; in uma_startup1()
3322 args.keg = NULL; in uma_zcreate()
3337 uma_keg_t keg; in uma_zsecond_create() local
3340 keg = primary->uz_keg; in uma_zsecond_create()
3343 args.size = keg->uk_size; in uma_zsecond_create()
3348 args.align = keg->uk_align; in uma_zsecond_create()
3349 args.flags = keg->uk_flags | UMA_ZONE_SECONDARY; in uma_zsecond_create()
3350 args.keg = keg; in uma_zsecond_create()
3911 keg_first_slab(uma_keg_t keg, int domain, bool rr) in keg_first_slab() argument
3919 KEG_LOCK_ASSERT(keg, domain); in keg_first_slab()
3924 dom = &keg->uk_domain[domain]; in keg_first_slab()
3945 keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr, int flags) in keg_fetch_free_slab() argument
3951 if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) in keg_fetch_free_slab()
3954 KEG_LOCK(keg, domain); in keg_fetch_free_slab()
3955 reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve; in keg_fetch_free_slab()
3956 if (keg->uk_domain[domain].ud_free_items <= reserve || in keg_fetch_free_slab()
3957 (slab = keg_first_slab(keg, domain, rr)) == NULL) { in keg_fetch_free_slab()
3958 KEG_UNLOCK(keg, domain); in keg_fetch_free_slab()
3965 keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags) in keg_fetch_slab() argument
3987 vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, in keg_fetch_slab()
3995 slab = keg_fetch_free_slab(keg, domain, rr, flags); in keg_fetch_slab()
4004 slab = keg_alloc_slab(keg, zone, domain, flags, aflags); in keg_fetch_slab()
4024 vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask, 0); in keg_fetch_slab()
4036 if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL) in keg_fetch_slab()
4043 slab_alloc_item(uma_keg_t keg, uma_slab_t slab) in slab_alloc_item() argument
4049 KEG_LOCK_ASSERT(keg, slab->us_domain); in slab_alloc_item()
4051 dom = &keg->uk_domain[slab->us_domain]; in slab_alloc_item()
4052 freei = BIT_FFS(keg->uk_ipers, &slab->us_free) - 1; in slab_alloc_item()
4053 BIT_CLR(keg->uk_ipers, freei, &slab->us_free); in slab_alloc_item()
4054 item = slab_item(slab, keg, freei); in slab_alloc_item()
4077 uma_keg_t keg; in zone_import() local
4085 keg = zone->uz_keg; in zone_import()
4088 if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL) in zone_import()
4093 dom = &keg->uk_domain[slab->us_domain]; in zone_import()
4095 bucket[i++] = slab_alloc_item(keg, slab); in zone_import()
4096 if (keg->uk_reserve > 0 && in zone_import()
4097 dom->ud_free_items <= keg->uk_reserve) { in zone_import()
4103 KEG_UNLOCK(keg, slab->us_domain); in zone_import()
4120 KEG_UNLOCK(keg, slab->us_domain); in zone_import()
4801 uma_keg_t keg; in slab_free_item() local
4805 keg = zone->uz_keg; in slab_free_item()
4806 KEG_LOCK_ASSERT(keg, slab->us_domain); in slab_free_item()
4809 dom = &keg->uk_domain[slab->us_domain]; in slab_free_item()
4810 if (slab->us_freecount + 1 == keg->uk_ipers) { in slab_free_item()
4820 freei = slab_item_index(slab, keg, item); in slab_free_item()
4821 BIT_SET(keg->uk_ipers, freei, &slab->us_free); in slab_free_item()
4834 uma_keg_t keg; in zone_release() local
4840 keg = zone->uz_keg; in zone_release()
4843 lock = KEG_LOCK(keg, 0); in zone_release()
4851 slab = hash_sfind(&keg->uk_hash, mem); in zone_release()
4853 slab = (uma_slab_t)(mem + keg->uk_pgoff); in zone_release()
4855 if (lock != KEG_LOCKPTR(keg, slab->us_domain)) { in zone_release()
4858 lock = KEG_LOCK(keg, slab->us_domain); in zone_release()
5052 uma_keg_get_allocs(uma_keg_t keg) in uma_keg_get_allocs() argument
5058 LIST_FOREACH(z, &keg->uk_zones, uz_link) in uma_keg_get_allocs()
5069 uma_keg_t keg; in uma_zone_set_init() local
5071 KEG_GET(zone, keg); in uma_zone_set_init()
5072 KEG_ASSERT_COLD(keg); in uma_zone_set_init()
5073 keg->uk_init = uminit; in uma_zone_set_init()
5080 uma_keg_t keg; in uma_zone_set_fini() local
5082 KEG_GET(zone, keg); in uma_zone_set_fini()
5083 KEG_ASSERT_COLD(keg); in uma_zone_set_fini()
5084 keg->uk_fini = fini; in uma_zone_set_fini()
5109 uma_keg_t keg; in uma_zone_set_freef() local
5111 KEG_GET(zone, keg); in uma_zone_set_freef()
5112 KEG_ASSERT_COLD(keg); in uma_zone_set_freef()
5113 keg->uk_freef = freef; in uma_zone_set_freef()
5120 uma_keg_t keg; in uma_zone_set_allocf() local
5122 KEG_GET(zone, keg); in uma_zone_set_allocf()
5123 KEG_ASSERT_COLD(keg); in uma_zone_set_allocf()
5124 keg->uk_allocf = allocf; in uma_zone_set_allocf()
5153 uma_keg_t keg; in uma_zone_reserve() local
5155 KEG_GET(zone, keg); in uma_zone_reserve()
5156 KEG_ASSERT_COLD(keg); in uma_zone_reserve()
5157 keg->uk_reserve = items; in uma_zone_reserve()
5164 uma_keg_t keg; local
5168 KEG_GET(zone, keg);
5169 KEG_ASSERT_COLD(keg);
5172 pages = howmany(count, keg->uk_ipers) * keg->uk_ppera;
5175 if (keg->uk_ppera > 1) {
5185 MPASS(keg->uk_kva == 0);
5186 keg->uk_kva = kva;
5187 keg->uk_offset = 0;
5188 zone->uz_max_items = pages * keg->uk_ipers;
5190 keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc;
5192 keg->uk_allocf = noobj_alloc;
5194 keg->uk_flags |= UMA_ZFLAG_LIMIT | UMA_ZONE_NOFREE;
5208 uma_keg_t keg; local
5211 KEG_GET(zone, keg);
5212 slabs = howmany(items, keg->uk_ipers);
5215 vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain,
5218 slab = keg_alloc_slab(keg, zone, domain, M_WAITOK,
5221 dom = &keg->uk_domain[slab->us_domain];
5230 KEG_UNLOCK(keg, slab->us_domain);
5234 vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask, 0);
5672 uma_keg_t keg = arg1; local
5675 total = keg->uk_ppera * PAGE_SIZE;
5676 if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0)
5677 total += slabzone(keg->uk_ipers)->uz_keg->uk_rsize;
5683 avail = keg->uk_ipers * roundup2(keg->uk_size, keg->uk_align + 1);
5684 if ((keg->uk_flags & UMA_ZONE_PCPU) != 0)
5705 uma_keg_t keg; local
5718 keg = zone->uz_keg;
5719 if ((keg->uk_flags & UMA_ZFLAG_HASH) == 0)
5720 return ((uma_slab_t)(mem + keg->uk_pgoff));
5721 KEG_LOCK(keg, 0);
5722 slab = hash_sfind(&keg->uk_hash, mem);
5723 KEG_UNLOCK(keg, 0);
5739 uma_dbg_kskip(uma_keg_t keg, void *mem) argument
5750 if (keg->uk_ipers > 1) {
5751 idx *= keg->uk_ipers;
5752 idx += ((uintptr_t)mem & PAGE_MASK) / keg->uk_rsize;
5771 uma_keg_t keg; local
5780 keg = zone->uz_keg;
5781 freei = slab_item_index(slab, keg, item);
5783 if (BIT_TEST_SET_ATOMIC(keg->uk_ipers, freei,
5784 slab_dbg_bits(slab, keg)))
5797 uma_keg_t keg; local
5806 keg = zone->uz_keg;
5807 freei = slab_item_index(slab, keg, item);
5809 if (freei >= keg->uk_ipers)
5813 if (slab_item(slab, keg, freei) != item)
5817 if (!BIT_TEST_CLR_ATOMIC(keg->uk_ipers, freei,
5818 slab_dbg_bits(slab, keg)))