1*bad3ecd0Sotto/* $OpenBSD: t8.2,v 1.1 2003/07/17 21:04:04 otto Exp $ */ 2*bad3ecd0Sotto/* $NetBSD: kern_malloc.c,v 1.15.4.2 1996/06/13 17:10:56 cgd Exp $ */ 3*bad3ecd0Sotto 4*bad3ecd0Sotto/* 5*bad3ecd0Sotto * Copyright (c) 1987, 1991, 1993 6*bad3ecd0Sotto * The Regents of the University of California. All rights reserved. 7*bad3ecd0Sotto * 8*bad3ecd0Sotto * Redistribution and use in source and binary forms, with or without 9*bad3ecd0Sotto * modification, are permitted provided that the following conditions 10*bad3ecd0Sotto * are met: 11*bad3ecd0Sotto * 1. Redistributions of source code must retain the above copyright 12*bad3ecd0Sotto * notice, this list of conditions and the following disclaimer. 13*bad3ecd0Sotto * 2. Redistributions in binary form must reproduce the above copyright 14*bad3ecd0Sotto * notice, this list of conditions and the following disclaimer in the 15*bad3ecd0Sotto * documentation and/or other materials provided with the distribution. 16*bad3ecd0Sotto * 3. Neither the name of the University nor the names of its contributors 17*bad3ecd0Sotto * may be used to endorse or promote products derived from this software 18*bad3ecd0Sotto * without specific prior written permission. 19*bad3ecd0Sotto * 20*bad3ecd0Sotto * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21*bad3ecd0Sotto * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22*bad3ecd0Sotto * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23*bad3ecd0Sotto * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24*bad3ecd0Sotto * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25*bad3ecd0Sotto * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26*bad3ecd0Sotto * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27*bad3ecd0Sotto * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28*bad3ecd0Sotto * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29*bad3ecd0Sotto * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30*bad3ecd0Sotto * SUCH DAMAGE. 31*bad3ecd0Sotto * 32*bad3ecd0Sotto * @(#)kern_malloc.c 8.3 (Berkeley) 1/4/94 33*bad3ecd0Sotto */ 34*bad3ecd0Sotto 35*bad3ecd0Sotto#include <sys/param.h> 36*bad3ecd0Sotto#include <sys/proc.h> 37*bad3ecd0Sotto#include <sys/kernel.h> 38*bad3ecd0Sotto#include <sys/malloc.h> 39*bad3ecd0Sotto#include <sys/systm.h> 40*bad3ecd0Sotto#include <sys/sysctl.h> 41*bad3ecd0Sotto 42*bad3ecd0Sotto#include <uvm/uvm_extern.h> 43*bad3ecd0Sotto 44*bad3ecd0Sottostatic struct vm_map_intrsafe kmem_map_store; 45*bad3ecd0Sottostruct vm_map *kmem_map = NULL; 46*bad3ecd0Sotto 47*bad3ecd0Sotto#ifdef NKMEMCLUSTERS 48*bad3ecd0Sotto#error NKMEMCLUSTERS is obsolete; remove it from your kernel config file and use NKMEMPAGES instead or let the kernel auto-size 49*bad3ecd0Sotto#endif 50*bad3ecd0Sotto 51*bad3ecd0Sotto/* 52*bad3ecd0Sotto * Default number of pages in kmem_map. We attempt to calculate this 53*bad3ecd0Sotto * at run-time, but allow it to be either patched or set in the kernel 54*bad3ecd0Sotto * config file. 55*bad3ecd0Sotto */ 56*bad3ecd0Sotto#ifndef NKMEMPAGES 57*bad3ecd0Sotto#define NKMEMPAGES 0 58*bad3ecd0Sotto#endif 59*bad3ecd0Sottoint nkmempages = NKMEMPAGES; 60*bad3ecd0Sotto 61*bad3ecd0Sotto/* 62*bad3ecd0Sotto * Defaults for lower- and upper-bounds for the kmem_map page count. 63*bad3ecd0Sotto * Can be overridden by kernel config options. 64*bad3ecd0Sotto */ 65*bad3ecd0Sotto#ifndef NKMEMPAGES_MIN 66*bad3ecd0Sotto#define NKMEMPAGES_MIN NKMEMPAGES_MIN_DEFAULT 67*bad3ecd0Sotto#endif 68*bad3ecd0Sotto 69*bad3ecd0Sotto#ifndef NKMEMPAGES_MAX 70*bad3ecd0Sotto#define NKMEMPAGES_MAX NKMEMPAGES_MAX_DEFAULT 71*bad3ecd0Sotto#endif 72*bad3ecd0Sotto 73*bad3ecd0Sottostruct kmembuckets bucket[MINBUCKET + 16]; 74*bad3ecd0Sottostruct kmemstats kmemstats[M_LAST]; 75*bad3ecd0Sottostruct kmemusage *kmemusage; 76*bad3ecd0Sottochar *kmembase, *kmemlimit; 77*bad3ecd0Sottochar buckstring[16 * sizeof("123456,")]; 78*bad3ecd0Sottoint buckstring_init = 0; 79*bad3ecd0Sotto#if defined(KMEMSTATS) || defined(DIAGNOSTIC) || defined(FFS_SOFTUPDATES) 80*bad3ecd0Sottochar *memname[] = INITKMEMNAMES; 81*bad3ecd0Sottochar *memall = NULL; 82*bad3ecd0Sottoextern struct lock sysctl_kmemlock; 83*bad3ecd0Sotto#endif 84*bad3ecd0Sotto 85*bad3ecd0Sotto#ifdef DIAGNOSTIC 86*bad3ecd0Sotto/* 87*bad3ecd0Sotto * This structure provides a set of masks to catch unaligned frees. 88*bad3ecd0Sotto */ 89*bad3ecd0Sottoconst long addrmask[] = { 0, 90*bad3ecd0Sotto 0x00000001, 0x00000003, 0x00000007, 0x0000000f, 91*bad3ecd0Sotto 0x0000001f, 0x0000003f, 0x0000007f, 0x000000ff, 92*bad3ecd0Sotto 0x000001ff, 0x000003ff, 0x000007ff, 0x00000fff, 93*bad3ecd0Sotto 0x00001fff, 0x00003fff, 0x00007fff, 0x0000ffff, 94*bad3ecd0Sotto}; 95*bad3ecd0Sotto 96*bad3ecd0Sotto/* 97*bad3ecd0Sotto * The WEIRD_ADDR is used as known text to copy into free objects so 98*bad3ecd0Sotto * that modifications after frees can be detected. 99*bad3ecd0Sotto */ 100*bad3ecd0Sotto#define WEIRD_ADDR ((unsigned) 0xdeadbeef) 101*bad3ecd0Sotto#define MAX_COPY 32 102*bad3ecd0Sotto 103*bad3ecd0Sotto/* 104*bad3ecd0Sotto * Normally the freelist structure is used only to hold the list pointer 105*bad3ecd0Sotto * for free objects. However, when running with diagnostics, the first 106*bad3ecd0Sotto * 8 bytes of the structure is unused except for diagnostic information, 107*bad3ecd0Sotto * and the free list pointer is at offset 8 in the structure. Since the 108*bad3ecd0Sotto * first 8 bytes is the portion of the structure most often modified, this 109*bad3ecd0Sotto * helps to detect memory reuse problems and avoid free list corruption. 110*bad3ecd0Sotto */ 111*bad3ecd0Sottostruct freelist { 112*bad3ecd0Sotto int32_t spare0; 113*bad3ecd0Sotto int16_t type; 114*bad3ecd0Sotto int16_t spare1; 115*bad3ecd0Sotto caddr_t next; 116*bad3ecd0Sotto}; 117*bad3ecd0Sotto#else /* !DIAGNOSTIC */ 118*bad3ecd0Sottostruct freelist { 119*bad3ecd0Sotto caddr_t next; 120*bad3ecd0Sotto}; 121*bad3ecd0Sotto#endif /* DIAGNOSTIC */ 122*bad3ecd0Sotto 123*bad3ecd0Sotto/* 124*bad3ecd0Sotto * Allocate a block of memory 125*bad3ecd0Sotto */ 126*bad3ecd0Sottovoid * 127*bad3ecd0Sottomalloc(size, type, flags) 128*bad3ecd0Sotto unsigned long size; 129*bad3ecd0Sotto int type, flags; 130*bad3ecd0Sotto{ 131*bad3ecd0Sotto register struct kmembuckets *kbp; 132*bad3ecd0Sotto register struct kmemusage *kup; 133*bad3ecd0Sotto register struct freelist *freep; 134*bad3ecd0Sotto long indx, npg, allocsize; 135*bad3ecd0Sotto int s; 136*bad3ecd0Sotto caddr_t va, cp, savedlist; 137*bad3ecd0Sotto#ifdef DIAGNOSTIC 138*bad3ecd0Sotto int32_t *end, *lp; 139*bad3ecd0Sotto int copysize; 140*bad3ecd0Sotto char *savedtype; 141*bad3ecd0Sotto#endif 142*bad3ecd0Sotto#ifdef KMEMSTATS 143*bad3ecd0Sotto register struct kmemstats *ksp = &kmemstats[type]; 144*bad3ecd0Sotto 145*bad3ecd0Sotto if (((unsigned long)type) >= M_LAST) 146*bad3ecd0Sotto panic("malloc - bogus type"); 147*bad3ecd0Sotto#endif 148*bad3ecd0Sotto 149*bad3ecd0Sotto#ifdef MALLOC_DEBUG 150*bad3ecd0Sotto if (debug_malloc(size, type, flags, (void **)&va)) 151*bad3ecd0Sotto return ((void *) va); 152*bad3ecd0Sotto#endif 153*bad3ecd0Sotto 154*bad3ecd0Sotto indx = BUCKETINDX(size); 155*bad3ecd0Sotto kbp = &bucket[indx]; 156*bad3ecd0Sotto s = splvm(); 157*bad3ecd0Sotto#ifdef KMEMSTATS 158*bad3ecd0Sotto while (ksp->ks_memuse >= ksp->ks_limit) { 159*bad3ecd0Sotto if (flags & M_NOWAIT) { 160*bad3ecd0Sotto splx(s); 161*bad3ecd0Sotto return ((void *) NULL); 162*bad3ecd0Sotto } 163*bad3ecd0Sotto if (ksp->ks_limblocks < 65535) 164*bad3ecd0Sotto ksp->ks_limblocks++; 165*bad3ecd0Sotto tsleep((caddr_t)ksp, PSWP+2, memname[type], 0); 166*bad3ecd0Sotto } 167*bad3ecd0Sotto ksp->ks_size |= 1 << indx; 168*bad3ecd0Sotto#endif 169*bad3ecd0Sotto#ifdef DIAGNOSTIC 170*bad3ecd0Sotto copysize = 1 << indx < MAX_COPY ? 1 << indx : MAX_COPY; 171*bad3ecd0Sotto#endif 172*bad3ecd0Sotto if (kbp->kb_next == NULL) { 173*bad3ecd0Sotto kbp->kb_last = NULL; 174*bad3ecd0Sotto if (size > MAXALLOCSAVE) 175*bad3ecd0Sotto allocsize = round_page(size); 176*bad3ecd0Sotto else 177*bad3ecd0Sotto allocsize = 1 << indx; 178*bad3ecd0Sotto npg = btoc(allocsize); 179*bad3ecd0Sotto va = (caddr_t) uvm_km_kmemalloc(kmem_map, uvmexp.kmem_object, 180*bad3ecd0Sotto (vsize_t)ctob(npg), 181*bad3ecd0Sotto (flags & M_NOWAIT) ? UVM_KMF_NOWAIT : 0); 182*bad3ecd0Sotto if (va == NULL) { 183*bad3ecd0Sotto /* 184*bad3ecd0Sotto * Kmem_malloc() can return NULL, even if it can 185*bad3ecd0Sotto * wait, if there is no map space available, because 186*bad3ecd0Sotto * it can't fix that problem. Neither can we, 187*bad3ecd0Sotto * right now. (We should release pages which 188*bad3ecd0Sotto * are completely free and which are in buckets 189*bad3ecd0Sotto * with too many free elements.) 190*bad3ecd0Sotto */ 191*bad3ecd0Sotto if ((flags & M_NOWAIT) == 0) 192*bad3ecd0Sotto panic("malloc: out of space in kmem_map"); 193*bad3ecd0Sotto splx(s); 194*bad3ecd0Sotto return ((void *) NULL); 195*bad3ecd0Sotto } 196*bad3ecd0Sotto#ifdef KMEMSTATS 197*bad3ecd0Sotto kbp->kb_total += kbp->kb_elmpercl; 198*bad3ecd0Sotto#endif 199*bad3ecd0Sotto kup = btokup(va); 200*bad3ecd0Sotto kup->ku_indx = indx; 201*bad3ecd0Sotto if (allocsize > MAXALLOCSAVE) { 202*bad3ecd0Sotto if (npg > 65535) 203*bad3ecd0Sotto panic("malloc: allocation too large"); 204*bad3ecd0Sotto kup->ku_pagecnt = npg; 205*bad3ecd0Sotto#ifdef KMEMSTATS 206*bad3ecd0Sotto ksp->ks_memuse += allocsize; 207*bad3ecd0Sotto#endif 208*bad3ecd0Sotto goto out; 209*bad3ecd0Sotto } 210*bad3ecd0Sotto#ifdef KMEMSTATS 211*bad3ecd0Sotto kup->ku_freecnt = kbp->kb_elmpercl; 212*bad3ecd0Sotto kbp->kb_totalfree += kbp->kb_elmpercl; 213*bad3ecd0Sotto#endif 214*bad3ecd0Sotto /* 215*bad3ecd0Sotto * Just in case we blocked while allocating memory, 216*bad3ecd0Sotto * and someone else also allocated memory for this 217*bad3ecd0Sotto * bucket, don't assume the list is still empty. 218*bad3ecd0Sotto */ 219*bad3ecd0Sotto savedlist = kbp->kb_next; 220*bad3ecd0Sotto kbp->kb_next = cp = va + (npg * PAGE_SIZE) - allocsize; 221*bad3ecd0Sotto for (;;) { 222*bad3ecd0Sotto freep = (struct freelist *)cp; 223*bad3ecd0Sotto#ifdef DIAGNOSTIC 224*bad3ecd0Sotto /* 225*bad3ecd0Sotto * Copy in known text to detect modification 226*bad3ecd0Sotto * after freeing. 227*bad3ecd0Sotto */ 228*bad3ecd0Sotto end = (int32_t *)&cp[copysize]; 229*bad3ecd0Sotto for (lp = (int32_t *)cp; lp < end; lp++) 230*bad3ecd0Sotto *lp = WEIRD_ADDR; 231*bad3ecd0Sotto freep->type = M_FREE; 232*bad3ecd0Sotto#endif /* DIAGNOSTIC */ 233*bad3ecd0Sotto if (cp <= va) 234*bad3ecd0Sotto break; 235*bad3ecd0Sotto cp -= allocsize; 236*bad3ecd0Sotto freep->next = cp; 237*bad3ecd0Sotto } 238*bad3ecd0Sotto freep->next = savedlist; 239*bad3ecd0Sotto if (kbp->kb_last == NULL) 240*bad3ecd0Sotto kbp->kb_last = (caddr_t)freep; 241*bad3ecd0Sotto } 242*bad3ecd0Sotto va = kbp->kb_next; 243*bad3ecd0Sotto kbp->kb_next = ((struct freelist *)va)->next; 244*bad3ecd0Sotto#ifdef DIAGNOSTIC 245*bad3ecd0Sotto freep = (struct freelist *)va; 246*bad3ecd0Sotto savedtype = (unsigned)freep->type < M_LAST ? 247*bad3ecd0Sotto memname[freep->type] : "???"; 248*bad3ecd0Sotto if (kbp->kb_next) { 249*bad3ecd0Sotto int rv; 250*bad3ecd0Sotto vaddr_t addr = (vaddr_t)kbp->kb_next; 251*bad3ecd0Sotto 252*bad3ecd0Sotto vm_map_lock(kmem_map); 253*bad3ecd0Sotto rv = uvm_map_checkprot(kmem_map, addr, 254*bad3ecd0Sotto addr + sizeof(struct freelist), VM_PROT_WRITE); 255*bad3ecd0Sotto vm_map_unlock(kmem_map); 256*bad3ecd0Sotto 257*bad3ecd0Sotto if (!rv) { 258*bad3ecd0Sotto printf("%s %d of object %p size 0x%lx %s %s (invalid addr %p)\n", 259*bad3ecd0Sotto "Data modified on freelist: word", 260*bad3ecd0Sotto (int32_t *)&kbp->kb_next - (int32_t *)kbp, va, size, 261*bad3ecd0Sotto "previous type", savedtype, kbp->kb_next); 262*bad3ecd0Sotto kbp->kb_next = NULL; 263*bad3ecd0Sotto } 264*bad3ecd0Sotto } 265*bad3ecd0Sotto 266*bad3ecd0Sotto /* Fill the fields that we've used with WEIRD_ADDR */ 267*bad3ecd0Sotto#if BYTE_ORDER == BIG_ENDIAN 268*bad3ecd0Sotto freep->type = WEIRD_ADDR >> 16; 269*bad3ecd0Sotto#endif 270*bad3ecd0Sotto#if BYTE_ORDER == LITTLE_ENDIAN 271*bad3ecd0Sotto freep->type = (short)WEIRD_ADDR; 272*bad3ecd0Sotto#endif 273*bad3ecd0Sotto end = (int32_t *)&freep->next + 274*bad3ecd0Sotto (sizeof(freep->next) / sizeof(int32_t)); 275*bad3ecd0Sotto for (lp = (int32_t *)&freep->next; lp < end; lp++) 276*bad3ecd0Sotto *lp = WEIRD_ADDR; 277*bad3ecd0Sotto 278*bad3ecd0Sotto /* and check that the data hasn't been modified. */ 279*bad3ecd0Sotto end = (int32_t *)&va[copysize]; 280*bad3ecd0Sotto for (lp = (int32_t *)va; lp < end; lp++) { 281*bad3ecd0Sotto if (*lp == WEIRD_ADDR) 282*bad3ecd0Sotto continue; 283*bad3ecd0Sotto printf("%s %d of object %p size 0x%lx %s %s (0x%x != 0x%x)\n", 284*bad3ecd0Sotto "Data modified on freelist: word", lp - (int32_t *)va, 285*bad3ecd0Sotto va, size, "previous type", savedtype, *lp, WEIRD_ADDR); 286*bad3ecd0Sotto break; 287*bad3ecd0Sotto } 288*bad3ecd0Sotto 289*bad3ecd0Sotto freep->spare0 = 0; 290*bad3ecd0Sotto#endif /* DIAGNOSTIC */ 291*bad3ecd0Sotto#ifdef KMEMSTATS 292*bad3ecd0Sotto kup = btokup(va); 293*bad3ecd0Sotto if (kup->ku_indx != indx) 294*bad3ecd0Sotto panic("malloc: wrong bucket"); 295*bad3ecd0Sotto if (kup->ku_freecnt == 0) 296*bad3ecd0Sotto panic("malloc: lost data"); 297*bad3ecd0Sotto kup->ku_freecnt--; 298*bad3ecd0Sotto kbp->kb_totalfree--; 299*bad3ecd0Sotto ksp->ks_memuse += 1 << indx; 300*bad3ecd0Sottoout: 301*bad3ecd0Sotto kbp->kb_calls++; 302*bad3ecd0Sotto ksp->ks_inuse++; 303*bad3ecd0Sotto ksp->ks_calls++; 304*bad3ecd0Sotto if (ksp->ks_memuse > ksp->ks_maxused) 305*bad3ecd0Sotto ksp->ks_maxused = ksp->ks_memuse; 306*bad3ecd0Sotto#else 307*bad3ecd0Sottoout: 308*bad3ecd0Sotto#endif 309*bad3ecd0Sotto splx(s); 310*bad3ecd0Sotto return ((void *) va); 311*bad3ecd0Sotto} 312*bad3ecd0Sotto 313*bad3ecd0Sotto/* 314*bad3ecd0Sotto * Free a block of memory allocated by malloc. 315*bad3ecd0Sotto */ 316*bad3ecd0Sottovoid 317*bad3ecd0Sottofree(addr, type) 318*bad3ecd0Sotto void *addr; 319*bad3ecd0Sotto int type; 320*bad3ecd0Sotto{ 321*bad3ecd0Sotto register struct kmembuckets *kbp; 322*bad3ecd0Sotto register struct kmemusage *kup; 323*bad3ecd0Sotto register struct freelist *freep; 324*bad3ecd0Sotto long size; 325*bad3ecd0Sotto int s; 326*bad3ecd0Sotto#ifdef DIAGNOSTIC 327*bad3ecd0Sotto caddr_t cp; 328*bad3ecd0Sotto int32_t *end, *lp; 329*bad3ecd0Sotto long alloc, copysize; 330*bad3ecd0Sotto#endif 331*bad3ecd0Sotto#ifdef KMEMSTATS 332*bad3ecd0Sotto register struct kmemstats *ksp = &kmemstats[type]; 333*bad3ecd0Sotto#endif 334*bad3ecd0Sotto 335*bad3ecd0Sotto#ifdef MALLOC_DEBUG 336*bad3ecd0Sotto if (debug_free(addr, type)) 337*bad3ecd0Sotto return; 338*bad3ecd0Sotto#endif 339*bad3ecd0Sotto 340*bad3ecd0Sotto#ifdef DIAGNOSTIC 341*bad3ecd0Sotto if (addr < (void *)kmembase || addr >= (void *)kmemlimit) 342*bad3ecd0Sotto panic("free: non-malloced addr %p type %s", addr, 343*bad3ecd0Sotto memname[type]); 344*bad3ecd0Sotto#endif 345*bad3ecd0Sotto 346*bad3ecd0Sotto kup = btokup(addr); 347*bad3ecd0Sotto size = 1 << kup->ku_indx; 348*bad3ecd0Sotto kbp = &bucket[kup->ku_indx]; 349*bad3ecd0Sotto s = splvm(); 350*bad3ecd0Sotto#ifdef DIAGNOSTIC 351*bad3ecd0Sotto /* 352*bad3ecd0Sotto * Check for returns of data that do not point to the 353*bad3ecd0Sotto * beginning of the allocation. 354*bad3ecd0Sotto */ 355*bad3ecd0Sotto if (size > PAGE_SIZE) 356*bad3ecd0Sotto alloc = addrmask[BUCKETINDX(PAGE_SIZE)]; 357*bad3ecd0Sotto else 358*bad3ecd0Sotto alloc = addrmask[kup->ku_indx]; 359*bad3ecd0Sotto if (((u_long)addr & alloc) != 0) 360*bad3ecd0Sotto panic("free: unaligned addr %p, size %ld, type %s, mask %ld", 361*bad3ecd0Sotto addr, size, memname[type], alloc); 362*bad3ecd0Sotto#endif /* DIAGNOSTIC */ 363*bad3ecd0Sotto if (size > MAXALLOCSAVE) { 364*bad3ecd0Sotto uvm_km_free(kmem_map, (vaddr_t)addr, ctob(kup->ku_pagecnt)); 365*bad3ecd0Sotto#ifdef KMEMSTATS 366*bad3ecd0Sotto size = kup->ku_pagecnt << PGSHIFT; 367*bad3ecd0Sotto ksp->ks_memuse -= size; 368*bad3ecd0Sotto kup->ku_indx = 0; 369*bad3ecd0Sotto kup->ku_pagecnt = 0; 370*bad3ecd0Sotto if (ksp->ks_memuse + size >= ksp->ks_limit && 371*bad3ecd0Sotto ksp->ks_memuse < ksp->ks_limit) 372*bad3ecd0Sotto wakeup((caddr_t)ksp); 373*bad3ecd0Sotto ksp->ks_inuse--; 374*bad3ecd0Sotto kbp->kb_total -= 1; 375*bad3ecd0Sotto#endif 376*bad3ecd0Sotto splx(s); 377*bad3ecd0Sotto return; 378*bad3ecd0Sotto } 379*bad3ecd0Sotto freep = (struct freelist *)addr; 380*bad3ecd0Sotto#ifdef DIAGNOSTIC 381*bad3ecd0Sotto /* 382*bad3ecd0Sotto * Check for multiple frees. Use a quick check to see if 383*bad3ecd0Sotto * it looks free before laboriously searching the freelist. 384*bad3ecd0Sotto */ 385*bad3ecd0Sotto if (freep->spare0 == WEIRD_ADDR) { 386*bad3ecd0Sotto for (cp = kbp->kb_next; cp; 387*bad3ecd0Sotto cp = ((struct freelist *)cp)->next) { 388*bad3ecd0Sotto if (addr != cp) 389*bad3ecd0Sotto continue; 390*bad3ecd0Sotto printf("multiply freed item %p\n", addr); 391*bad3ecd0Sotto panic("free: duplicated free"); 392*bad3ecd0Sotto } 393*bad3ecd0Sotto } 394*bad3ecd0Sotto /* 395*bad3ecd0Sotto * Copy in known text to detect modification after freeing 396*bad3ecd0Sotto * and to make it look free. Also, save the type being freed 397*bad3ecd0Sotto * so we can list likely culprit if modification is detected 398*bad3ecd0Sotto * when the object is reallocated. 399*bad3ecd0Sotto */ 400*bad3ecd0Sotto copysize = size < MAX_COPY ? size : MAX_COPY; 401*bad3ecd0Sotto end = (int32_t *)&((caddr_t)addr)[copysize]; 402*bad3ecd0Sotto for (lp = (int32_t *)addr; lp < end; lp++) 403*bad3ecd0Sotto *lp = WEIRD_ADDR; 404*bad3ecd0Sotto freep->type = type; 405*bad3ecd0Sotto#endif /* DIAGNOSTIC */ 406*bad3ecd0Sotto#ifdef KMEMSTATS 407*bad3ecd0Sotto kup->ku_freecnt++; 408*bad3ecd0Sotto if (kup->ku_freecnt >= kbp->kb_elmpercl) { 409*bad3ecd0Sotto if (kup->ku_freecnt > kbp->kb_elmpercl) 410*bad3ecd0Sotto panic("free: multiple frees"); 411*bad3ecd0Sotto else if (kbp->kb_totalfree > kbp->kb_highwat) 412*bad3ecd0Sotto kbp->kb_couldfree++; 413*bad3ecd0Sotto } 414*bad3ecd0Sotto kbp->kb_totalfree++; 415*bad3ecd0Sotto ksp->ks_memuse -= size; 416*bad3ecd0Sotto if (ksp->ks_memuse + size >= ksp->ks_limit && 417*bad3ecd0Sotto ksp->ks_memuse < ksp->ks_limit) 418*bad3ecd0Sotto wakeup((caddr_t)ksp); 419*bad3ecd0Sotto ksp->ks_inuse--; 420*bad3ecd0Sotto#endif 421*bad3ecd0Sotto if (kbp->kb_next == NULL) 422*bad3ecd0Sotto kbp->kb_next = addr; 423*bad3ecd0Sotto else 424*bad3ecd0Sotto ((struct freelist *)kbp->kb_last)->next = addr; 425*bad3ecd0Sotto freep->next = NULL; 426*bad3ecd0Sotto kbp->kb_last = addr; 427*bad3ecd0Sotto splx(s); 428*bad3ecd0Sotto} 429*bad3ecd0Sotto 430*bad3ecd0Sotto/* 431*bad3ecd0Sotto * Compute the number of pages that kmem_map will map, that is, 432*bad3ecd0Sotto * the size of the kernel malloc arena. 433*bad3ecd0Sotto */ 434*bad3ecd0Sottovoid 435*bad3ecd0Sottokmeminit_nkmempages() 436*bad3ecd0Sotto{ 437*bad3ecd0Sotto int npages; 438*bad3ecd0Sotto 439*bad3ecd0Sotto if (nkmempages != 0) { 440*bad3ecd0Sotto /* 441*bad3ecd0Sotto * It's already been set (by us being here before, or 442*bad3ecd0Sotto * by patching or kernel config options), bail out now. 443*bad3ecd0Sotto */ 444*bad3ecd0Sotto return; 445*bad3ecd0Sotto } 446*bad3ecd0Sotto 447*bad3ecd0Sotto /* 448*bad3ecd0Sotto * We use the following (simple) formula: 449*bad3ecd0Sotto * 450*bad3ecd0Sotto * - Starting point is physical memory / 4. 451*bad3ecd0Sotto * 452*bad3ecd0Sotto * - Clamp it down to NKMEMPAGES_MAX. 453*bad3ecd0Sotto * 454*bad3ecd0Sotto * - Round it up to NKMEMPAGES_MIN. 455*bad3ecd0Sotto */ 456*bad3ecd0Sotto npages = physmem / 4; 457*bad3ecd0Sotto 458*bad3ecd0Sotto if (npages > NKMEMPAGES_MAX) 459*bad3ecd0Sotto npages = NKMEMPAGES_MAX; 460*bad3ecd0Sotto 461*bad3ecd0Sotto if (npages < NKMEMPAGES_MIN) 462*bad3ecd0Sotto npages = NKMEMPAGES_MIN; 463*bad3ecd0Sotto 464*bad3ecd0Sotto nkmempages = npages; 465*bad3ecd0Sotto} 466*bad3ecd0Sotto 467*bad3ecd0Sotto/* 468*bad3ecd0Sotto * Initialize the kernel memory allocator 469*bad3ecd0Sotto */ 470*bad3ecd0Sottovoid 471*bad3ecd0Sottokmeminit() 472*bad3ecd0Sotto{ 473*bad3ecd0Sotto vaddr_t base, limit; 474*bad3ecd0Sotto#ifdef KMEMSTATS 475*bad3ecd0Sotto long indx; 476*bad3ecd0Sotto#endif 477*bad3ecd0Sotto 478*bad3ecd0Sotto#ifdef DIAGNOSTIC 479*bad3ecd0Sotto if (sizeof(struct freelist) > (1 << MINBUCKET)) 480*bad3ecd0Sotto panic("kmeminit: minbucket too small/struct freelist too big"); 481*bad3ecd0Sotto#endif 482*bad3ecd0Sotto 483*bad3ecd0Sotto /* 484*bad3ecd0Sotto * Compute the number of kmem_map pages, if we have not 485*bad3ecd0Sotto * done so already. 486*bad3ecd0Sotto */ 487*bad3ecd0Sotto kmeminit_nkmempages(); 488*bad3ecd0Sotto base = vm_map_min(kernel_map); 489*bad3ecd0Sotto kmem_map = uvm_km_suballoc(kernel_map, &base, &limit, 490*bad3ecd0Sotto (vsize_t)(nkmempages * PAGE_SIZE), VM_MAP_INTRSAFE, FALSE, 491*bad3ecd0Sotto &kmem_map_store.vmi_map); 492*bad3ecd0Sotto kmembase = (char *)base; 493*bad3ecd0Sotto kmemlimit = (char *)limit; 494*bad3ecd0Sotto kmemusage = (struct kmemusage *) uvm_km_zalloc(kernel_map, 495*bad3ecd0Sotto (vsize_t)(nkmempages * sizeof(struct kmemusage))); 496*bad3ecd0Sotto#ifdef KMEMSTATS 497*bad3ecd0Sotto for (indx = 0; indx < MINBUCKET + 16; indx++) { 498*bad3ecd0Sotto if (1 << indx >= PAGE_SIZE) 499*bad3ecd0Sotto bucket[indx].kb_elmpercl = 1; 500*bad3ecd0Sotto else 501*bad3ecd0Sotto bucket[indx].kb_elmpercl = PAGE_SIZE / (1 << indx); 502*bad3ecd0Sotto bucket[indx].kb_highwat = 5 * bucket[indx].kb_elmpercl; 503*bad3ecd0Sotto } 504*bad3ecd0Sotto for (indx = 0; indx < M_LAST; indx++) 505*bad3ecd0Sotto kmemstats[indx].ks_limit = nkmempages * PAGE_SIZE * 6 / 10; 506*bad3ecd0Sotto#endif 507*bad3ecd0Sotto#ifdef MALLOC_DEBUG 508*bad3ecd0Sotto debug_malloc_init(); 509*bad3ecd0Sotto#endif 510*bad3ecd0Sotto} 511*bad3ecd0Sotto 512*bad3ecd0Sotto/* 513*bad3ecd0Sotto * Return kernel malloc statistics information. 514*bad3ecd0Sotto */ 515*bad3ecd0Sottoint 516*bad3ecd0Sottosysctl_malloc(name, namelen, oldp, oldlenp, newp, newlen, p) 517*bad3ecd0Sotto int *name; 518*bad3ecd0Sotto u_int namelen; 519*bad3ecd0Sotto void *oldp; 520*bad3ecd0Sotto size_t *oldlenp; 521*bad3ecd0Sotto void *newp; 522*bad3ecd0Sotto size_t newlen; 523*bad3ecd0Sotto struct proc *p; 524*bad3ecd0Sotto{ 525*bad3ecd0Sotto struct kmembuckets kb; 526*bad3ecd0Sotto int i, siz; 527*bad3ecd0Sotto 528*bad3ecd0Sotto if (namelen != 2 && name[0] != KERN_MALLOC_BUCKETS && 529*bad3ecd0Sotto name[0] != KERN_MALLOC_KMEMNAMES) 530*bad3ecd0Sotto return (ENOTDIR); /* overloaded */ 531*bad3ecd0Sotto 532*bad3ecd0Sotto switch (name[0]) { 533*bad3ecd0Sotto case KERN_MALLOC_BUCKETS: 534*bad3ecd0Sotto /* Initialize the first time */ 535*bad3ecd0Sotto if (buckstring_init == 0) { 536*bad3ecd0Sotto buckstring_init = 1; 537*bad3ecd0Sotto bzero(buckstring, sizeof(buckstring)); 538*bad3ecd0Sotto for (siz = 0, i = MINBUCKET; i < MINBUCKET + 16; i++) { 539*bad3ecd0Sotto snprintf(buckstring + siz, 540*bad3ecd0Sotto sizeof buckstring - siz, 541*bad3ecd0Sotto "%d,", (u_int)(1<<i)); 542*bad3ecd0Sotto siz += strlen(buckstring + siz); 543*bad3ecd0Sotto } 544*bad3ecd0Sotto /* Remove trailing comma */ 545*bad3ecd0Sotto if (siz) 546*bad3ecd0Sotto buckstring[siz - 1] = '\0'; 547*bad3ecd0Sotto } 548*bad3ecd0Sotto return (sysctl_rdstring(oldp, oldlenp, newp, buckstring)); 549*bad3ecd0Sotto 550*bad3ecd0Sotto case KERN_MALLOC_BUCKET: 551*bad3ecd0Sotto bcopy(&bucket[BUCKETINDX(name[1])], &kb, sizeof(kb)); 552*bad3ecd0Sotto kb.kb_next = kb.kb_last = 0; 553*bad3ecd0Sotto return (sysctl_rdstruct(oldp, oldlenp, newp, &kb, sizeof(kb))); 554*bad3ecd0Sotto case KERN_MALLOC_KMEMSTATS: 555*bad3ecd0Sotto#ifdef KMEMSTATS 556*bad3ecd0Sotto if ((name[1] < 0) || (name[1] >= M_LAST)) 557*bad3ecd0Sotto return (EINVAL); 558*bad3ecd0Sotto return (sysctl_rdstruct(oldp, oldlenp, newp, 559*bad3ecd0Sotto &kmemstats[name[1]], sizeof(struct kmemstats))); 560*bad3ecd0Sotto#else 561*bad3ecd0Sotto return (EOPNOTSUPP); 562*bad3ecd0Sotto#endif 563*bad3ecd0Sotto case KERN_MALLOC_KMEMNAMES: 564*bad3ecd0Sotto#if defined(KMEMSTATS) || defined(DIAGNOSTIC) || defined(FFS_SOFTUPDATES) 565*bad3ecd0Sotto if (memall == NULL) { 566*bad3ecd0Sotto int totlen; 567*bad3ecd0Sotto 568*bad3ecd0Sotto i = lockmgr(&sysctl_kmemlock, LK_EXCLUSIVE, NULL, p); 569*bad3ecd0Sotto if (i) 570*bad3ecd0Sotto return (i); 571*bad3ecd0Sotto 572*bad3ecd0Sotto /* Figure out how large a buffer we need */ 573*bad3ecd0Sotto for (totlen = 0, i = 0; i < M_LAST; i++) { 574*bad3ecd0Sotto if (memname[i]) 575*bad3ecd0Sotto totlen += strlen(memname[i]); 576*bad3ecd0Sotto totlen++; 577*bad3ecd0Sotto } 578*bad3ecd0Sotto memall = malloc(totlen + M_LAST, M_SYSCTL, M_WAITOK); 579*bad3ecd0Sotto bzero(memall, totlen + M_LAST); 580*bad3ecd0Sotto for (siz = 0, i = 0; i < M_LAST; i++) { 581*bad3ecd0Sotto snprintf(memall + siz, 582*bad3ecd0Sotto totlen + M_LAST - siz, 583*bad3ecd0Sotto "%s,", memname[i] ? memname[i] : ""); 584*bad3ecd0Sotto siz += strlen(memall + siz); 585*bad3ecd0Sotto } 586*bad3ecd0Sotto /* Remove trailing comma */ 587*bad3ecd0Sotto if (siz) 588*bad3ecd0Sotto memall[siz - 1] = '\0'; 589*bad3ecd0Sotto 590*bad3ecd0Sotto /* Now, convert all spaces to underscores */ 591*bad3ecd0Sotto for (i = 0; i < totlen; i++) 592*bad3ecd0Sotto if (memall[i] == ' ') 593*bad3ecd0Sotto memall[i] = '_'; 594*bad3ecd0Sotto lockmgr(&sysctl_kmemlock, LK_RELEASE, NULL, p); 595*bad3ecd0Sotto } 596*bad3ecd0Sotto return (sysctl_rdstring(oldp, oldlenp, newp, memall)); 597*bad3ecd0Sotto#else 598*bad3ecd0Sotto return (EOPNOTSUPP); 599*bad3ecd0Sotto#endif 600*bad3ecd0Sotto default: 601*bad3ecd0Sotto return (EOPNOTSUPP); 602*bad3ecd0Sotto } 603*bad3ecd0Sotto /* NOTREACHED */ 604*bad3ecd0Sotto} 605*bad3ecd0Sotto 606*bad3ecd0Sotto/* 607*bad3ecd0Sotto * Round up a size to how much malloc would actually allocate. 608*bad3ecd0Sotto */ 609*bad3ecd0Sottosize_t 610*bad3ecd0Sottomalloc_roundup(size_t sz) 611*bad3ecd0Sotto{ 612*bad3ecd0Sotto if (sz > MAXALLOCSAVE) 613*bad3ecd0Sotto return round_page(sz); 614*bad3ecd0Sotto 615*bad3ecd0Sotto return (1 << BUCKETINDX(sz)); 616*bad3ecd0Sotto} 617