1 /* 2 * Copyright (c) 1997, 1998 John S. Dyson 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice immediately at the beginning of the file, without modification, 10 * this list of conditions, and the following disclaimer. 11 * 2. Absolutely no warranty of function or purpose is made by the author 12 * John S. Dyson. 13 * 14 * $FreeBSD: src/sys/vm/vm_zone.c,v 1.30.2.6 2002/10/10 19:50:16 dillon Exp $ 15 * $DragonFly: src/sys/vm/vm_zone.c,v 1.2 2003/06/17 04:29:00 dillon Exp $ 16 */ 17 18 #include <sys/param.h> 19 #include <sys/systm.h> 20 #include <sys/kernel.h> 21 #include <sys/lock.h> 22 #include <sys/malloc.h> 23 #include <sys/sysctl.h> 24 #include <sys/vmmeter.h> 25 26 #include <vm/vm.h> 27 #include <vm/vm_object.h> 28 #include <vm/vm_page.h> 29 #include <vm/vm_map.h> 30 #include <vm/vm_kern.h> 31 #include <vm/vm_extern.h> 32 #include <vm/vm_zone.h> 33 34 static MALLOC_DEFINE(M_ZONE, "ZONE", "Zone header"); 35 36 #define ZONE_ERROR_INVALID 0 37 #define ZONE_ERROR_NOTFREE 1 38 #define ZONE_ERROR_ALREADYFREE 2 39 40 #define ZONE_ROUNDING 32 41 42 #define ZENTRY_FREE 0x12342378 43 /* 44 * void *zalloc(vm_zone_t zone) -- 45 * Returns an item from a specified zone. 46 * 47 * void zfree(vm_zone_t zone, void *item) -- 48 * Frees an item back to a specified zone. 49 */ 50 static __inline__ void * 51 _zalloc(vm_zone_t z) 52 { 53 void *item; 54 55 #ifdef INVARIANTS 56 if (z == 0) 57 zerror(ZONE_ERROR_INVALID); 58 #endif 59 60 if (z->zfreecnt <= z->zfreemin) { 61 item = _zget(z); 62 /* 63 * PANICFAIL allows the caller to assume that the zalloc() 64 * will always succeed. If it doesn't, we panic here. 65 */ 66 if (item == NULL && (z->zflags & ZONE_PANICFAIL)) 67 panic("zalloc(%s) failed", z->zname); 68 return(item); 69 } 70 71 item = z->zitems; 72 z->zitems = ((void **) item)[0]; 73 #ifdef INVARIANTS 74 KASSERT(item != NULL, ("zitems unexpectedly NULL")); 75 if (((void **) item)[1] != (void *) ZENTRY_FREE) 76 zerror(ZONE_ERROR_NOTFREE); 77 ((void **) item)[1] = 0; 78 #endif 79 80 z->zfreecnt--; 81 z->znalloc++; 82 return item; 83 } 84 85 static __inline__ void 86 _zfree(vm_zone_t z, void *item) 87 { 88 ((void **) item)[0] = z->zitems; 89 #ifdef INVARIANTS 90 if (((void **) item)[1] == (void *) ZENTRY_FREE) 91 zerror(ZONE_ERROR_ALREADYFREE); 92 ((void **) item)[1] = (void *) ZENTRY_FREE; 93 #endif 94 z->zitems = item; 95 z->zfreecnt++; 96 } 97 98 /* 99 * This file comprises a very simple zone allocator. This is used 100 * in lieu of the malloc allocator, where needed or more optimal. 101 * 102 * Note that the initial implementation of this had coloring, and 103 * absolutely no improvement (actually perf degradation) occurred. 104 * 105 * Note also that the zones are type stable. The only restriction is 106 * that the first two longwords of a data structure can be changed 107 * between allocations. Any data that must be stable between allocations 108 * must reside in areas after the first two longwords. 109 * 110 * zinitna, zinit, zbootinit are the initialization routines. 111 * zalloc, zfree, are the interrupt/lock unsafe allocation/free routines. 112 * zalloci, zfreei, are the interrupt/lock safe allocation/free routines. 113 */ 114 115 static struct vm_zone *zlist; 116 static int sysctl_vm_zone(SYSCTL_HANDLER_ARGS); 117 static int zone_kmem_pages, zone_kern_pages, zone_kmem_kvaspace; 118 119 /* 120 * Create a zone, but don't allocate the zone structure. If the 121 * zone had been previously created by the zone boot code, initialize 122 * various parts of the zone code. 123 * 124 * If waits are not allowed during allocation (e.g. during interrupt 125 * code), a-priori allocate the kernel virtual space, and allocate 126 * only pages when needed. 127 * 128 * Arguments: 129 * z pointer to zone structure. 130 * obj pointer to VM object (opt). 131 * name name of zone. 132 * size size of zone entries. 133 * nentries number of zone entries allocated (only ZONE_INTERRUPT.) 134 * flags ZONE_INTERRUPT -- items can be allocated at interrupt time. 135 * zalloc number of pages allocated when memory is needed. 136 * 137 * Note that when using ZONE_INTERRUPT, the size of the zone is limited 138 * by the nentries argument. The size of the memory allocatable is 139 * unlimited if ZONE_INTERRUPT is not set. 140 * 141 */ 142 int 143 zinitna(vm_zone_t z, vm_object_t obj, char *name, int size, 144 int nentries, int flags, int zalloc) 145 { 146 int totsize; 147 148 if ((z->zflags & ZONE_BOOT) == 0) { 149 z->zsize = (size + ZONE_ROUNDING - 1) & ~(ZONE_ROUNDING - 1); 150 simple_lock_init(&z->zlock); 151 z->zfreecnt = 0; 152 z->ztotal = 0; 153 z->zmax = 0; 154 z->zname = name; 155 z->znalloc = 0; 156 z->zitems = NULL; 157 158 z->znext = zlist; 159 zlist = z; 160 } 161 162 z->zflags |= flags; 163 164 /* 165 * If we cannot wait, allocate KVA space up front, and we will fill 166 * in pages as needed. 167 */ 168 if (z->zflags & ZONE_INTERRUPT) { 169 170 totsize = round_page(z->zsize * nentries); 171 zone_kmem_kvaspace += totsize; 172 173 z->zkva = kmem_alloc_pageable(kernel_map, totsize); 174 if (z->zkva == 0) { 175 zlist = z->znext; 176 return 0; 177 } 178 179 z->zpagemax = totsize / PAGE_SIZE; 180 if (obj == NULL) { 181 z->zobj = vm_object_allocate(OBJT_DEFAULT, z->zpagemax); 182 } else { 183 z->zobj = obj; 184 _vm_object_allocate(OBJT_DEFAULT, z->zpagemax, obj); 185 } 186 z->zallocflag = VM_ALLOC_INTERRUPT; 187 z->zmax += nentries; 188 } else { 189 z->zallocflag = VM_ALLOC_SYSTEM; 190 z->zmax = 0; 191 } 192 193 194 if (z->zsize > PAGE_SIZE) 195 z->zfreemin = 1; 196 else 197 z->zfreemin = PAGE_SIZE / z->zsize; 198 199 z->zpagecount = 0; 200 if (zalloc) 201 z->zalloc = zalloc; 202 else 203 z->zalloc = 1; 204 205 return 1; 206 } 207 208 /* 209 * Subroutine same as zinitna, except zone data structure is allocated 210 * automatically by malloc. This routine should normally be used, except 211 * in certain tricky startup conditions in the VM system -- then 212 * zbootinit and zinitna can be used. Zinit is the standard zone 213 * initialization call. 214 */ 215 vm_zone_t 216 zinit(char *name, int size, int nentries, int flags, int zalloc) 217 { 218 vm_zone_t z; 219 220 z = (vm_zone_t) malloc(sizeof (struct vm_zone), M_ZONE, M_NOWAIT); 221 if (z == NULL) 222 return NULL; 223 224 z->zflags = 0; 225 if (zinitna(z, NULL, name, size, nentries, flags, zalloc) == 0) { 226 free(z, M_ZONE); 227 return NULL; 228 } 229 230 return z; 231 } 232 233 /* 234 * Initialize a zone before the system is fully up. This routine should 235 * only be called before full VM startup. 236 */ 237 void 238 zbootinit(vm_zone_t z, char *name, int size, void *item, int nitems) 239 { 240 int i; 241 242 z->zname = name; 243 z->zsize = size; 244 z->zpagemax = 0; 245 z->zobj = NULL; 246 z->zflags = ZONE_BOOT; 247 z->zfreemin = 0; 248 z->zallocflag = 0; 249 z->zpagecount = 0; 250 z->zalloc = 0; 251 z->znalloc = 0; 252 simple_lock_init(&z->zlock); 253 254 bzero(item, nitems * z->zsize); 255 z->zitems = NULL; 256 for (i = 0; i < nitems; i++) { 257 ((void **) item)[0] = z->zitems; 258 #ifdef INVARIANTS 259 ((void **) item)[1] = (void *) ZENTRY_FREE; 260 #endif 261 z->zitems = item; 262 (char *) item += z->zsize; 263 } 264 z->zfreecnt = nitems; 265 z->zmax = nitems; 266 z->ztotal = nitems; 267 268 if (zlist == 0) { 269 zlist = z; 270 } else { 271 z->znext = zlist; 272 zlist = z; 273 } 274 } 275 276 /* 277 * Zone critical region locks. 278 */ 279 static __inline int 280 zlock(vm_zone_t z) 281 { 282 int s; 283 284 s = splhigh(); 285 simple_lock(&z->zlock); 286 return s; 287 } 288 289 static __inline void 290 zunlock(vm_zone_t z, int s) 291 { 292 simple_unlock(&z->zlock); 293 splx(s); 294 } 295 296 /* 297 * void *zalloc(vm_zone_t zone) -- 298 * Returns an item from a specified zone. 299 * 300 * void zfree(vm_zone_t zone, void *item) -- 301 * Frees an item back to a specified zone. 302 * 303 * void *zalloci(vm_zone_t zone) -- 304 * Returns an item from a specified zone, interrupt safe. 305 * 306 * void zfreei(vm_zone_t zone, void *item) -- 307 * Frees an item back to a specified zone, interrupt safe. 308 * 309 */ 310 311 void * 312 zalloc(vm_zone_t z) 313 { 314 #if defined(SMP) 315 return zalloci(z); 316 #else 317 return _zalloc(z); 318 #endif 319 } 320 321 void 322 zfree(vm_zone_t z, void *item) 323 { 324 #ifdef SMP 325 zfreei(z, item); 326 #else 327 _zfree(z, item); 328 #endif 329 } 330 331 /* 332 * Zone allocator/deallocator. These are interrupt / (or potentially SMP) 333 * safe. The raw zalloc/zfree routines are not interrupt safe, but are fast. 334 */ 335 void * 336 zalloci(vm_zone_t z) 337 { 338 int s; 339 void *item; 340 341 s = zlock(z); 342 item = _zalloc(z); 343 zunlock(z, s); 344 return item; 345 } 346 347 void 348 zfreei(vm_zone_t z, void *item) 349 { 350 int s; 351 352 s = zlock(z); 353 _zfree(z, item); 354 zunlock(z, s); 355 return; 356 } 357 358 /* 359 * Internal zone routine. Not to be called from external (non vm_zone) code. 360 */ 361 void * 362 _zget(vm_zone_t z) 363 { 364 int i; 365 vm_page_t m; 366 int nitems, nbytes; 367 void *item; 368 369 if (z == NULL) 370 panic("zget: null zone"); 371 372 if (z->zflags & ZONE_INTERRUPT) { 373 nbytes = z->zpagecount * PAGE_SIZE; 374 nbytes -= nbytes % z->zsize; 375 item = (char *) z->zkva + nbytes; 376 for (i = 0; ((i < z->zalloc) && (z->zpagecount < z->zpagemax)); 377 i++) { 378 vm_offset_t zkva; 379 380 m = vm_page_alloc(z->zobj, z->zpagecount, 381 z->zallocflag); 382 if (m == NULL) 383 break; 384 385 zkva = z->zkva + z->zpagecount * PAGE_SIZE; 386 pmap_kenter(zkva, VM_PAGE_TO_PHYS(m)); 387 bzero((caddr_t) zkva, PAGE_SIZE); 388 z->zpagecount++; 389 zone_kmem_pages++; 390 cnt.v_wire_count++; 391 } 392 nitems = ((z->zpagecount * PAGE_SIZE) - nbytes) / z->zsize; 393 } else { 394 nbytes = z->zalloc * PAGE_SIZE; 395 396 /* 397 * Check to see if the kernel map is already locked. We could allow 398 * for recursive locks, but that eliminates a valuable debugging 399 * mechanism, and opens up the kernel map for potential corruption 400 * by inconsistent data structure manipulation. We could also use 401 * the interrupt allocation mechanism, but that has size limitations. 402 * Luckily, we have kmem_map that is a submap of kernel map available 403 * for memory allocation, and manipulation of that map doesn't affect 404 * the kernel map structures themselves. 405 * 406 * We can wait, so just do normal map allocation in the appropriate 407 * map. 408 */ 409 if (lockstatus(&kernel_map->lock, NULL)) { 410 int s; 411 s = splvm(); 412 #ifdef SMP 413 simple_unlock(&z->zlock); 414 #endif 415 item = (void *) kmem_malloc(kmem_map, nbytes, M_WAITOK); 416 #ifdef SMP 417 simple_lock(&z->zlock); 418 #endif 419 if (item != NULL) 420 zone_kmem_pages += z->zalloc; 421 splx(s); 422 } else { 423 #ifdef SMP 424 simple_unlock(&z->zlock); 425 #endif 426 item = (void *) kmem_alloc(kernel_map, nbytes); 427 #ifdef SMP 428 simple_lock(&z->zlock); 429 #endif 430 if (item != NULL) 431 zone_kern_pages += z->zalloc; 432 } 433 if (item != NULL) { 434 bzero(item, nbytes); 435 } else { 436 nbytes = 0; 437 } 438 nitems = nbytes / z->zsize; 439 } 440 z->ztotal += nitems; 441 442 /* 443 * Save one for immediate allocation 444 */ 445 if (nitems != 0) { 446 nitems -= 1; 447 for (i = 0; i < nitems; i++) { 448 ((void **) item)[0] = z->zitems; 449 #ifdef INVARIANTS 450 ((void **) item)[1] = (void *) ZENTRY_FREE; 451 #endif 452 z->zitems = item; 453 (char *) item += z->zsize; 454 } 455 z->zfreecnt += nitems; 456 z->znalloc++; 457 } else if (z->zfreecnt > 0) { 458 item = z->zitems; 459 z->zitems = ((void **) item)[0]; 460 #ifdef INVARIANTS 461 if (((void **) item)[1] != (void *) ZENTRY_FREE) 462 zerror(ZONE_ERROR_NOTFREE); 463 ((void **) item)[1] = 0; 464 #endif 465 z->zfreecnt--; 466 z->znalloc++; 467 } else { 468 item = NULL; 469 } 470 471 return item; 472 } 473 474 static int 475 sysctl_vm_zone(SYSCTL_HANDLER_ARGS) 476 { 477 int error=0; 478 vm_zone_t curzone, nextzone; 479 char tmpbuf[128]; 480 char tmpname[14]; 481 482 snprintf(tmpbuf, sizeof(tmpbuf), 483 "\nITEM SIZE LIMIT USED FREE REQUESTS\n"); 484 error = SYSCTL_OUT(req, tmpbuf, strlen(tmpbuf)); 485 if (error) 486 return (error); 487 488 for (curzone = zlist; curzone; curzone = nextzone) { 489 int i; 490 int len; 491 int offset; 492 493 nextzone = curzone->znext; 494 len = strlen(curzone->zname); 495 if (len >= (sizeof(tmpname) - 1)) 496 len = (sizeof(tmpname) - 1); 497 for(i = 0; i < sizeof(tmpname) - 1; i++) 498 tmpname[i] = ' '; 499 tmpname[i] = 0; 500 memcpy(tmpname, curzone->zname, len); 501 tmpname[len] = ':'; 502 offset = 0; 503 if (curzone == zlist) { 504 offset = 1; 505 tmpbuf[0] = '\n'; 506 } 507 508 snprintf(tmpbuf + offset, sizeof(tmpbuf) - offset, 509 "%s %6.6u, %8.8u, %6.6u, %6.6u, %8.8u\n", 510 tmpname, curzone->zsize, curzone->zmax, 511 (curzone->ztotal - curzone->zfreecnt), 512 curzone->zfreecnt, curzone->znalloc); 513 514 len = strlen((char *)tmpbuf); 515 if (nextzone == NULL) 516 tmpbuf[len - 1] = 0; 517 518 error = SYSCTL_OUT(req, tmpbuf, len); 519 520 if (error) 521 return (error); 522 } 523 return (0); 524 } 525 526 #ifdef INVARIANT_SUPPORT 527 void 528 zerror(int error) 529 { 530 char *msg; 531 532 switch (error) { 533 case ZONE_ERROR_INVALID: 534 msg = "zone: invalid zone"; 535 break; 536 case ZONE_ERROR_NOTFREE: 537 msg = "zone: entry not free"; 538 break; 539 case ZONE_ERROR_ALREADYFREE: 540 msg = "zone: freeing free entry"; 541 break; 542 default: 543 msg = "zone: invalid error"; 544 break; 545 } 546 panic(msg); 547 } 548 #endif 549 550 SYSCTL_OID(_vm, OID_AUTO, zone, CTLTYPE_STRING|CTLFLAG_RD, \ 551 NULL, 0, sysctl_vm_zone, "A", "Zone Info"); 552 553 SYSCTL_INT(_vm, OID_AUTO, zone_kmem_pages, 554 CTLFLAG_RD, &zone_kmem_pages, 0, "Number of interrupt safe pages allocated by zone"); 555 SYSCTL_INT(_vm, OID_AUTO, zone_kmem_kvaspace, 556 CTLFLAG_RD, &zone_kmem_kvaspace, 0, "KVA space allocated by zone"); 557 SYSCTL_INT(_vm, OID_AUTO, zone_kern_pages, 558 CTLFLAG_RD, &zone_kern_pages, 0, "Number of non-interrupt safe pages allocated by zone"); 559