1 /* 2 * Copyright (c) 1985, Avadis Tevanian, Jr., Michael Wayne Young 3 * Copyright (c) 1987 Carnegie-Mellon University 4 * Copyright (c) 1991 Regents of the University of California. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to Berkeley by 8 * The Mach Operating System project at Carnegie-Mellon University. 9 * 10 * The CMU software License Agreement specifies the terms and conditions 11 * for use and redistribution. 12 * 13 * @(#)vm_kern.c 7.1 (Berkeley) 12/05/90 14 */ 15 16 /* 17 * Kernel memory management. 18 */ 19 20 #include "types.h" 21 22 #include "../vm/vm_param.h" 23 #include "../vm/vm_map.h" 24 #include "../vm/vm_page.h" 25 #include "../vm/vm_pageout.h" 26 #include "../vm/vm_kern.h" 27 28 /* 29 * kmem_alloc_pageable: 30 * 31 * Allocate pageable memory to the kernel's address map. 32 * map must be "kernel_map" below. 33 */ 34 35 vm_offset_t kmem_alloc_pageable(map, size) 36 vm_map_t map; 37 register vm_size_t size; 38 { 39 vm_offset_t addr; 40 register int result; 41 42 #if 0 43 if (map != kernel_map) 44 panic("kmem_alloc_pageable: not called with kernel_map"); 45 #endif 0 46 47 size = round_page(size); 48 49 addr = vm_map_min(map); 50 result = vm_map_find(map, VM_OBJECT_NULL, (vm_offset_t) 0, 51 &addr, size, TRUE); 52 if (result != KERN_SUCCESS) { 53 return(0); 54 } 55 56 return(addr); 57 } 58 59 /* 60 * Allocate wired-down memory in the kernel's address map 61 * or a submap. 62 */ 63 vm_offset_t kmem_alloc(map, size) 64 register vm_map_t map; 65 register vm_size_t size; 66 { 67 vm_offset_t addr; 68 register int result; 69 register vm_offset_t offset; 70 extern vm_object_t kernel_object; 71 vm_offset_t i; 72 73 size = round_page(size); 74 75 /* 76 * Use the kernel object for wired-down kernel pages. 77 * Assume that no region of the kernel object is 78 * referenced more than once. 79 */ 80 81 addr = vm_map_min(map); 82 result = vm_map_find(map, VM_OBJECT_NULL, (vm_offset_t) 0, 83 &addr, size, TRUE); 84 if (result != KERN_SUCCESS) { 85 return(0); 86 } 87 88 /* 89 * Since we didn't know where the new region would 90 * start, we couldn't supply the correct offset into 91 * the kernel object. Re-allocate that address 92 * region with the correct offset. 93 */ 94 95 offset = addr - VM_MIN_KERNEL_ADDRESS; 96 vm_object_reference(kernel_object); 97 98 vm_map_lock(map); 99 vm_map_delete(map, addr, addr + size); 100 vm_map_insert(map, kernel_object, offset, addr, addr + size); 101 vm_map_unlock(map); 102 103 /* 104 * Guarantee that there are pages already in this object 105 * before calling vm_map_pageable. This is to prevent the 106 * following scenario: 107 * 108 * 1) Threads have swapped out, so that there is a 109 * pager for the kernel_object. 110 * 2) The kmsg zone is empty, and so we are kmem_allocing 111 * a new page for it. 112 * 3) vm_map_pageable calls vm_fault; there is no page, 113 * but there is a pager, so we call 114 * pager_data_request. But the kmsg zone is empty, 115 * so we must kmem_alloc. 116 * 4) goto 1 117 * 5) Even if the kmsg zone is not empty: when we get 118 * the data back from the pager, it will be (very 119 * stale) non-zero data. kmem_alloc is defined to 120 * return zero-filled memory. 121 * 122 * We're intentionally not activating the pages we allocate 123 * to prevent a race with page-out. vm_map_pageable will wire 124 * the pages. 125 */ 126 127 vm_object_lock(kernel_object); 128 for (i = 0 ; i < size; i+= PAGE_SIZE) { 129 vm_page_t mem; 130 131 while ((mem = vm_page_alloc(kernel_object, offset+i)) 132 == VM_PAGE_NULL) { 133 vm_object_unlock(kernel_object); 134 VM_WAIT; 135 vm_object_lock(kernel_object); 136 } 137 vm_page_zero_fill(mem); 138 mem->busy = FALSE; 139 } 140 vm_object_unlock(kernel_object); 141 142 /* 143 * And finally, mark the data as non-pageable. 144 */ 145 146 (void) vm_map_pageable(map, (vm_offset_t) addr, addr + size, FALSE); 147 148 /* 149 * Try to coalesce the map 150 */ 151 152 vm_map_simplify(map, addr); 153 154 return(addr); 155 } 156 157 /* 158 * kmem_free: 159 * 160 * Release a region of kernel virtual memory allocated 161 * with kmem_alloc, and return the physical pages 162 * associated with that region. 163 */ 164 void kmem_free(map, addr, size) 165 vm_map_t map; 166 register vm_offset_t addr; 167 vm_size_t size; 168 { 169 (void) vm_map_remove(map, trunc_page(addr), round_page(addr + size)); 170 } 171 172 /* 173 * kmem_suballoc: 174 * 175 * Allocates a map to manage a subrange 176 * of the kernel virtual address space. 177 * 178 * Arguments are as follows: 179 * 180 * parent Map to take range from 181 * size Size of range to find 182 * min, max Returned endpoints of map 183 * pageable Can the region be paged 184 */ 185 vm_map_t kmem_suballoc(parent, min, max, size, pageable) 186 register vm_map_t parent; 187 vm_offset_t *min, *max; 188 register vm_size_t size; 189 boolean_t pageable; 190 { 191 register int ret; 192 vm_map_t result; 193 194 size = round_page(size); 195 196 *min = (vm_offset_t) vm_map_min(parent); 197 ret = vm_map_find(parent, VM_OBJECT_NULL, (vm_offset_t) 0, 198 min, size, TRUE); 199 if (ret != KERN_SUCCESS) { 200 printf("kmem_suballoc: bad status return of %d.\n", ret); 201 panic("kmem_suballoc"); 202 } 203 *max = *min + size; 204 pmap_reference(vm_map_pmap(parent)); 205 result = vm_map_create(vm_map_pmap(parent), *min, *max, pageable); 206 if (result == VM_MAP_NULL) 207 panic("kmem_suballoc: cannot create submap"); 208 if ((ret = vm_map_submap(parent, *min, *max, result)) != KERN_SUCCESS) 209 panic("kmem_suballoc: unable to change range to submap"); 210 return(result); 211 } 212 213 /* 214 * vm_move: 215 * 216 * Move memory from source to destination map, possibly deallocating 217 * the source map reference to the memory. 218 * 219 * Parameters are as follows: 220 * 221 * src_map Source address map 222 * src_addr Address within source map 223 * dst_map Destination address map 224 * num_bytes Amount of data (in bytes) to copy/move 225 * src_dealloc Should source be removed after copy? 226 * 227 * Assumes the src and dst maps are not already locked. 228 * 229 * Returns new destination address or 0 (if a failure occurs). 230 */ 231 vm_offset_t vm_move(src_map,src_addr,dst_map,num_bytes,src_dealloc) 232 vm_map_t src_map; 233 register vm_offset_t src_addr; 234 register vm_map_t dst_map; 235 vm_offset_t num_bytes; 236 boolean_t src_dealloc; 237 { 238 register vm_offset_t src_start; /* Beginning of region */ 239 register vm_size_t src_size; /* Size of rounded region */ 240 vm_offset_t dst_start; /* destination address */ 241 register int result; 242 243 /* 244 * Page-align the source region 245 */ 246 247 src_start = trunc_page(src_addr); 248 src_size = round_page(src_addr + num_bytes) - src_start; 249 250 /* 251 * If there's no destination, we can be at most deallocating 252 * the source range. 253 */ 254 if (dst_map == VM_MAP_NULL) { 255 if (src_dealloc) 256 if (vm_deallocate(src_map, src_start, src_size) 257 != KERN_SUCCESS) { 258 printf("vm_move: deallocate of source"); 259 printf(" failed, dealloc_only clause\n"); 260 } 261 return(0); 262 } 263 264 /* 265 * Allocate a place to put the copy 266 */ 267 268 dst_start = (vm_offset_t) 0; 269 if ((result = vm_allocate(dst_map, &dst_start, src_size, TRUE)) 270 == KERN_SUCCESS) { 271 /* 272 * Perform the copy, asking for deallocation if desired 273 */ 274 result = vm_map_copy(dst_map, src_map, dst_start, src_size, 275 src_start, FALSE, src_dealloc); 276 } 277 278 /* 279 * Return the destination address corresponding to 280 * the source address given (rather than the front 281 * of the newly-allocated page). 282 */ 283 284 if (result == KERN_SUCCESS) 285 return(dst_start + (src_addr - src_start)); 286 return(0); 287 } 288 289 /* 290 * Allocate wired-down memory in the kernel's address map for the higher 291 * level kernel memory allocator (kern/kern_malloc.c). We cannot use 292 * kmem_alloc() because we may need to allocate memory at interrupt 293 * level where we cannot block (canwait == FALSE). 294 * 295 * This routine has its own private kernel submap (kmem_map) and object 296 * (kmem_object). This, combined with the fact that only malloc uses 297 * this routine, ensures that we will never block in map or object waits. 298 * 299 * Note that this still only works in a uni-processor environment and 300 * when called at splhigh(). 301 * 302 * We don't worry about expanding the map (adding entries) since entries 303 * for wired maps are statically allocated. 304 */ 305 vm_offset_t 306 kmem_malloc(map, size, canwait) 307 register vm_map_t map; 308 register vm_size_t size; 309 boolean_t canwait; 310 { 311 register vm_offset_t offset, i; 312 vm_map_entry_t entry; 313 vm_offset_t addr; 314 vm_page_t m; 315 extern vm_object_t kmem_object; 316 317 if (map != kmem_map && map != mb_map) 318 panic("kern_malloc_alloc: map != {kmem,mb}_map"); 319 320 size = round_page(size); 321 addr = vm_map_min(map); 322 323 if (vm_map_find(map, VM_OBJECT_NULL, (vm_offset_t)0, 324 &addr, size, TRUE) != KERN_SUCCESS) 325 return(0); 326 327 /* 328 * Since we didn't know where the new region would start, 329 * we couldn't supply the correct offset into the kmem object. 330 * Re-allocate that address region with the correct offset. 331 */ 332 offset = addr - vm_map_min(kmem_map); 333 vm_object_reference(kmem_object); 334 335 vm_map_lock(map); 336 vm_map_delete(map, addr, addr + size); 337 vm_map_insert(map, kmem_object, offset, addr, addr + size); 338 339 /* 340 * If we can wait, just mark the range as wired 341 * (will fault pages as necessary). 342 */ 343 if (canwait) { 344 vm_map_unlock(map); 345 (void) vm_map_pageable(map, (vm_offset_t) addr, addr + size, 346 FALSE); 347 vm_map_simplify(map, addr); 348 return(addr); 349 } 350 351 /* 352 * If we cannot wait then we must allocate all memory up front, 353 * pulling it off the active queue to prevent pageout. 354 */ 355 vm_object_lock(kmem_object); 356 for (i = 0; i < size; i += PAGE_SIZE) { 357 m = vm_page_alloc(kmem_object, offset + i); 358 359 /* 360 * Ran out of space, free everything up and return. 361 * Don't need to lock page queues here as we know 362 * that the pages we got aren't on any queues. 363 */ 364 if (m == VM_PAGE_NULL) { 365 while (i != 0) { 366 i -= PAGE_SIZE; 367 m = vm_page_lookup(kmem_object, offset + i); 368 vm_page_free(m); 369 } 370 vm_object_unlock(kmem_object); 371 vm_map_delete(map, addr, addr + size); 372 vm_map_unlock(map); 373 return(0); 374 } 375 #if 0 376 vm_page_zero_fill(m); 377 #endif 378 m->busy = FALSE; 379 } 380 vm_object_unlock(kmem_object); 381 382 /* 383 * Mark map entry as non-pageable. 384 * Assert: vm_map_insert() will never be able to extend the previous 385 * entry so there will be a new entry exactly corresponding to this 386 * address range and it will have wired_count == 0. 387 */ 388 if (!vm_map_lookup_entry(map, addr, &entry) || 389 entry->start != addr || entry->end != addr + size || 390 entry->wired_count) 391 panic("kmem_malloc: entry not found or misaligned"); 392 entry->wired_count++; 393 394 /* 395 * Loop thru pages, entering them in the pmap. 396 * (We cannot add them to the wired count without 397 * wrapping the vm_page_queue_lock in splimp...) 398 */ 399 for (i = 0; i < size; i += PAGE_SIZE) { 400 vm_object_lock(kmem_object); 401 m = vm_page_lookup(kmem_object, offset + i); 402 vm_object_unlock(kmem_object); 403 pmap_enter(map->pmap, addr + i, VM_PAGE_TO_PHYS(m), 404 VM_PROT_DEFAULT, TRUE); 405 } 406 vm_map_unlock(map); 407 408 vm_map_simplify(map, addr); 409 return(addr); 410 } 411 412 /* 413 * kmem_alloc_wait 414 * 415 * Allocates pageable memory from a sub-map of the kernel. If the submap 416 * has no room, the caller sleeps waiting for more memory in the submap. 417 * 418 */ 419 vm_offset_t kmem_alloc_wait(map, size) 420 vm_map_t map; 421 vm_size_t size; 422 { 423 vm_offset_t addr; 424 int result; 425 426 size = round_page(size); 427 428 do { 429 /* 430 * To make this work for more than one map, 431 * use the map's lock to lock out sleepers/wakers. 432 * Unfortunately, vm_map_find also grabs the map lock. 433 */ 434 vm_map_lock(map); 435 lock_set_recursive(&map->lock); 436 437 addr = vm_map_min(map); 438 result = vm_map_find(map, VM_OBJECT_NULL, (vm_offset_t) 0, 439 &addr, size, TRUE); 440 441 lock_clear_recursive(&map->lock); 442 if (result != KERN_SUCCESS) { 443 444 if ( (vm_map_max(map) - vm_map_min(map)) < size ) { 445 vm_map_unlock(map); 446 return(0); 447 } 448 449 assert_wait((int)map, TRUE); 450 vm_map_unlock(map); 451 thread_block(); 452 } 453 else { 454 vm_map_unlock(map); 455 } 456 457 } while (result != KERN_SUCCESS); 458 459 return(addr); 460 } 461 462 /* 463 * kmem_free_wakeup 464 * 465 * Returns memory to a submap of the kernel, and wakes up any threads 466 * waiting for memory in that map. 467 */ 468 void kmem_free_wakeup(map, addr, size) 469 vm_map_t map; 470 vm_offset_t addr; 471 vm_size_t size; 472 { 473 vm_map_lock(map); 474 (void) vm_map_delete(map, trunc_page(addr), round_page(addr + size)); 475 thread_wakeup((int)map); 476 vm_map_unlock(map); 477 } 478 479 /* 480 * kmem_init: 481 * 482 * Initialize the kernel's virtual memory map, taking 483 * into account all memory allocated up to this time. 484 */ 485 void kmem_init(start, end) 486 vm_offset_t start; 487 vm_offset_t end; 488 { 489 vm_offset_t addr; 490 extern vm_map_t kernel_map; 491 492 addr = VM_MIN_KERNEL_ADDRESS; 493 kernel_map = vm_map_create(pmap_kernel(), addr, end, FALSE); 494 (void) vm_map_find(kernel_map, VM_OBJECT_NULL, (vm_offset_t) 0, 495 &addr, (start - VM_MIN_KERNEL_ADDRESS), 496 FALSE); 497 } 498