1 /*
2 * Copyright (c) 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 *
5 * This code is derived from software contributed to Berkeley by
6 * The Mach Operating System project at Carnegie-Mellon University.
7 *
8 * %sccs.include.redist.c%
9 *
10 * @(#)vm_kern.c 8.4 (Berkeley) 01/09/95
11 *
12 *
13 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
14 * All rights reserved.
15 *
16 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
17 *
18 * Permission to use, copy, modify and distribute this software and
19 * its documentation is hereby granted, provided that both the copyright
20 * notice and this permission notice appear in all copies of the
21 * software, derivative works or modified versions, and any portions
22 * thereof, and that both notices appear in supporting documentation.
23 *
24 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
25 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
26 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
27 *
28 * Carnegie Mellon requests users of this software to return to
29 *
30 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
31 * School of Computer Science
32 * Carnegie Mellon University
33 * Pittsburgh PA 15213-3890
34 *
35 * any improvements or extensions that they make and grant Carnegie the
36 * rights to redistribute these changes.
37 */
38
39 /*
40 * Kernel memory management.
41 */
42
43 #include <sys/param.h>
44 #include <sys/systm.h>
45
46 #include <vm/vm.h>
47 #include <vm/vm_page.h>
48 #include <vm/vm_pageout.h>
49 #include <vm/vm_kern.h>
50
51 /*
52 * kmem_alloc_pageable:
53 *
54 * Allocate pageable memory to the kernel's address map.
55 * map must be "kernel_map" below.
56 */
57 vm_offset_t
kmem_alloc_pageable(map,size)58 kmem_alloc_pageable(map, size)
59 vm_map_t map;
60 register vm_size_t size;
61 {
62 vm_offset_t addr;
63 register int result;
64
65 #if 0
66 if (map != kernel_map)
67 panic("kmem_alloc_pageable: not called with kernel_map");
68 #endif
69
70 size = round_page(size);
71
72 addr = vm_map_min(map);
73 result = vm_map_find(map, NULL, (vm_offset_t) 0,
74 &addr, size, TRUE);
75 if (result != KERN_SUCCESS) {
76 return(0);
77 }
78
79 return(addr);
80 }
81
82 /*
83 * Allocate wired-down memory in the kernel's address map
84 * or a submap.
85 */
86 vm_offset_t
kmem_alloc(map,size)87 kmem_alloc(map, size)
88 register vm_map_t map;
89 register vm_size_t size;
90 {
91 vm_offset_t addr;
92 register vm_offset_t offset;
93 extern vm_object_t kernel_object;
94 vm_offset_t i;
95
96 size = round_page(size);
97
98 /*
99 * Use the kernel object for wired-down kernel pages.
100 * Assume that no region of the kernel object is
101 * referenced more than once.
102 */
103
104 /*
105 * Locate sufficient space in the map. This will give us the
106 * final virtual address for the new memory, and thus will tell
107 * us the offset within the kernel map.
108 */
109 vm_map_lock(map);
110 if (vm_map_findspace(map, 0, size, &addr)) {
111 vm_map_unlock(map);
112 return (0);
113 }
114 offset = addr - VM_MIN_KERNEL_ADDRESS;
115 vm_object_reference(kernel_object);
116 vm_map_insert(map, kernel_object, offset, addr, addr + size);
117 vm_map_unlock(map);
118
119 /*
120 * Guarantee that there are pages already in this object
121 * before calling vm_map_pageable. This is to prevent the
122 * following scenario:
123 *
124 * 1) Threads have swapped out, so that there is a
125 * pager for the kernel_object.
126 * 2) The kmsg zone is empty, and so we are kmem_allocing
127 * a new page for it.
128 * 3) vm_map_pageable calls vm_fault; there is no page,
129 * but there is a pager, so we call
130 * pager_data_request. But the kmsg zone is empty,
131 * so we must kmem_alloc.
132 * 4) goto 1
133 * 5) Even if the kmsg zone is not empty: when we get
134 * the data back from the pager, it will be (very
135 * stale) non-zero data. kmem_alloc is defined to
136 * return zero-filled memory.
137 *
138 * We're intentionally not activating the pages we allocate
139 * to prevent a race with page-out. vm_map_pageable will wire
140 * the pages.
141 */
142
143 vm_object_lock(kernel_object);
144 for (i = 0 ; i < size; i+= PAGE_SIZE) {
145 vm_page_t mem;
146
147 while ((mem = vm_page_alloc(kernel_object, offset+i)) == NULL) {
148 vm_object_unlock(kernel_object);
149 VM_WAIT;
150 vm_object_lock(kernel_object);
151 }
152 vm_page_zero_fill(mem);
153 mem->flags &= ~PG_BUSY;
154 }
155 vm_object_unlock(kernel_object);
156
157 /*
158 * And finally, mark the data as non-pageable.
159 */
160
161 (void) vm_map_pageable(map, (vm_offset_t) addr, addr + size, FALSE);
162
163 /*
164 * Try to coalesce the map
165 */
166
167 vm_map_simplify(map, addr);
168
169 return(addr);
170 }
171
172 /*
173 * kmem_free:
174 *
175 * Release a region of kernel virtual memory allocated
176 * with kmem_alloc, and return the physical pages
177 * associated with that region.
178 */
179 void
kmem_free(map,addr,size)180 kmem_free(map, addr, size)
181 vm_map_t map;
182 register vm_offset_t addr;
183 vm_size_t size;
184 {
185 (void) vm_map_remove(map, trunc_page(addr), round_page(addr + size));
186 }
187
188 /*
189 * kmem_suballoc:
190 *
191 * Allocates a map to manage a subrange
192 * of the kernel virtual address space.
193 *
194 * Arguments are as follows:
195 *
196 * parent Map to take range from
197 * size Size of range to find
198 * min, max Returned endpoints of map
199 * pageable Can the region be paged
200 */
201 vm_map_t
kmem_suballoc(parent,min,max,size,pageable)202 kmem_suballoc(parent, min, max, size, pageable)
203 register vm_map_t parent;
204 vm_offset_t *min, *max;
205 register vm_size_t size;
206 boolean_t pageable;
207 {
208 register int ret;
209 vm_map_t result;
210
211 size = round_page(size);
212
213 *min = (vm_offset_t) vm_map_min(parent);
214 ret = vm_map_find(parent, NULL, (vm_offset_t) 0,
215 min, size, TRUE);
216 if (ret != KERN_SUCCESS) {
217 printf("kmem_suballoc: bad status return of %d.\n", ret);
218 panic("kmem_suballoc");
219 }
220 *max = *min + size;
221 pmap_reference(vm_map_pmap(parent));
222 result = vm_map_create(vm_map_pmap(parent), *min, *max, pageable);
223 if (result == NULL)
224 panic("kmem_suballoc: cannot create submap");
225 if ((ret = vm_map_submap(parent, *min, *max, result)) != KERN_SUCCESS)
226 panic("kmem_suballoc: unable to change range to submap");
227 return(result);
228 }
229
230 /*
231 * Allocate wired-down memory in the kernel's address map for the higher
232 * level kernel memory allocator (kern/kern_malloc.c). We cannot use
233 * kmem_alloc() because we may need to allocate memory at interrupt
234 * level where we cannot block (canwait == FALSE).
235 *
236 * This routine has its own private kernel submap (kmem_map) and object
237 * (kmem_object). This, combined with the fact that only malloc uses
238 * this routine, ensures that we will never block in map or object waits.
239 *
240 * Note that this still only works in a uni-processor environment and
241 * when called at splhigh().
242 *
243 * We don't worry about expanding the map (adding entries) since entries
244 * for wired maps are statically allocated.
245 */
246 vm_offset_t
kmem_malloc(map,size,canwait)247 kmem_malloc(map, size, canwait)
248 register vm_map_t map;
249 register vm_size_t size;
250 boolean_t canwait;
251 {
252 register vm_offset_t offset, i;
253 vm_map_entry_t entry;
254 vm_offset_t addr;
255 vm_page_t m;
256 extern vm_object_t kmem_object;
257
258 if (map != kmem_map && map != mb_map)
259 panic("kern_malloc_alloc: map != {kmem,mb}_map");
260
261 size = round_page(size);
262 addr = vm_map_min(map);
263
264 /*
265 * Locate sufficient space in the map. This will give us the
266 * final virtual address for the new memory, and thus will tell
267 * us the offset within the kernel map.
268 */
269 vm_map_lock(map);
270 if (vm_map_findspace(map, 0, size, &addr)) {
271 vm_map_unlock(map);
272 if (canwait) /* XXX should wait */
273 panic("kmem_malloc: %s too small",
274 map == kmem_map ? "kmem_map" : "mb_map");
275 return (0);
276 }
277 offset = addr - vm_map_min(kmem_map);
278 vm_object_reference(kmem_object);
279 vm_map_insert(map, kmem_object, offset, addr, addr + size);
280
281 /*
282 * If we can wait, just mark the range as wired
283 * (will fault pages as necessary).
284 */
285 if (canwait) {
286 vm_map_unlock(map);
287 (void) vm_map_pageable(map, (vm_offset_t) addr, addr + size,
288 FALSE);
289 vm_map_simplify(map, addr);
290 return(addr);
291 }
292
293 /*
294 * If we cannot wait then we must allocate all memory up front,
295 * pulling it off the active queue to prevent pageout.
296 */
297 vm_object_lock(kmem_object);
298 for (i = 0; i < size; i += PAGE_SIZE) {
299 m = vm_page_alloc(kmem_object, offset + i);
300
301 /*
302 * Ran out of space, free everything up and return.
303 * Don't need to lock page queues here as we know
304 * that the pages we got aren't on any queues.
305 */
306 if (m == NULL) {
307 while (i != 0) {
308 i -= PAGE_SIZE;
309 m = vm_page_lookup(kmem_object, offset + i);
310 vm_page_free(m);
311 }
312 vm_object_unlock(kmem_object);
313 vm_map_delete(map, addr, addr + size);
314 vm_map_unlock(map);
315 return(0);
316 }
317 #if 0
318 vm_page_zero_fill(m);
319 #endif
320 m->flags &= ~PG_BUSY;
321 }
322 vm_object_unlock(kmem_object);
323
324 /*
325 * Mark map entry as non-pageable.
326 * Assert: vm_map_insert() will never be able to extend the previous
327 * entry so there will be a new entry exactly corresponding to this
328 * address range and it will have wired_count == 0.
329 */
330 if (!vm_map_lookup_entry(map, addr, &entry) ||
331 entry->start != addr || entry->end != addr + size ||
332 entry->wired_count)
333 panic("kmem_malloc: entry not found or misaligned");
334 entry->wired_count++;
335
336 /*
337 * Loop thru pages, entering them in the pmap.
338 * (We cannot add them to the wired count without
339 * wrapping the vm_page_queue_lock in splimp...)
340 */
341 for (i = 0; i < size; i += PAGE_SIZE) {
342 vm_object_lock(kmem_object);
343 m = vm_page_lookup(kmem_object, offset + i);
344 vm_object_unlock(kmem_object);
345 pmap_enter(map->pmap, addr + i, VM_PAGE_TO_PHYS(m),
346 VM_PROT_DEFAULT, TRUE);
347 }
348 vm_map_unlock(map);
349
350 vm_map_simplify(map, addr);
351 return(addr);
352 }
353
354 /*
355 * kmem_alloc_wait
356 *
357 * Allocates pageable memory from a sub-map of the kernel. If the submap
358 * has no room, the caller sleeps waiting for more memory in the submap.
359 *
360 */
361 vm_offset_t
kmem_alloc_wait(map,size)362 kmem_alloc_wait(map, size)
363 vm_map_t map;
364 vm_size_t size;
365 {
366 vm_offset_t addr;
367
368 size = round_page(size);
369
370 for (;;) {
371 /*
372 * To make this work for more than one map,
373 * use the map's lock to lock out sleepers/wakers.
374 */
375 vm_map_lock(map);
376 if (vm_map_findspace(map, 0, size, &addr) == 0)
377 break;
378 /* no space now; see if we can ever get space */
379 if (vm_map_max(map) - vm_map_min(map) < size) {
380 vm_map_unlock(map);
381 return (0);
382 }
383 assert_wait(map, TRUE);
384 vm_map_unlock(map);
385 thread_block();
386 }
387 vm_map_insert(map, NULL, (vm_offset_t)0, addr, addr + size);
388 vm_map_unlock(map);
389 return (addr);
390 }
391
392 /*
393 * kmem_free_wakeup
394 *
395 * Returns memory to a submap of the kernel, and wakes up any threads
396 * waiting for memory in that map.
397 */
398 void
kmem_free_wakeup(map,addr,size)399 kmem_free_wakeup(map, addr, size)
400 vm_map_t map;
401 vm_offset_t addr;
402 vm_size_t size;
403 {
404 vm_map_lock(map);
405 (void) vm_map_delete(map, trunc_page(addr), round_page(addr + size));
406 thread_wakeup(map);
407 vm_map_unlock(map);
408 }
409
410 /*
411 * Create the kernel map; insert a mapping covering kernel text, data, bss,
412 * and all space allocated thus far (`boostrap' data). The new map will thus
413 * map the range between VM_MIN_KERNEL_ADDRESS and `start' as allocated, and
414 * the range between `start' and `end' as free.
415 */
416 void
kmem_init(start,end)417 kmem_init(start, end)
418 vm_offset_t start, end;
419 {
420 register vm_map_t m;
421
422 m = vm_map_create(kernel_pmap, VM_MIN_KERNEL_ADDRESS, end, FALSE);
423 vm_map_lock(m);
424 /* N.B.: cannot use kgdb to debug, starting with this assignment ... */
425 kernel_map = m;
426 (void) vm_map_insert(m, NULL, (vm_offset_t)0,
427 VM_MIN_KERNEL_ADDRESS, start);
428 /* ... and ending with the completion of the above `insert' */
429 vm_map_unlock(m);
430 }
431