1 /* $NetBSD: uvm_amap.c,v 1.107 2012/04/08 20:47:10 chs Exp $ */
2
3 /*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 /*
29 * uvm_amap.c: amap operations
30 */
31
32 /*
33 * this file contains functions that perform operations on amaps. see
34 * uvm_amap.h for a brief explanation of the role of amaps in uvm.
35 */
36
37 #include <sys/cdefs.h>
38 __KERNEL_RCSID(0, "$NetBSD: uvm_amap.c,v 1.107 2012/04/08 20:47:10 chs Exp $");
39
40 #include "opt_uvmhist.h"
41
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
45 #include <sys/kmem.h>
46 #include <sys/pool.h>
47 #include <sys/atomic.h>
48
49 #include <uvm/uvm.h>
50 #include <uvm/uvm_swap.h>
51
52 /*
53 * cache for allocation of vm_map structures. note that in order to
54 * avoid an endless loop, the amap cache's allocator cannot allocate
55 * memory from an amap (it currently goes through the kernel uobj, so
56 * we are ok).
57 */
58 static struct pool_cache uvm_amap_cache;
59 static kmutex_t amap_list_lock;
60 static LIST_HEAD(, vm_amap) amap_list;
61
62 /*
63 * local functions
64 */
65
66 static inline void
amap_list_insert(struct vm_amap * amap)67 amap_list_insert(struct vm_amap *amap)
68 {
69
70 mutex_enter(&amap_list_lock);
71 LIST_INSERT_HEAD(&amap_list, amap, am_list);
72 mutex_exit(&amap_list_lock);
73 }
74
75 static inline void
amap_list_remove(struct vm_amap * amap)76 amap_list_remove(struct vm_amap *amap)
77 {
78
79 mutex_enter(&amap_list_lock);
80 LIST_REMOVE(amap, am_list);
81 mutex_exit(&amap_list_lock);
82 }
83
84 static int
amap_roundup_slots(int slots)85 amap_roundup_slots(int slots)
86 {
87
88 return kmem_roundup_size(slots * sizeof(int)) / sizeof(int);
89 }
90
91 #ifdef UVM_AMAP_PPREF
92 /*
93 * what is ppref? ppref is an _optional_ amap feature which is used
94 * to keep track of reference counts on a per-page basis. it is enabled
95 * when UVM_AMAP_PPREF is defined.
96 *
97 * when enabled, an array of ints is allocated for the pprefs. this
98 * array is allocated only when a partial reference is added to the
99 * map (either by unmapping part of the amap, or gaining a reference
100 * to only a part of an amap). if the allocation of the array fails
101 * (KM_NOSLEEP), then we set the array pointer to PPREF_NONE to indicate
102 * that we tried to do ppref's but couldn't alloc the array so just
103 * give up (after all, this is an optional feature!).
104 *
105 * the array is divided into page sized "chunks." for chunks of length 1,
106 * the chunk reference count plus one is stored in that chunk's slot.
107 * for chunks of length > 1 the first slot contains (the reference count
108 * plus one) * -1. [the negative value indicates that the length is
109 * greater than one.] the second slot of the chunk contains the length
110 * of the chunk. here is an example:
111 *
112 * actual REFS: 2 2 2 2 3 1 1 0 0 0 4 4 0 1 1 1
113 * ppref: -3 4 x x 4 -2 2 -1 3 x -5 2 1 -2 3 x
114 * <----------><-><----><-------><----><-><------->
115 * (x = don't care)
116 *
117 * this allows us to allow one int to contain the ref count for the whole
118 * chunk. note that the "plus one" part is needed because a reference
119 * count of zero is neither positive or negative (need a way to tell
120 * if we've got one zero or a bunch of them).
121 *
122 * here are some in-line functions to help us.
123 */
124
125 /*
126 * pp_getreflen: get the reference and length for a specific offset
127 *
128 * => ppref's amap must be locked
129 */
130 static inline void
pp_getreflen(int * ppref,int offset,int * refp,int * lenp)131 pp_getreflen(int *ppref, int offset, int *refp, int *lenp)
132 {
133
134 if (ppref[offset] > 0) { /* chunk size must be 1 */
135 *refp = ppref[offset] - 1; /* don't forget to adjust */
136 *lenp = 1;
137 } else {
138 *refp = (ppref[offset] * -1) - 1;
139 *lenp = ppref[offset+1];
140 }
141 }
142
143 /*
144 * pp_setreflen: set the reference and length for a specific offset
145 *
146 * => ppref's amap must be locked
147 */
148 static inline void
pp_setreflen(int * ppref,int offset,int ref,int len)149 pp_setreflen(int *ppref, int offset, int ref, int len)
150 {
151 if (len == 0)
152 return;
153 if (len == 1) {
154 ppref[offset] = ref + 1;
155 } else {
156 ppref[offset] = (ref + 1) * -1;
157 ppref[offset+1] = len;
158 }
159 }
160 #endif /* UVM_AMAP_PPREF */
161
162 /*
163 * amap_alloc1: allocate an amap, but do not initialise the overlay.
164 *
165 * => Note: lock is not set.
166 */
167 static struct vm_amap *
amap_alloc1(int slots,int padslots,int flags)168 amap_alloc1(int slots, int padslots, int flags)
169 {
170 const bool nowait = (flags & UVM_FLAG_NOWAIT) != 0;
171 const km_flag_t kmflags = nowait ? KM_NOSLEEP : KM_SLEEP;
172 struct vm_amap *amap;
173 int totalslots;
174
175 amap = pool_cache_get(&uvm_amap_cache, nowait ? PR_NOWAIT : PR_WAITOK);
176 if (amap == NULL) {
177 return NULL;
178 }
179 totalslots = amap_roundup_slots(slots + padslots);
180 amap->am_lock = NULL;
181 amap->am_ref = 1;
182 amap->am_flags = 0;
183 #ifdef UVM_AMAP_PPREF
184 amap->am_ppref = NULL;
185 #endif
186 amap->am_maxslot = totalslots;
187 amap->am_nslot = slots;
188 amap->am_nused = 0;
189
190 /*
191 * Note: since allocations are likely big, we expect to reduce the
192 * memory fragmentation by allocating them in separate blocks.
193 */
194 amap->am_slots = kmem_alloc(totalslots * sizeof(int), kmflags);
195 if (amap->am_slots == NULL)
196 goto fail1;
197
198 amap->am_bckptr = kmem_alloc(totalslots * sizeof(int), kmflags);
199 if (amap->am_bckptr == NULL)
200 goto fail2;
201
202 amap->am_anon = kmem_alloc(totalslots * sizeof(struct vm_anon *),
203 kmflags);
204 if (amap->am_anon == NULL)
205 goto fail3;
206
207 return amap;
208
209 fail3:
210 kmem_free(amap->am_bckptr, totalslots * sizeof(int));
211 fail2:
212 kmem_free(amap->am_slots, totalslots * sizeof(int));
213 fail1:
214 pool_cache_put(&uvm_amap_cache, amap);
215
216 /*
217 * XXX hack to tell the pagedaemon how many pages we need,
218 * since we can need more than it would normally free.
219 */
220 if (nowait) {
221 extern u_int uvm_extrapages;
222 atomic_add_int(&uvm_extrapages,
223 ((sizeof(int) * 2 + sizeof(struct vm_anon *)) *
224 totalslots) >> PAGE_SHIFT);
225 }
226 return NULL;
227 }
228
229 /*
230 * amap_alloc: allocate an amap to manage "sz" bytes of anonymous VM
231 *
232 * => caller should ensure sz is a multiple of PAGE_SIZE
233 * => reference count to new amap is set to one
234 * => new amap is returned unlocked
235 */
236
237 struct vm_amap *
amap_alloc(vaddr_t sz,vaddr_t padsz,int waitf)238 amap_alloc(vaddr_t sz, vaddr_t padsz, int waitf)
239 {
240 struct vm_amap *amap;
241 int slots, padslots;
242 UVMHIST_FUNC("amap_alloc"); UVMHIST_CALLED(maphist);
243
244 AMAP_B2SLOT(slots, sz);
245 AMAP_B2SLOT(padslots, padsz);
246
247 amap = amap_alloc1(slots, padslots, waitf);
248 if (amap) {
249 memset(amap->am_anon, 0,
250 amap->am_maxslot * sizeof(struct vm_anon *));
251 amap->am_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
252 amap_list_insert(amap);
253 }
254
255 UVMHIST_LOG(maphist,"<- done, amap = 0x%x, sz=%d", amap, sz, 0, 0);
256 return(amap);
257 }
258
259 /*
260 * uvm_amap_init: initialize the amap system.
261 */
262 void
uvm_amap_init(void)263 uvm_amap_init(void)
264 {
265
266 mutex_init(&amap_list_lock, MUTEX_DEFAULT, IPL_NONE);
267
268 pool_cache_bootstrap(&uvm_amap_cache, sizeof(struct vm_amap), 0, 0, 0,
269 "amappl", NULL, IPL_NONE, NULL, NULL, NULL);
270 }
271
272 /*
273 * amap_free: free an amap
274 *
275 * => the amap must be unlocked
276 * => the amap should have a zero reference count and be empty
277 */
278 void
amap_free(struct vm_amap * amap)279 amap_free(struct vm_amap *amap)
280 {
281 int slots;
282
283 UVMHIST_FUNC("amap_free"); UVMHIST_CALLED(maphist);
284
285 KASSERT(amap->am_ref == 0 && amap->am_nused == 0);
286 KASSERT((amap->am_flags & AMAP_SWAPOFF) == 0);
287 if (amap->am_lock != NULL) {
288 KASSERT(!mutex_owned(amap->am_lock));
289 mutex_obj_free(amap->am_lock);
290 }
291 slots = amap->am_maxslot;
292 kmem_free(amap->am_slots, slots * sizeof(*amap->am_slots));
293 kmem_free(amap->am_bckptr, slots * sizeof(*amap->am_bckptr));
294 kmem_free(amap->am_anon, slots * sizeof(*amap->am_anon));
295 #ifdef UVM_AMAP_PPREF
296 if (amap->am_ppref && amap->am_ppref != PPREF_NONE)
297 kmem_free(amap->am_ppref, slots * sizeof(*amap->am_ppref));
298 #endif
299 pool_cache_put(&uvm_amap_cache, amap);
300 UVMHIST_LOG(maphist,"<- done, freed amap = 0x%x", amap, 0, 0, 0);
301 }
302
303 /*
304 * amap_extend: extend the size of an amap (if needed)
305 *
306 * => called from uvm_map when we want to extend an amap to cover
307 * a new mapping (rather than allocate a new one)
308 * => amap should be unlocked (we will lock it)
309 * => to safely extend an amap it should have a reference count of
310 * one (thus it can't be shared)
311 */
312 int
amap_extend(struct vm_map_entry * entry,vsize_t addsize,int flags)313 amap_extend(struct vm_map_entry *entry, vsize_t addsize, int flags)
314 {
315 struct vm_amap *amap = entry->aref.ar_amap;
316 int slotoff = entry->aref.ar_pageoff;
317 int slotmapped, slotadd, slotneed, slotadded, slotalloc;
318 int slotadj, slotspace;
319 int oldnslots;
320 #ifdef UVM_AMAP_PPREF
321 int *newppref, *oldppref;
322 #endif
323 int i, *newsl, *newbck, *oldsl, *oldbck;
324 struct vm_anon **newover, **oldover, *tofree;
325 const km_flag_t kmflags =
326 (flags & AMAP_EXTEND_NOWAIT) ? KM_NOSLEEP : KM_SLEEP;
327
328 UVMHIST_FUNC("amap_extend"); UVMHIST_CALLED(maphist);
329
330 UVMHIST_LOG(maphist, " (entry=0x%x, addsize=0x%x, flags=0x%x)",
331 entry, addsize, flags, 0);
332
333 /*
334 * first, determine how many slots we need in the amap. don't
335 * forget that ar_pageoff could be non-zero: this means that
336 * there are some unused slots before us in the amap.
337 */
338
339 amap_lock(amap);
340 KASSERT(amap_refs(amap) == 1); /* amap can't be shared */
341 AMAP_B2SLOT(slotmapped, entry->end - entry->start); /* slots mapped */
342 AMAP_B2SLOT(slotadd, addsize); /* slots to add */
343 if (flags & AMAP_EXTEND_FORWARDS) {
344 slotneed = slotoff + slotmapped + slotadd;
345 slotadj = 0;
346 slotspace = 0;
347 }
348 else {
349 slotneed = slotadd + slotmapped;
350 slotadj = slotadd - slotoff;
351 slotspace = amap->am_maxslot - slotmapped;
352 }
353 tofree = NULL;
354
355 /*
356 * case 1: we already have enough slots in the map and thus
357 * only need to bump the reference counts on the slots we are
358 * adding.
359 */
360
361 if (flags & AMAP_EXTEND_FORWARDS) {
362 if (amap->am_nslot >= slotneed) {
363 #ifdef UVM_AMAP_PPREF
364 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
365 amap_pp_adjref(amap, slotoff + slotmapped,
366 slotadd, 1, &tofree);
367 }
368 #endif
369 uvm_anon_freelst(amap, tofree);
370 UVMHIST_LOG(maphist,
371 "<- done (case 1f), amap = 0x%x, sltneed=%d",
372 amap, slotneed, 0, 0);
373 return 0;
374 }
375 } else {
376 if (slotadj <= 0) {
377 slotoff -= slotadd;
378 entry->aref.ar_pageoff = slotoff;
379 #ifdef UVM_AMAP_PPREF
380 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
381 amap_pp_adjref(amap, slotoff, slotadd, 1,
382 &tofree);
383 }
384 #endif
385 uvm_anon_freelst(amap, tofree);
386 UVMHIST_LOG(maphist,
387 "<- done (case 1b), amap = 0x%x, sltneed=%d",
388 amap, slotneed, 0, 0);
389 return 0;
390 }
391 }
392
393 /*
394 * case 2: we pre-allocated slots for use and we just need to
395 * bump nslot up to take account for these slots.
396 */
397
398 if (amap->am_maxslot >= slotneed) {
399 if (flags & AMAP_EXTEND_FORWARDS) {
400 #ifdef UVM_AMAP_PPREF
401 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
402 if ((slotoff + slotmapped) < amap->am_nslot)
403 amap_pp_adjref(amap,
404 slotoff + slotmapped,
405 (amap->am_nslot -
406 (slotoff + slotmapped)), 1,
407 &tofree);
408 pp_setreflen(amap->am_ppref, amap->am_nslot, 1,
409 slotneed - amap->am_nslot);
410 }
411 #endif
412 amap->am_nslot = slotneed;
413 uvm_anon_freelst(amap, tofree);
414
415 /*
416 * no need to zero am_anon since that was done at
417 * alloc time and we never shrink an allocation.
418 */
419
420 UVMHIST_LOG(maphist,"<- done (case 2f), amap = 0x%x, "
421 "slotneed=%d", amap, slotneed, 0, 0);
422 return 0;
423 } else {
424 #ifdef UVM_AMAP_PPREF
425 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
426 /*
427 * Slide up the ref counts on the pages that
428 * are actually in use.
429 */
430 memmove(amap->am_ppref + slotspace,
431 amap->am_ppref + slotoff,
432 slotmapped * sizeof(int));
433 /*
434 * Mark the (adjusted) gap at the front as
435 * referenced/not referenced.
436 */
437 pp_setreflen(amap->am_ppref,
438 0, 0, slotspace - slotadd);
439 pp_setreflen(amap->am_ppref,
440 slotspace - slotadd, 1, slotadd);
441 }
442 #endif
443
444 /*
445 * Slide the anon pointers up and clear out
446 * the space we just made.
447 */
448 memmove(amap->am_anon + slotspace,
449 amap->am_anon + slotoff,
450 slotmapped * sizeof(struct vm_anon*));
451 memset(amap->am_anon + slotoff, 0,
452 (slotspace - slotoff) * sizeof(struct vm_anon *));
453
454 /*
455 * Slide the backpointers up, but don't bother
456 * wiping out the old slots.
457 */
458 memmove(amap->am_bckptr + slotspace,
459 amap->am_bckptr + slotoff,
460 slotmapped * sizeof(int));
461
462 /*
463 * Adjust all the useful active slot numbers.
464 */
465 for (i = 0; i < amap->am_nused; i++)
466 amap->am_slots[i] += (slotspace - slotoff);
467
468 /*
469 * We just filled all the empty space in the
470 * front of the amap by activating a few new
471 * slots.
472 */
473 amap->am_nslot = amap->am_maxslot;
474 entry->aref.ar_pageoff = slotspace - slotadd;
475 amap_unlock(amap);
476
477 UVMHIST_LOG(maphist,"<- done (case 2b), amap = 0x%x, "
478 "slotneed=%d", amap, slotneed, 0, 0);
479 return 0;
480 }
481 }
482
483 /*
484 * Case 3: we need to allocate a new amap and copy all the amap
485 * data over from old amap to the new one. Drop the lock before
486 * performing allocation.
487 *
488 * Note: since allocations are likely big, we expect to reduce the
489 * memory fragmentation by allocating them in separate blocks.
490 */
491
492 amap_unlock(amap);
493
494 if (slotneed >= UVM_AMAP_LARGE) {
495 return E2BIG;
496 }
497
498 slotalloc = amap_roundup_slots(slotneed);
499 #ifdef UVM_AMAP_PPREF
500 newppref = NULL;
501 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
502 /* Will be handled later if fails. */
503 newppref = kmem_alloc(slotalloc * sizeof(*newppref), kmflags);
504 }
505 #endif
506 newsl = kmem_alloc(slotalloc * sizeof(*newsl), kmflags);
507 newbck = kmem_alloc(slotalloc * sizeof(*newbck), kmflags);
508 newover = kmem_alloc(slotalloc * sizeof(*newover), kmflags);
509 if (newsl == NULL || newbck == NULL || newover == NULL) {
510 #ifdef UVM_AMAP_PPREF
511 if (newppref != NULL) {
512 kmem_free(newppref, slotalloc * sizeof(*newppref));
513 }
514 #endif
515 if (newsl != NULL) {
516 kmem_free(newsl, slotalloc * sizeof(*newsl));
517 }
518 if (newbck != NULL) {
519 kmem_free(newbck, slotalloc * sizeof(*newbck));
520 }
521 if (newover != NULL) {
522 kmem_free(newover, slotalloc * sizeof(*newover));
523 }
524 return ENOMEM;
525 }
526 amap_lock(amap);
527 KASSERT(amap->am_maxslot < slotneed);
528
529 /*
530 * Copy everything over to new allocated areas.
531 */
532
533 slotadded = slotalloc - amap->am_nslot;
534 if (!(flags & AMAP_EXTEND_FORWARDS))
535 slotspace = slotalloc - slotmapped;
536
537 /* do am_slots */
538 oldsl = amap->am_slots;
539 if (flags & AMAP_EXTEND_FORWARDS)
540 memcpy(newsl, oldsl, sizeof(int) * amap->am_nused);
541 else
542 for (i = 0; i < amap->am_nused; i++)
543 newsl[i] = oldsl[i] + slotspace - slotoff;
544 amap->am_slots = newsl;
545
546 /* do am_anon */
547 oldover = amap->am_anon;
548 if (flags & AMAP_EXTEND_FORWARDS) {
549 memcpy(newover, oldover,
550 sizeof(struct vm_anon *) * amap->am_nslot);
551 memset(newover + amap->am_nslot, 0,
552 sizeof(struct vm_anon *) * slotadded);
553 } else {
554 memcpy(newover + slotspace, oldover + slotoff,
555 sizeof(struct vm_anon *) * slotmapped);
556 memset(newover, 0,
557 sizeof(struct vm_anon *) * slotspace);
558 }
559 amap->am_anon = newover;
560
561 /* do am_bckptr */
562 oldbck = amap->am_bckptr;
563 if (flags & AMAP_EXTEND_FORWARDS)
564 memcpy(newbck, oldbck, sizeof(int) * amap->am_nslot);
565 else
566 memcpy(newbck + slotspace, oldbck + slotoff,
567 sizeof(int) * slotmapped);
568 amap->am_bckptr = newbck;
569
570 #ifdef UVM_AMAP_PPREF
571 /* do ppref */
572 oldppref = amap->am_ppref;
573 if (newppref) {
574 if (flags & AMAP_EXTEND_FORWARDS) {
575 memcpy(newppref, oldppref,
576 sizeof(int) * amap->am_nslot);
577 memset(newppref + amap->am_nslot, 0,
578 sizeof(int) * slotadded);
579 } else {
580 memcpy(newppref + slotspace, oldppref + slotoff,
581 sizeof(int) * slotmapped);
582 }
583 amap->am_ppref = newppref;
584 if ((flags & AMAP_EXTEND_FORWARDS) &&
585 (slotoff + slotmapped) < amap->am_nslot)
586 amap_pp_adjref(amap, slotoff + slotmapped,
587 (amap->am_nslot - (slotoff + slotmapped)), 1,
588 &tofree);
589 if (flags & AMAP_EXTEND_FORWARDS)
590 pp_setreflen(newppref, amap->am_nslot, 1,
591 slotneed - amap->am_nslot);
592 else {
593 pp_setreflen(newppref, 0, 0,
594 slotalloc - slotneed);
595 pp_setreflen(newppref, slotalloc - slotneed, 1,
596 slotneed - slotmapped);
597 }
598 } else {
599 if (amap->am_ppref)
600 amap->am_ppref = PPREF_NONE;
601 }
602 #endif
603
604 /* update master values */
605 if (flags & AMAP_EXTEND_FORWARDS)
606 amap->am_nslot = slotneed;
607 else {
608 entry->aref.ar_pageoff = slotspace - slotadd;
609 amap->am_nslot = slotalloc;
610 }
611 oldnslots = amap->am_maxslot;
612 amap->am_maxslot = slotalloc;
613
614 uvm_anon_freelst(amap, tofree);
615
616 kmem_free(oldsl, oldnslots * sizeof(*oldsl));
617 kmem_free(oldbck, oldnslots * sizeof(*oldbck));
618 kmem_free(oldover, oldnslots * sizeof(*oldover));
619 #ifdef UVM_AMAP_PPREF
620 if (oldppref && oldppref != PPREF_NONE)
621 kmem_free(oldppref, oldnslots * sizeof(*oldppref));
622 #endif
623 UVMHIST_LOG(maphist,"<- done (case 3), amap = 0x%x, slotneed=%d",
624 amap, slotneed, 0, 0);
625 return 0;
626 }
627
628 /*
629 * amap_share_protect: change protection of anons in a shared amap
630 *
631 * for shared amaps, given the current data structure layout, it is
632 * not possible for us to directly locate all maps referencing the
633 * shared anon (to change the protection). in order to protect data
634 * in shared maps we use pmap_page_protect(). [this is useful for IPC
635 * mechanisms like map entry passing that may want to write-protect
636 * all mappings of a shared amap.] we traverse am_anon or am_slots
637 * depending on the current state of the amap.
638 *
639 * => entry's map and amap must be locked by the caller
640 */
641 void
amap_share_protect(struct vm_map_entry * entry,vm_prot_t prot)642 amap_share_protect(struct vm_map_entry *entry, vm_prot_t prot)
643 {
644 struct vm_amap *amap = entry->aref.ar_amap;
645 u_int slots, lcv, slot, stop;
646 struct vm_anon *anon;
647
648 KASSERT(mutex_owned(amap->am_lock));
649
650 AMAP_B2SLOT(slots, (entry->end - entry->start));
651 stop = entry->aref.ar_pageoff + slots;
652
653 if (slots < amap->am_nused) {
654 /*
655 * Cheaper to traverse am_anon.
656 */
657 for (lcv = entry->aref.ar_pageoff ; lcv < stop ; lcv++) {
658 anon = amap->am_anon[lcv];
659 if (anon == NULL) {
660 continue;
661 }
662 if (anon->an_page) {
663 pmap_page_protect(anon->an_page, prot);
664 }
665 }
666 return;
667 }
668
669 /*
670 * Cheaper to traverse am_slots.
671 */
672 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) {
673 slot = amap->am_slots[lcv];
674 if (slot < entry->aref.ar_pageoff || slot >= stop) {
675 continue;
676 }
677 anon = amap->am_anon[slot];
678 if (anon->an_page) {
679 pmap_page_protect(anon->an_page, prot);
680 }
681 }
682 }
683
684 /*
685 * amap_wipeout: wipeout all anon's in an amap; then free the amap!
686 *
687 * => Called from amap_unref(), when reference count drops to zero.
688 * => amap must be locked.
689 */
690
691 void
amap_wipeout(struct vm_amap * amap)692 amap_wipeout(struct vm_amap *amap)
693 {
694 struct vm_anon *tofree = NULL;
695 u_int lcv;
696
697 UVMHIST_FUNC("amap_wipeout"); UVMHIST_CALLED(maphist);
698 UVMHIST_LOG(maphist,"(amap=0x%x)", amap, 0,0,0);
699
700 KASSERT(mutex_owned(amap->am_lock));
701 KASSERT(amap->am_ref == 0);
702
703 if (__predict_false(amap->am_flags & AMAP_SWAPOFF)) {
704 /*
705 * Note: amap_swap_off() will call us again.
706 */
707 amap_unlock(amap);
708 return;
709 }
710 amap_list_remove(amap);
711
712 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) {
713 struct vm_anon *anon;
714 u_int slot;
715
716 slot = amap->am_slots[lcv];
717 anon = amap->am_anon[slot];
718 KASSERT(anon != NULL && anon->an_ref != 0);
719
720 KASSERT(anon->an_lock == amap->am_lock);
721 UVMHIST_LOG(maphist," processing anon 0x%x, ref=%d", anon,
722 anon->an_ref, 0, 0);
723
724 /*
725 * Drop the reference. Defer freeing.
726 */
727
728 if (--anon->an_ref == 0) {
729 anon->an_link = tofree;
730 tofree = anon;
731 }
732 if (curlwp->l_cpu->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) {
733 preempt();
734 }
735 }
736
737 /*
738 * Finally, destroy the amap.
739 */
740
741 amap->am_nused = 0;
742 uvm_anon_freelst(amap, tofree);
743 amap_free(amap);
744 UVMHIST_LOG(maphist,"<- done!", 0,0,0,0);
745 }
746
747 /*
748 * amap_copy: ensure that a map entry's "needs_copy" flag is false
749 * by copying the amap if necessary.
750 *
751 * => an entry with a null amap pointer will get a new (blank) one.
752 * => the map that the map entry belongs to must be locked by caller.
753 * => the amap currently attached to "entry" (if any) must be unlocked.
754 * => if canchunk is true, then we may clip the entry into a chunk
755 * => "startva" and "endva" are used only if canchunk is true. they are
756 * used to limit chunking (e.g. if you have a large space that you
757 * know you are going to need to allocate amaps for, there is no point
758 * in allowing that to be chunked)
759 */
760
761 void
amap_copy(struct vm_map * map,struct vm_map_entry * entry,int flags,vaddr_t startva,vaddr_t endva)762 amap_copy(struct vm_map *map, struct vm_map_entry *entry, int flags,
763 vaddr_t startva, vaddr_t endva)
764 {
765 const int waitf = (flags & AMAP_COPY_NOWAIT) ? UVM_FLAG_NOWAIT : 0;
766 struct vm_amap *amap, *srcamap;
767 struct vm_anon *tofree;
768 u_int slots, lcv;
769 vsize_t len;
770
771 UVMHIST_FUNC("amap_copy"); UVMHIST_CALLED(maphist);
772 UVMHIST_LOG(maphist, " (map=%p, entry=%p, flags=%d)",
773 map, entry, flags, 0);
774
775 KASSERT(map != kernel_map); /* we use nointr pool */
776
777 srcamap = entry->aref.ar_amap;
778 len = entry->end - entry->start;
779
780 /*
781 * Is there an amap to copy? If not, create one.
782 */
783
784 if (srcamap == NULL) {
785 const bool canchunk = (flags & AMAP_COPY_NOCHUNK) == 0;
786
787 /*
788 * Check to see if we have a large amap that we can
789 * chunk. We align startva/endva to chunk-sized
790 * boundaries and then clip to them.
791 */
792
793 if (canchunk && atop(len) >= UVM_AMAP_LARGE) {
794 vsize_t chunksize;
795
796 /* Convert slots to bytes. */
797 chunksize = UVM_AMAP_CHUNK << PAGE_SHIFT;
798 startva = (startva / chunksize) * chunksize;
799 endva = roundup(endva, chunksize);
800 UVMHIST_LOG(maphist, " chunk amap ==> clip 0x%x->0x%x"
801 "to 0x%x->0x%x", entry->start, entry->end, startva,
802 endva);
803 UVM_MAP_CLIP_START(map, entry, startva);
804
805 /* Watch out for endva wrap-around! */
806 if (endva >= startva) {
807 UVM_MAP_CLIP_END(map, entry, endva);
808 }
809 }
810
811 if ((flags & AMAP_COPY_NOMERGE) == 0 &&
812 uvm_mapent_trymerge(map, entry, UVM_MERGE_COPYING)) {
813 return;
814 }
815
816 UVMHIST_LOG(maphist, "<- done [creating new amap 0x%x->0x%x]",
817 entry->start, entry->end, 0, 0);
818
819 /*
820 * Allocate an initialised amap and install it.
821 * Note: we must update the length after clipping.
822 */
823 len = entry->end - entry->start;
824 entry->aref.ar_pageoff = 0;
825 entry->aref.ar_amap = amap_alloc(len, 0, waitf);
826 if (entry->aref.ar_amap != NULL) {
827 entry->etype &= ~UVM_ET_NEEDSCOPY;
828 }
829 return;
830 }
831
832 /*
833 * First check and see if we are the only map entry referencing
834 * he amap we currently have. If so, then just take it over instead
835 * of copying it. Note that we are reading am_ref without lock held
836 * as the value value can only be one if we have the only reference
837 * to the amap (via our locked map). If the value is greater than
838 * one, then allocate amap and re-check the value.
839 */
840
841 if (srcamap->am_ref == 1) {
842 entry->etype &= ~UVM_ET_NEEDSCOPY;
843 UVMHIST_LOG(maphist, "<- done [ref cnt = 1, took it over]",
844 0, 0, 0, 0);
845 return;
846 }
847
848 UVMHIST_LOG(maphist," amap=%p, ref=%d, must copy it",
849 srcamap, srcamap->am_ref, 0, 0);
850
851 /*
852 * Allocate a new amap (note: not initialised, no lock set, etc).
853 */
854
855 AMAP_B2SLOT(slots, len);
856 amap = amap_alloc1(slots, 0, waitf);
857 if (amap == NULL) {
858 UVMHIST_LOG(maphist, " amap_alloc1 failed", 0,0,0,0);
859 return;
860 }
861
862 amap_lock(srcamap);
863
864 /*
865 * Re-check the reference count with the lock held. If it has
866 * dropped to one - we can take over the existing map.
867 */
868
869 if (srcamap->am_ref == 1) {
870 /* Just take over the existing amap. */
871 entry->etype &= ~UVM_ET_NEEDSCOPY;
872 amap_unlock(srcamap);
873 /* Destroy the new (unused) amap. */
874 amap->am_ref--;
875 amap_free(amap);
876 return;
877 }
878
879 /*
880 * Copy the slots. Zero the padded part.
881 */
882
883 UVMHIST_LOG(maphist, " copying amap now",0, 0, 0, 0);
884 for (lcv = 0 ; lcv < slots; lcv++) {
885 amap->am_anon[lcv] =
886 srcamap->am_anon[entry->aref.ar_pageoff + lcv];
887 if (amap->am_anon[lcv] == NULL)
888 continue;
889 KASSERT(amap->am_anon[lcv]->an_lock == srcamap->am_lock);
890 KASSERT(amap->am_anon[lcv]->an_ref > 0);
891 KASSERT(amap->am_nused < amap->am_maxslot);
892 amap->am_anon[lcv]->an_ref++;
893 amap->am_bckptr[lcv] = amap->am_nused;
894 amap->am_slots[amap->am_nused] = lcv;
895 amap->am_nused++;
896 }
897 memset(&amap->am_anon[lcv], 0,
898 (amap->am_maxslot - lcv) * sizeof(struct vm_anon *));
899
900 /*
901 * Drop our reference to the old amap (srcamap) and unlock.
902 * Since the reference count on srcamap is greater than one,
903 * (we checked above), it cannot drop to zero while it is locked.
904 */
905
906 srcamap->am_ref--;
907 KASSERT(srcamap->am_ref > 0);
908
909 if (srcamap->am_ref == 1 && (srcamap->am_flags & AMAP_SHARED) != 0) {
910 srcamap->am_flags &= ~AMAP_SHARED;
911 }
912 tofree = NULL;
913 #ifdef UVM_AMAP_PPREF
914 if (srcamap->am_ppref && srcamap->am_ppref != PPREF_NONE) {
915 amap_pp_adjref(srcamap, entry->aref.ar_pageoff,
916 len >> PAGE_SHIFT, -1, &tofree);
917 }
918 #endif
919
920 /*
921 * If we referenced any anons, then share the source amap's lock.
922 * Otherwise, we have nothing in common, so allocate a new one.
923 */
924
925 KASSERT(amap->am_lock == NULL);
926 if (amap->am_nused != 0) {
927 amap->am_lock = srcamap->am_lock;
928 mutex_obj_hold(amap->am_lock);
929 }
930 uvm_anon_freelst(srcamap, tofree);
931
932 if (amap->am_lock == NULL) {
933 amap->am_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
934 }
935 amap_list_insert(amap);
936
937 /*
938 * Install new amap.
939 */
940
941 entry->aref.ar_pageoff = 0;
942 entry->aref.ar_amap = amap;
943 entry->etype &= ~UVM_ET_NEEDSCOPY;
944 UVMHIST_LOG(maphist, "<- done",0, 0, 0, 0);
945 }
946
947 /*
948 * amap_cow_now: resolve all copy-on-write faults in an amap now for fork(2)
949 *
950 * called during fork(2) when the parent process has a wired map
951 * entry. in that case we want to avoid write-protecting pages
952 * in the parent's map (e.g. like what you'd do for a COW page)
953 * so we resolve the COW here.
954 *
955 * => assume parent's entry was wired, thus all pages are resident.
956 * => assume pages that are loaned out (loan_count) are already mapped
957 * read-only in all maps, and thus no need for us to worry about them
958 * => assume both parent and child vm_map's are locked
959 * => caller passes child's map/entry in to us
960 * => if we run out of memory we will unlock the amap and sleep _with_ the
961 * parent and child vm_map's locked(!). we have to do this since
962 * we are in the middle of a fork(2) and we can't let the parent
963 * map change until we are done copying all the map entrys.
964 * => XXXCDC: out of memory should cause fork to fail, but there is
965 * currently no easy way to do this (needs fix)
966 * => page queues must be unlocked (we may lock them)
967 */
968
969 void
amap_cow_now(struct vm_map * map,struct vm_map_entry * entry)970 amap_cow_now(struct vm_map *map, struct vm_map_entry *entry)
971 {
972 struct vm_amap *amap = entry->aref.ar_amap;
973 struct vm_anon *anon, *nanon;
974 struct vm_page *pg, *npg;
975 u_int lcv, slot;
976
977 /*
978 * note that if we unlock the amap then we must ReStart the "lcv" for
979 * loop because some other process could reorder the anon's in the
980 * am_anon[] array on us while the lock is dropped.
981 */
982
983 ReStart:
984 amap_lock(amap);
985 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) {
986 slot = amap->am_slots[lcv];
987 anon = amap->am_anon[slot];
988 KASSERT(anon->an_lock == amap->am_lock);
989
990 /*
991 * If anon has only one reference - we must have already
992 * copied it. This can happen if we needed to sleep waiting
993 * for memory in a previous run through this loop. The new
994 * page might even have been paged out, since is not wired.
995 */
996
997 if (anon->an_ref == 1) {
998 KASSERT(anon->an_page != NULL || anon->an_swslot != 0);
999 continue;
1000 }
1001
1002 /*
1003 * The old page must be resident since the parent is wired.
1004 */
1005
1006 pg = anon->an_page;
1007 KASSERT(pg != NULL);
1008 KASSERT(pg->wire_count > 0);
1009
1010 /*
1011 * If the page is loaned then it must already be mapped
1012 * read-only and we don't need to copy it.
1013 */
1014
1015 if (pg->loan_count != 0) {
1016 continue;
1017 }
1018 KASSERT(pg->uanon == anon && pg->uobject == NULL);
1019
1020 /*
1021 * If the page is busy, then we have to unlock, wait for
1022 * it and then restart.
1023 */
1024
1025 if (pg->flags & PG_BUSY) {
1026 pg->flags |= PG_WANTED;
1027 UVM_UNLOCK_AND_WAIT(pg, amap->am_lock, false,
1028 "cownow", 0);
1029 goto ReStart;
1030 }
1031
1032 /*
1033 * Perform a copy-on-write.
1034 * First - get a new anon and a page.
1035 */
1036
1037 nanon = uvm_analloc();
1038 if (nanon) {
1039 nanon->an_lock = amap->am_lock;
1040 npg = uvm_pagealloc(NULL, 0, nanon, 0);
1041 } else {
1042 npg = NULL;
1043 }
1044 if (nanon == NULL || npg == NULL) {
1045 amap_unlock(amap);
1046 if (nanon) {
1047 nanon->an_lock = NULL;
1048 nanon->an_ref--;
1049 KASSERT(nanon->an_ref == 0);
1050 uvm_anon_free(nanon);
1051 }
1052 uvm_wait("cownowpage");
1053 goto ReStart;
1054 }
1055
1056 /*
1057 * Copy the data and replace anon with the new one.
1058 * Also, setup its lock (share the with amap's lock).
1059 */
1060
1061 uvm_pagecopy(pg, npg);
1062 anon->an_ref--;
1063 KASSERT(anon->an_ref > 0);
1064 amap->am_anon[slot] = nanon;
1065
1066 /*
1067 * Drop PG_BUSY on new page. Since its owner was locked all
1068 * this time - it cannot be PG_RELEASED or PG_WANTED.
1069 */
1070
1071 mutex_enter(&uvm_pageqlock);
1072 uvm_pageactivate(npg);
1073 mutex_exit(&uvm_pageqlock);
1074 npg->flags &= ~(PG_BUSY|PG_FAKE);
1075 UVM_PAGE_OWN(npg, NULL);
1076 }
1077 amap_unlock(amap);
1078 }
1079
1080 /*
1081 * amap_splitref: split a single reference into two separate references
1082 *
1083 * => called from uvm_map's clip routines
1084 * => origref's map should be locked
1085 * => origref->ar_amap should be unlocked (we will lock)
1086 */
1087 void
amap_splitref(struct vm_aref * origref,struct vm_aref * splitref,vaddr_t offset)1088 amap_splitref(struct vm_aref *origref, struct vm_aref *splitref, vaddr_t offset)
1089 {
1090 struct vm_amap *amap = origref->ar_amap;
1091 u_int leftslots;
1092
1093 KASSERT(splitref->ar_amap == origref->ar_amap);
1094 AMAP_B2SLOT(leftslots, offset);
1095 KASSERT(leftslots != 0);
1096
1097 amap_lock(amap);
1098 KASSERT(amap->am_nslot - origref->ar_pageoff - leftslots > 0);
1099
1100 #ifdef UVM_AMAP_PPREF
1101 /* Establish ppref before we add a duplicate reference to the amap. */
1102 if (amap->am_ppref == NULL) {
1103 amap_pp_establish(amap, origref->ar_pageoff);
1104 }
1105 #endif
1106 /* Note: not a share reference. */
1107 amap->am_ref++;
1108 splitref->ar_pageoff = origref->ar_pageoff + leftslots;
1109 amap_unlock(amap);
1110 }
1111
1112 #ifdef UVM_AMAP_PPREF
1113
1114 /*
1115 * amap_pp_establish: add a ppref array to an amap, if possible.
1116 *
1117 * => amap should be locked by caller.
1118 */
1119 void
amap_pp_establish(struct vm_amap * amap,vaddr_t offset)1120 amap_pp_establish(struct vm_amap *amap, vaddr_t offset)
1121 {
1122 const size_t sz = amap->am_maxslot * sizeof(*amap->am_ppref);
1123
1124 KASSERT(mutex_owned(amap->am_lock));
1125
1126 amap->am_ppref = kmem_zalloc(sz, KM_NOSLEEP);
1127 if (amap->am_ppref == NULL) {
1128 /* Failure - just do not use ppref. */
1129 amap->am_ppref = PPREF_NONE;
1130 return;
1131 }
1132 pp_setreflen(amap->am_ppref, 0, 0, offset);
1133 pp_setreflen(amap->am_ppref, offset, amap->am_ref,
1134 amap->am_nslot - offset);
1135 }
1136
1137 /*
1138 * amap_pp_adjref: adjust reference count to a part of an amap using the
1139 * per-page reference count array.
1140 *
1141 * => caller must check that ppref != PPREF_NONE before calling.
1142 * => map and amap must be locked.
1143 */
1144 void
amap_pp_adjref(struct vm_amap * amap,int curslot,vsize_t slotlen,int adjval,struct vm_anon ** tofree)1145 amap_pp_adjref(struct vm_amap *amap, int curslot, vsize_t slotlen, int adjval,
1146 struct vm_anon **tofree)
1147 {
1148 int stopslot, *ppref, lcv, prevlcv;
1149 int ref, len, prevref, prevlen;
1150
1151 KASSERT(mutex_owned(amap->am_lock));
1152
1153 stopslot = curslot + slotlen;
1154 ppref = amap->am_ppref;
1155 prevlcv = 0;
1156
1157 /*
1158 * Advance to the correct place in the array, fragment if needed.
1159 */
1160
1161 for (lcv = 0 ; lcv < curslot ; lcv += len) {
1162 pp_getreflen(ppref, lcv, &ref, &len);
1163 if (lcv + len > curslot) { /* goes past start? */
1164 pp_setreflen(ppref, lcv, ref, curslot - lcv);
1165 pp_setreflen(ppref, curslot, ref, len - (curslot -lcv));
1166 len = curslot - lcv; /* new length of entry @ lcv */
1167 }
1168 prevlcv = lcv;
1169 }
1170 if (lcv == 0) {
1171 /*
1172 * Ensure that the "prevref == ref" test below always
1173 * fails, since we are starting from the beginning of
1174 * the ppref array; that is, there is no previous chunk.
1175 */
1176 prevref = -1;
1177 prevlen = 0;
1178 } else {
1179 pp_getreflen(ppref, prevlcv, &prevref, &prevlen);
1180 }
1181
1182 /*
1183 * Now adjust reference counts in range. Merge the first
1184 * changed entry with the last unchanged entry if possible.
1185 */
1186 KASSERT(lcv == curslot);
1187 for (/* lcv already set */; lcv < stopslot ; lcv += len) {
1188 pp_getreflen(ppref, lcv, &ref, &len);
1189 if (lcv + len > stopslot) { /* goes past end? */
1190 pp_setreflen(ppref, lcv, ref, stopslot - lcv);
1191 pp_setreflen(ppref, stopslot, ref,
1192 len - (stopslot - lcv));
1193 len = stopslot - lcv;
1194 }
1195 ref += adjval;
1196 KASSERT(ref >= 0);
1197 KASSERT(ref <= amap->am_ref);
1198 if (lcv == prevlcv + prevlen && ref == prevref) {
1199 pp_setreflen(ppref, prevlcv, ref, prevlen + len);
1200 } else {
1201 pp_setreflen(ppref, lcv, ref, len);
1202 }
1203 if (ref == 0) {
1204 amap_wiperange(amap, lcv, len, tofree);
1205 }
1206 }
1207 }
1208
1209 /*
1210 * amap_wiperange: wipe out a range of an amap.
1211 * Note: different from amap_wipeout because the amap is kept intact.
1212 *
1213 * => Both map and amap must be locked by caller.
1214 */
1215 void
amap_wiperange(struct vm_amap * amap,int slotoff,int slots,struct vm_anon ** tofree)1216 amap_wiperange(struct vm_amap *amap, int slotoff, int slots,
1217 struct vm_anon **tofree)
1218 {
1219 u_int lcv, stop, slotend;
1220 bool byanon;
1221
1222 KASSERT(mutex_owned(amap->am_lock));
1223
1224 /*
1225 * We can either traverse the amap by am_anon or by am_slots.
1226 * Determine which way is less expensive.
1227 */
1228
1229 if (slots < amap->am_nused) {
1230 byanon = true;
1231 lcv = slotoff;
1232 stop = slotoff + slots;
1233 slotend = 0;
1234 } else {
1235 byanon = false;
1236 lcv = 0;
1237 stop = amap->am_nused;
1238 slotend = slotoff + slots;
1239 }
1240
1241 while (lcv < stop) {
1242 struct vm_anon *anon;
1243 u_int curslot, ptr, last;
1244
1245 if (byanon) {
1246 curslot = lcv++; /* lcv advances here */
1247 if (amap->am_anon[curslot] == NULL)
1248 continue;
1249 } else {
1250 curslot = amap->am_slots[lcv];
1251 if (curslot < slotoff || curslot >= slotend) {
1252 lcv++; /* lcv advances here */
1253 continue;
1254 }
1255 stop--; /* drop stop, since anon will be removed */
1256 }
1257 anon = amap->am_anon[curslot];
1258 KASSERT(anon->an_lock == amap->am_lock);
1259
1260 /*
1261 * Remove anon from the amap.
1262 */
1263
1264 amap->am_anon[curslot] = NULL;
1265 ptr = amap->am_bckptr[curslot];
1266 last = amap->am_nused - 1;
1267 if (ptr != last) {
1268 amap->am_slots[ptr] = amap->am_slots[last];
1269 amap->am_bckptr[amap->am_slots[ptr]] = ptr;
1270 }
1271 amap->am_nused--;
1272
1273 /*
1274 * Drop its reference count.
1275 */
1276
1277 KASSERT(anon->an_lock == amap->am_lock);
1278 if (--anon->an_ref == 0) {
1279 /*
1280 * Eliminated the last reference to an anon - defer
1281 * freeing as uvm_anon_freelst() will unlock the amap.
1282 */
1283 anon->an_link = *tofree;
1284 *tofree = anon;
1285 }
1286 }
1287 }
1288
1289 #endif
1290
1291 #if defined(VMSWAP)
1292
1293 /*
1294 * amap_swap_off: pagein anonymous pages in amaps and drop swap slots.
1295 *
1296 * => called with swap_syscall_lock held.
1297 * => note that we don't always traverse all anons.
1298 * eg. amaps being wiped out, released anons.
1299 * => return true if failed.
1300 */
1301
1302 bool
amap_swap_off(int startslot,int endslot)1303 amap_swap_off(int startslot, int endslot)
1304 {
1305 struct vm_amap *am;
1306 struct vm_amap *am_next;
1307 struct vm_amap marker_prev;
1308 struct vm_amap marker_next;
1309 bool rv = false;
1310
1311 #if defined(DIAGNOSTIC)
1312 memset(&marker_prev, 0, sizeof(marker_prev));
1313 memset(&marker_next, 0, sizeof(marker_next));
1314 #endif /* defined(DIAGNOSTIC) */
1315
1316 mutex_enter(&amap_list_lock);
1317 for (am = LIST_FIRST(&amap_list); am != NULL && !rv; am = am_next) {
1318 int i;
1319
1320 LIST_INSERT_BEFORE(am, &marker_prev, am_list);
1321 LIST_INSERT_AFTER(am, &marker_next, am_list);
1322
1323 if (!amap_lock_try(am)) {
1324 mutex_exit(&amap_list_lock);
1325 preempt();
1326 mutex_enter(&amap_list_lock);
1327 am_next = LIST_NEXT(&marker_prev, am_list);
1328 if (am_next == &marker_next) {
1329 am_next = LIST_NEXT(am_next, am_list);
1330 } else {
1331 KASSERT(LIST_NEXT(am_next, am_list) ==
1332 &marker_next);
1333 }
1334 LIST_REMOVE(&marker_prev, am_list);
1335 LIST_REMOVE(&marker_next, am_list);
1336 continue;
1337 }
1338
1339 mutex_exit(&amap_list_lock);
1340
1341 if (am->am_nused <= 0) {
1342 amap_unlock(am);
1343 goto next;
1344 }
1345
1346 for (i = 0; i < am->am_nused; i++) {
1347 int slot;
1348 int swslot;
1349 struct vm_anon *anon;
1350
1351 slot = am->am_slots[i];
1352 anon = am->am_anon[slot];
1353 KASSERT(anon->an_lock == am->am_lock);
1354
1355 swslot = anon->an_swslot;
1356 if (swslot < startslot || endslot <= swslot) {
1357 continue;
1358 }
1359
1360 am->am_flags |= AMAP_SWAPOFF;
1361
1362 rv = uvm_anon_pagein(am, anon);
1363 amap_lock(am);
1364
1365 am->am_flags &= ~AMAP_SWAPOFF;
1366 if (amap_refs(am) == 0) {
1367 amap_wipeout(am);
1368 am = NULL;
1369 break;
1370 }
1371 if (rv) {
1372 break;
1373 }
1374 i = 0;
1375 }
1376
1377 if (am) {
1378 amap_unlock(am);
1379 }
1380
1381 next:
1382 mutex_enter(&amap_list_lock);
1383 KASSERT(LIST_NEXT(&marker_prev, am_list) == &marker_next ||
1384 LIST_NEXT(LIST_NEXT(&marker_prev, am_list), am_list) ==
1385 &marker_next);
1386 am_next = LIST_NEXT(&marker_next, am_list);
1387 LIST_REMOVE(&marker_prev, am_list);
1388 LIST_REMOVE(&marker_next, am_list);
1389 }
1390 mutex_exit(&amap_list_lock);
1391
1392 return rv;
1393 }
1394
1395 #endif /* defined(VMSWAP) */
1396
1397 /*
1398 * amap_lookup: look up a page in an amap.
1399 *
1400 * => amap should be locked by caller.
1401 */
1402 struct vm_anon *
amap_lookup(struct vm_aref * aref,vaddr_t offset)1403 amap_lookup(struct vm_aref *aref, vaddr_t offset)
1404 {
1405 struct vm_amap *amap = aref->ar_amap;
1406 struct vm_anon *an;
1407 u_int slot;
1408
1409 UVMHIST_FUNC("amap_lookup"); UVMHIST_CALLED(maphist);
1410 KASSERT(mutex_owned(amap->am_lock));
1411
1412 AMAP_B2SLOT(slot, offset);
1413 slot += aref->ar_pageoff;
1414 an = amap->am_anon[slot];
1415
1416 UVMHIST_LOG(maphist, "<- done (amap=0x%x, offset=0x%x, result=0x%x)",
1417 amap, offset, an, 0);
1418
1419 KASSERT(slot < amap->am_nslot);
1420 KASSERT(an == NULL || an->an_ref != 0);
1421 KASSERT(an == NULL || an->an_lock == amap->am_lock);
1422 return an;
1423 }
1424
1425 /*
1426 * amap_lookups: look up a range of pages in an amap.
1427 *
1428 * => amap should be locked by caller.
1429 */
1430 void
amap_lookups(struct vm_aref * aref,vaddr_t offset,struct vm_anon ** anons,int npages)1431 amap_lookups(struct vm_aref *aref, vaddr_t offset, struct vm_anon **anons,
1432 int npages)
1433 {
1434 struct vm_amap *amap = aref->ar_amap;
1435 u_int slot;
1436
1437 UVMHIST_FUNC("amap_lookups"); UVMHIST_CALLED(maphist);
1438 KASSERT(mutex_owned(amap->am_lock));
1439
1440 AMAP_B2SLOT(slot, offset);
1441 slot += aref->ar_pageoff;
1442
1443 UVMHIST_LOG(maphist, " slot=%u, npages=%d, nslot=%d",
1444 slot, npages, amap->am_nslot, 0);
1445
1446 KASSERT((slot + (npages - 1)) < amap->am_nslot);
1447 memcpy(anons, &amap->am_anon[slot], npages * sizeof(struct vm_anon *));
1448
1449 #if defined(DIAGNOSTIC)
1450 for (int i = 0; i < npages; i++) {
1451 struct vm_anon * const an = anons[i];
1452 if (an == NULL) {
1453 continue;
1454 }
1455 KASSERT(an->an_ref != 0);
1456 KASSERT(an->an_lock == amap->am_lock);
1457 }
1458 #endif
1459 UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0);
1460 }
1461
1462 /*
1463 * amap_add: add (or replace) a page to an amap.
1464 *
1465 * => amap should be locked by caller.
1466 * => anon must have the lock associated with this amap.
1467 */
1468 void
amap_add(struct vm_aref * aref,vaddr_t offset,struct vm_anon * anon,bool replace)1469 amap_add(struct vm_aref *aref, vaddr_t offset, struct vm_anon *anon,
1470 bool replace)
1471 {
1472 struct vm_amap *amap = aref->ar_amap;
1473 u_int slot;
1474
1475 UVMHIST_FUNC("amap_add"); UVMHIST_CALLED(maphist);
1476 KASSERT(mutex_owned(amap->am_lock));
1477 KASSERT(anon->an_lock == amap->am_lock);
1478
1479 AMAP_B2SLOT(slot, offset);
1480 slot += aref->ar_pageoff;
1481 KASSERT(slot < amap->am_nslot);
1482
1483 if (replace) {
1484 struct vm_anon *oanon = amap->am_anon[slot];
1485
1486 KASSERT(oanon != NULL);
1487 if (oanon->an_page && (amap->am_flags & AMAP_SHARED) != 0) {
1488 pmap_page_protect(oanon->an_page, VM_PROT_NONE);
1489 /*
1490 * XXX: suppose page is supposed to be wired somewhere?
1491 */
1492 }
1493 } else {
1494 KASSERT(amap->am_anon[slot] == NULL);
1495 KASSERT(amap->am_nused < amap->am_maxslot);
1496 amap->am_bckptr[slot] = amap->am_nused;
1497 amap->am_slots[amap->am_nused] = slot;
1498 amap->am_nused++;
1499 }
1500 amap->am_anon[slot] = anon;
1501 UVMHIST_LOG(maphist,
1502 "<- done (amap=0x%x, offset=0x%x, anon=0x%x, rep=%d)",
1503 amap, offset, anon, replace);
1504 }
1505
1506 /*
1507 * amap_unadd: remove a page from an amap.
1508 *
1509 * => amap should be locked by caller.
1510 */
1511 void
amap_unadd(struct vm_aref * aref,vaddr_t offset)1512 amap_unadd(struct vm_aref *aref, vaddr_t offset)
1513 {
1514 struct vm_amap *amap = aref->ar_amap;
1515 u_int slot, ptr, last;
1516
1517 UVMHIST_FUNC("amap_unadd"); UVMHIST_CALLED(maphist);
1518 KASSERT(mutex_owned(amap->am_lock));
1519
1520 AMAP_B2SLOT(slot, offset);
1521 slot += aref->ar_pageoff;
1522 KASSERT(slot < amap->am_nslot);
1523 KASSERT(amap->am_anon[slot] != NULL);
1524 KASSERT(amap->am_anon[slot]->an_lock == amap->am_lock);
1525
1526 amap->am_anon[slot] = NULL;
1527 ptr = amap->am_bckptr[slot];
1528
1529 last = amap->am_nused - 1;
1530 if (ptr != last) {
1531 /* Move the last entry to keep the slots contiguous. */
1532 amap->am_slots[ptr] = amap->am_slots[last];
1533 amap->am_bckptr[amap->am_slots[ptr]] = ptr;
1534 }
1535 amap->am_nused--;
1536 UVMHIST_LOG(maphist, "<- done (amap=0x%x, slot=0x%x)", amap, slot,0, 0);
1537 }
1538
1539 /*
1540 * amap_adjref_anons: adjust the reference count(s) on amap and its anons.
1541 */
1542 static void
amap_adjref_anons(struct vm_amap * amap,vaddr_t offset,vsize_t len,int refv,bool all)1543 amap_adjref_anons(struct vm_amap *amap, vaddr_t offset, vsize_t len,
1544 int refv, bool all)
1545 {
1546 struct vm_anon *tofree = NULL;
1547
1548 #ifdef UVM_AMAP_PPREF
1549 KASSERT(mutex_owned(amap->am_lock));
1550
1551 /*
1552 * We must establish the ppref array before changing am_ref
1553 * so that the ppref values match the current amap refcount.
1554 */
1555
1556 if (amap->am_ppref == NULL && !all && len != amap->am_nslot) {
1557 amap_pp_establish(amap, offset);
1558 }
1559 #endif
1560
1561 amap->am_ref += refv;
1562
1563 #ifdef UVM_AMAP_PPREF
1564 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
1565 if (all) {
1566 amap_pp_adjref(amap, 0, amap->am_nslot, refv, &tofree);
1567 } else {
1568 amap_pp_adjref(amap, offset, len, refv, &tofree);
1569 }
1570 }
1571 #endif
1572 uvm_anon_freelst(amap, tofree);
1573 }
1574
1575 /*
1576 * amap_ref: gain a reference to an amap.
1577 *
1578 * => amap must not be locked (we will lock).
1579 * => "offset" and "len" are in units of pages.
1580 * => Called at fork time to gain the child's reference.
1581 */
1582 void
amap_ref(struct vm_amap * amap,vaddr_t offset,vsize_t len,int flags)1583 amap_ref(struct vm_amap *amap, vaddr_t offset, vsize_t len, int flags)
1584 {
1585 UVMHIST_FUNC("amap_ref"); UVMHIST_CALLED(maphist);
1586
1587 amap_lock(amap);
1588 if (flags & AMAP_SHARED) {
1589 amap->am_flags |= AMAP_SHARED;
1590 }
1591 amap_adjref_anons(amap, offset, len, 1, (flags & AMAP_REFALL) != 0);
1592
1593 UVMHIST_LOG(maphist,"<- done! amap=0x%x", amap, 0, 0, 0);
1594 }
1595
1596 /*
1597 * amap_unref: remove a reference to an amap.
1598 *
1599 * => All pmap-level references to this amap must be already removed.
1600 * => Called from uvm_unmap_detach(); entry is already removed from the map.
1601 * => We will lock amap, so it must be unlocked.
1602 */
1603 void
amap_unref(struct vm_amap * amap,vaddr_t offset,vsize_t len,bool all)1604 amap_unref(struct vm_amap *amap, vaddr_t offset, vsize_t len, bool all)
1605 {
1606 UVMHIST_FUNC("amap_unref"); UVMHIST_CALLED(maphist);
1607
1608 amap_lock(amap);
1609
1610 UVMHIST_LOG(maphist," amap=0x%x refs=%d, nused=%d",
1611 amap, amap->am_ref, amap->am_nused, 0);
1612 KASSERT(amap->am_ref > 0);
1613
1614 if (amap->am_ref == 1) {
1615
1616 /*
1617 * If the last reference - wipeout and destroy the amap.
1618 */
1619 amap->am_ref--;
1620 amap_wipeout(amap);
1621 UVMHIST_LOG(maphist,"<- done (was last ref)!", 0, 0, 0, 0);
1622 return;
1623 }
1624
1625 /*
1626 * Otherwise, drop the reference count(s) on anons.
1627 */
1628
1629 if (amap->am_ref == 2 && (amap->am_flags & AMAP_SHARED) != 0) {
1630 amap->am_flags &= ~AMAP_SHARED;
1631 }
1632 amap_adjref_anons(amap, offset, len, -1, all);
1633
1634 UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
1635 }
1636