1 /* $NetBSD: uvm_physseg.c,v 1.18 2023/04/09 09:00:56 riastradh Exp $ */
2
3 /*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * Copyright (c) 1991, 1993, The Regents of the University of California.
6 *
7 * All rights reserved.
8 *
9 * This code is derived from software contributed to Berkeley by
10 * The Mach Operating System project at Carnegie-Mellon University.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. Neither the name of the University nor the names of its contributors
21 * may be used to endorse or promote products derived from this software
22 * without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 * @(#)vm_page.h 7.3 (Berkeley) 4/21/91
37 * from: Id: uvm_page.h,v 1.1.2.6 1998/02/04 02:31:42 chuck Exp
38 *
39 *
40 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
41 * All rights reserved.
42 *
43 * Permission to use, copy, modify and distribute this software and
44 * its documentation is hereby granted, provided that both the copyright
45 * notice and this permission notice appear in all copies of the
46 * software, derivative works or modified versions, and any portions
47 * thereof, and that both notices appear in supporting documentation.
48 *
49 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
50 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
51 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
52 *
53 * Carnegie Mellon requests users of this software to return to
54 *
55 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
56 * School of Computer Science
57 * Carnegie Mellon University
58 * Pittsburgh PA 15213-3890
59 *
60 * any improvements or extensions that they make and grant Carnegie the
61 * rights to redistribute these changes.
62 */
63
64 /*
65 * Consolidated API from uvm_page.c and others.
66 * Consolidated and designed by Cherry G. Mathew <cherry@zyx.in>
67 * rbtree(3) backing implementation by:
68 * Santhosh N. Raju <santhosh.raju@gmail.com>
69 */
70
71 #ifdef _KERNEL_OPT
72 #include "opt_uvm.h"
73 #endif
74
75 #include <sys/param.h>
76 #include <sys/types.h>
77 #include <sys/extent.h>
78 #include <sys/kmem.h>
79
80 #include <uvm/uvm.h>
81 #include <uvm/uvm_page.h>
82 #include <uvm/uvm_param.h>
83 #include <uvm/uvm_pdpolicy.h>
84 #include <uvm/uvm_physseg.h>
85
86 /*
87 * uvm_physseg: describes one segment of physical memory
88 */
89 struct uvm_physseg {
90 /* used during RB tree lookup for PHYS_TO_VM_PAGE(). */
91 struct rb_node rb_node; /* tree information */
92 paddr_t start; /* PF# of first page in segment */
93 paddr_t end; /* (PF# of last page in segment) + 1 */
94 struct vm_page *pgs; /* vm_page structures (from start) */
95
96 /* less performance sensitive fields. */
97 paddr_t avail_start; /* PF# of first free page in segment */
98 paddr_t avail_end; /* (PF# of last free page in segment) +1 */
99 struct extent *ext; /* extent(9) structure to manage pgs[] */
100 int free_list; /* which free list they belong on */
101 u_int start_hint; /* start looking for free pages here */
102 #ifdef __HAVE_PMAP_PHYSSEG
103 struct pmap_physseg pmseg; /* pmap specific (MD) data */
104 #endif
105 };
106
107 /*
108 * These functions are reserved for uvm(9) internal use and are not
109 * exported in the header file uvm_physseg.h
110 *
111 * Thus they are redefined here.
112 */
113 void uvm_physseg_init_seg(uvm_physseg_t, struct vm_page *);
114 void uvm_physseg_seg_chomp_slab(uvm_physseg_t, struct vm_page *, size_t);
115
116 /* returns a pgs array */
117 struct vm_page *uvm_physseg_seg_alloc_from_slab(uvm_physseg_t, size_t);
118
119 #if defined(UVM_HOTPLUG) /* rbtree impementation */
120
121 #define HANDLE_TO_PHYSSEG_NODE(h) ((struct uvm_physseg *)(h))
122 #define PHYSSEG_NODE_TO_HANDLE(u) ((uvm_physseg_t)(u))
123
124 struct uvm_physseg_graph {
125 struct rb_tree rb_tree; /* Tree for entries */
126 int nentries; /* Number of entries */
127 } __aligned(COHERENCY_UNIT);
128
129 static struct uvm_physseg_graph uvm_physseg_graph __read_mostly;
130
131 /*
132 * Note on kmem(9) allocator usage:
133 * We take the conservative approach that plug/unplug are allowed to
134 * fail in high memory stress situations.
135 *
136 * We want to avoid re-entrant situations in which one plug/unplug
137 * operation is waiting on a previous one to complete, since this
138 * makes the design more complicated than necessary.
139 *
140 * We may review this and change its behaviour, once the use cases
141 * become more obvious.
142 */
143
144 /*
145 * Special alloc()/free() functions for boot time support:
146 * We assume that alloc() at boot time is only for new 'vm_physseg's
147 * This allows us to use a static array for memory allocation at boot
148 * time. Thus we avoid using kmem(9) which is not ready at this point
149 * in boot.
150 *
151 * After kmem(9) is ready, we use it. We currently discard any free()s
152 * to this static array, since the size is small enough to be a
153 * trivial waste on all architectures we run on.
154 */
155
156 static size_t nseg = 0;
157 static struct uvm_physseg uvm_physseg[VM_PHYSSEG_MAX];
158
159 static void *
uvm_physseg_alloc(size_t sz)160 uvm_physseg_alloc(size_t sz)
161 {
162 /*
163 * During boot time, we only support allocating vm_physseg
164 * entries from the static array.
165 * We need to assert for this.
166 */
167
168 if (__predict_false(uvm.page_init_done == false)) {
169 if (sz % sizeof(struct uvm_physseg))
170 panic("%s: tried to alloc size other than multiple"
171 " of struct uvm_physseg at boot\n", __func__);
172
173 size_t n = sz / sizeof(struct uvm_physseg);
174 nseg += n;
175
176 KASSERT(nseg > 0);
177 KASSERT(nseg <= VM_PHYSSEG_MAX);
178
179 return &uvm_physseg[nseg - n];
180 }
181
182 return kmem_zalloc(sz, KM_NOSLEEP);
183 }
184
185 static void
uvm_physseg_free(void * p,size_t sz)186 uvm_physseg_free(void *p, size_t sz)
187 {
188 /*
189 * This is a bit tricky. We do allow simulation of free()
190 * during boot (for eg: when MD code is "steal"ing memory,
191 * and the segment has been exhausted (and thus needs to be
192 * free() - ed.
193 * free() also complicates things because we leak the
194 * free(). Therefore calling code can't assume that free()-ed
195 * memory is available for alloc() again, at boot time.
196 *
197 * Thus we can't explicitly disallow free()s during
198 * boot time. However, the same restriction for alloc()
199 * applies to free(). We only allow uvm_physseg related free()s
200 * via this function during boot time.
201 */
202
203 if (__predict_false(uvm.page_init_done == false)) {
204 if (sz % sizeof(struct uvm_physseg))
205 panic("%s: tried to free size other than struct uvm_physseg"
206 " at boot\n", __func__);
207
208 }
209
210 /*
211 * Could have been in a single if(){} block - split for
212 * clarity
213 */
214
215 if ((struct uvm_physseg *)p >= uvm_physseg &&
216 (struct uvm_physseg *)p < (uvm_physseg + VM_PHYSSEG_MAX)) {
217 if (sz % sizeof(struct uvm_physseg))
218 panic("%s: tried to free() other than struct uvm_physseg"
219 " from static array\n", __func__);
220
221 if ((sz / sizeof(struct uvm_physseg)) >= VM_PHYSSEG_MAX)
222 panic("%s: tried to free() the entire static array!", __func__);
223 return; /* Nothing to free */
224 }
225
226 kmem_free(p, sz);
227 }
228
229 /* XXX: Multi page size */
230 bool
uvm_physseg_plug(paddr_t pfn,size_t pages,uvm_physseg_t * psp)231 uvm_physseg_plug(paddr_t pfn, size_t pages, uvm_physseg_t *psp)
232 {
233 int preload;
234 size_t slabpages;
235 struct uvm_physseg *ps, *current_ps = NULL;
236 struct vm_page *slab = NULL, *pgs = NULL;
237
238 #ifdef DEBUG
239 paddr_t off;
240 uvm_physseg_t upm;
241 upm = uvm_physseg_find(pfn, &off);
242
243 ps = HANDLE_TO_PHYSSEG_NODE(upm);
244
245 if (ps != NULL) /* XXX; do we allow "update" plugs ? */
246 return false;
247 #endif
248
249 /*
250 * do we have room?
251 */
252
253 ps = uvm_physseg_alloc(sizeof (struct uvm_physseg));
254 if (ps == NULL) {
255 printf("uvm_page_physload: unable to load physical memory "
256 "segment\n");
257 printf("\t%d segments allocated, ignoring 0x%"PRIxPADDR" -> 0x%"PRIxPADDR"\n",
258 VM_PHYSSEG_MAX, pfn, pfn + pages + 1);
259 printf("\tincrease VM_PHYSSEG_MAX\n");
260 return false;
261 }
262
263 /* span init */
264 ps->start = pfn;
265 ps->end = pfn + pages;
266
267 /*
268 * XXX: Ugly hack because uvmexp.npages accounts for only
269 * those pages in the segment included below as well - this
270 * should be legacy and removed.
271 */
272
273 ps->avail_start = ps->start;
274 ps->avail_end = ps->end;
275
276 /*
277 * check to see if this is a "preload" (i.e. uvm_page_init hasn't been
278 * called yet, so kmem is not available).
279 */
280
281 preload = 1; /* We are going to assume it is a preload */
282
283 RB_TREE_FOREACH(current_ps, &(uvm_physseg_graph.rb_tree)) {
284 /* If there are non NULL pages then we are not in a preload */
285 if (current_ps->pgs != NULL) {
286 preload = 0;
287 /* Try to scavenge from earlier unplug()s. */
288 pgs = uvm_physseg_seg_alloc_from_slab(current_ps, pages);
289
290 if (pgs != NULL) {
291 break;
292 }
293 }
294 }
295
296
297 /*
298 * if VM is already running, attempt to kmem_alloc vm_page structures
299 */
300
301 if (!preload) {
302 if (pgs == NULL) { /* Brand new */
303 /* Iteratively try alloc down from uvmexp.npages */
304 for (slabpages = (size_t) uvmexp.npages; slabpages >= pages; slabpages--) {
305 slab = kmem_zalloc(sizeof *pgs * (long unsigned int)slabpages, KM_NOSLEEP);
306 if (slab != NULL)
307 break;
308 }
309
310 if (slab == NULL) {
311 uvm_physseg_free(ps, sizeof(struct uvm_physseg));
312 return false;
313 }
314
315 uvm_physseg_seg_chomp_slab(ps, slab, (size_t) slabpages);
316 /* We allocate enough for this plug */
317 pgs = uvm_physseg_seg_alloc_from_slab(ps, pages);
318
319 if (pgs == NULL) {
320 printf("unable to uvm_physseg_seg_alloc_from_slab() from backend\n");
321 return false;
322 }
323 } else {
324 /* Reuse scavenged extent */
325 ps->ext = current_ps->ext;
326 }
327
328 physmem += pages;
329 uvmpdpol_reinit();
330 } else { /* Boot time - see uvm_page.c:uvm_page_init() */
331 pgs = NULL;
332 ps->pgs = pgs;
333 }
334
335 /*
336 * now insert us in the proper place in uvm_physseg_graph.rb_tree
337 */
338
339 current_ps = rb_tree_insert_node(&(uvm_physseg_graph.rb_tree), ps);
340 if (current_ps != ps) {
341 panic("uvm_page_physload: Duplicate address range detected!");
342 }
343 uvm_physseg_graph.nentries++;
344
345 /*
346 * uvm_pagefree() requires the PHYS_TO_VM_PAGE(pgs[i]) on the
347 * newly allocated pgs[] to return the correct value. This is
348 * a bit of a chicken and egg problem, since it needs
349 * uvm_physseg_find() to succeed. For this, the node needs to
350 * be inserted *before* uvm_physseg_init_seg() happens.
351 *
352 * During boot, this happens anyway, since
353 * uvm_physseg_init_seg() is called later on and separately
354 * from uvm_page.c:uvm_page_init().
355 * In the case of hotplug we need to ensure this.
356 */
357
358 if (__predict_true(!preload))
359 uvm_physseg_init_seg(ps, pgs);
360
361 if (psp != NULL)
362 *psp = ps;
363
364 return true;
365 }
366
367 static int
uvm_physseg_compare_nodes(void * ctx,const void * nnode1,const void * nnode2)368 uvm_physseg_compare_nodes(void *ctx, const void *nnode1, const void *nnode2)
369 {
370 const struct uvm_physseg *enode1 = nnode1;
371 const struct uvm_physseg *enode2 = nnode2;
372
373 KASSERT(enode1->start < enode2->start || enode1->start >= enode2->end);
374 KASSERT(enode2->start < enode1->start || enode2->start >= enode1->end);
375
376 if (enode1->start < enode2->start)
377 return -1;
378 if (enode1->start >= enode2->end)
379 return 1;
380 return 0;
381 }
382
383 static int
uvm_physseg_compare_key(void * ctx,const void * nnode,const void * pkey)384 uvm_physseg_compare_key(void *ctx, const void *nnode, const void *pkey)
385 {
386 const struct uvm_physseg *enode = nnode;
387 const paddr_t pa = *(const paddr_t *) pkey;
388
389 if(enode->start <= pa && pa < enode->end)
390 return 0;
391 if (enode->start < pa)
392 return -1;
393 if (enode->end > pa)
394 return 1;
395
396 return 0;
397 }
398
399 static const rb_tree_ops_t uvm_physseg_tree_ops = {
400 .rbto_compare_nodes = uvm_physseg_compare_nodes,
401 .rbto_compare_key = uvm_physseg_compare_key,
402 .rbto_node_offset = offsetof(struct uvm_physseg, rb_node),
403 .rbto_context = NULL
404 };
405
406 /*
407 * uvm_physseg_init: init the physmem
408 *
409 * => physmem unit should not be in use at this point
410 */
411
412 void
uvm_physseg_init(void)413 uvm_physseg_init(void)
414 {
415 rb_tree_init(&(uvm_physseg_graph.rb_tree), &uvm_physseg_tree_ops);
416 uvm_physseg_graph.nentries = 0;
417 }
418
419 uvm_physseg_t
uvm_physseg_get_next(uvm_physseg_t upm)420 uvm_physseg_get_next(uvm_physseg_t upm)
421 {
422 /* next of invalid is invalid, not fatal */
423 if (uvm_physseg_valid_p(upm) == false)
424 return UVM_PHYSSEG_TYPE_INVALID;
425
426 return (uvm_physseg_t) rb_tree_iterate(&(uvm_physseg_graph.rb_tree), upm,
427 RB_DIR_RIGHT);
428 }
429
430 uvm_physseg_t
uvm_physseg_get_prev(uvm_physseg_t upm)431 uvm_physseg_get_prev(uvm_physseg_t upm)
432 {
433 /* prev of invalid is invalid, not fatal */
434 if (uvm_physseg_valid_p(upm) == false)
435 return UVM_PHYSSEG_TYPE_INVALID;
436
437 return (uvm_physseg_t) rb_tree_iterate(&(uvm_physseg_graph.rb_tree), upm,
438 RB_DIR_LEFT);
439 }
440
441 uvm_physseg_t
uvm_physseg_get_last(void)442 uvm_physseg_get_last(void)
443 {
444 return (uvm_physseg_t) RB_TREE_MAX(&(uvm_physseg_graph.rb_tree));
445 }
446
447 uvm_physseg_t
uvm_physseg_get_first(void)448 uvm_physseg_get_first(void)
449 {
450 return (uvm_physseg_t) RB_TREE_MIN(&(uvm_physseg_graph.rb_tree));
451 }
452
453 paddr_t
uvm_physseg_get_highest_frame(void)454 uvm_physseg_get_highest_frame(void)
455 {
456 struct uvm_physseg *ps =
457 (uvm_physseg_t) RB_TREE_MAX(&(uvm_physseg_graph.rb_tree));
458
459 return ps->end - 1;
460 }
461
462 /*
463 * uvm_page_physunload: unload physical memory and return it to
464 * caller.
465 */
466 bool
uvm_page_physunload(uvm_physseg_t upm,int freelist,paddr_t * paddrp)467 uvm_page_physunload(uvm_physseg_t upm, int freelist, paddr_t *paddrp)
468 {
469 struct uvm_physseg *seg;
470
471 if (__predict_true(uvm.page_init_done == true))
472 panic("%s: unload attempted after uvm_page_init()\n", __func__);
473
474 seg = HANDLE_TO_PHYSSEG_NODE(upm);
475
476 if (seg->free_list != freelist) {
477 return false;
478 }
479
480 /*
481 * During cold boot, what we're about to unplug hasn't been
482 * put on the uvm freelist, nor has uvmexp.npages been
483 * updated. (This happens in uvm_page.c:uvm_page_init())
484 *
485 * For hotplug, we assume here that the pages being unloaded
486 * here are completely out of sight of uvm (ie; not on any uvm
487 * lists), and that uvmexp.npages has been suitably
488 * decremented before we're called.
489 *
490 * XXX: will avail_end == start if avail_start < avail_end?
491 */
492
493 /* try from front */
494 if (seg->avail_start == seg->start &&
495 seg->avail_start < seg->avail_end) {
496 *paddrp = ctob(seg->avail_start);
497 return uvm_physseg_unplug(seg->avail_start, 1);
498 }
499
500 /* try from rear */
501 if (seg->avail_end == seg->end &&
502 seg->avail_start < seg->avail_end) {
503 *paddrp = ctob(seg->avail_end - 1);
504 return uvm_physseg_unplug(seg->avail_end - 1, 1);
505 }
506
507 return false;
508 }
509
510 bool
uvm_page_physunload_force(uvm_physseg_t upm,int freelist,paddr_t * paddrp)511 uvm_page_physunload_force(uvm_physseg_t upm, int freelist, paddr_t *paddrp)
512 {
513 struct uvm_physseg *seg;
514
515 seg = HANDLE_TO_PHYSSEG_NODE(upm);
516
517 if (__predict_true(uvm.page_init_done == true))
518 panic("%s: unload attempted after uvm_page_init()\n", __func__);
519 /* any room in this bank? */
520 if (seg->avail_start >= seg->avail_end) {
521 return false; /* nope */
522 }
523
524 *paddrp = ctob(seg->avail_start);
525
526 /* Always unplug from front */
527 return uvm_physseg_unplug(seg->avail_start, 1);
528 }
529
530
531 /*
532 * vm_physseg_find: find vm_physseg structure that belongs to a PA
533 */
534 uvm_physseg_t
uvm_physseg_find(paddr_t pframe,psize_t * offp)535 uvm_physseg_find(paddr_t pframe, psize_t *offp)
536 {
537 struct uvm_physseg * ps = NULL;
538
539 ps = rb_tree_find_node(&(uvm_physseg_graph.rb_tree), &pframe);
540
541 if(ps != NULL && offp != NULL)
542 *offp = pframe - ps->start;
543
544 return ps;
545 }
546
547 #else /* UVM_HOTPLUG */
548
549 /*
550 * physical memory config is stored in vm_physmem.
551 */
552
553 #define VM_PHYSMEM_PTR(i) (&vm_physmem[i])
554 #if VM_PHYSSEG_MAX == 1
555 #define VM_PHYSMEM_PTR_SWAP(i, j) /* impossible */
556 #else
557 #define VM_PHYSMEM_PTR_SWAP(i, j) \
558 do { vm_physmem[(i)] = vm_physmem[(j)]; } while (0)
559 #endif
560
561 #define HANDLE_TO_PHYSSEG_NODE(h) (VM_PHYSMEM_PTR((int)h))
562 #define PHYSSEG_NODE_TO_HANDLE(u) ((int)((vsize_t) (u - vm_physmem) / sizeof(struct uvm_physseg)))
563
564 static struct uvm_physseg vm_physmem[VM_PHYSSEG_MAX]; /* XXXCDC: uvm.physmem */
565 static int vm_nphysseg = 0; /* XXXCDC: uvm.nphysseg */
566 #define vm_nphysmem vm_nphysseg
567
568 void
uvm_physseg_init(void)569 uvm_physseg_init(void)
570 {
571 /* XXX: Provisioning for rb_tree related init(s) */
572 return;
573 }
574
575 int
uvm_physseg_get_next(uvm_physseg_t lcv)576 uvm_physseg_get_next(uvm_physseg_t lcv)
577 {
578 /* next of invalid is invalid, not fatal */
579 if (uvm_physseg_valid_p(lcv) == false)
580 return UVM_PHYSSEG_TYPE_INVALID;
581
582 return (lcv + 1);
583 }
584
585 int
uvm_physseg_get_prev(uvm_physseg_t lcv)586 uvm_physseg_get_prev(uvm_physseg_t lcv)
587 {
588 /* prev of invalid is invalid, not fatal */
589 if (uvm_physseg_valid_p(lcv) == false)
590 return UVM_PHYSSEG_TYPE_INVALID;
591
592 return (lcv - 1);
593 }
594
595 int
uvm_physseg_get_last(void)596 uvm_physseg_get_last(void)
597 {
598 return (vm_nphysseg - 1);
599 }
600
601 int
uvm_physseg_get_first(void)602 uvm_physseg_get_first(void)
603 {
604 return 0;
605 }
606
607 paddr_t
uvm_physseg_get_highest_frame(void)608 uvm_physseg_get_highest_frame(void)
609 {
610 int lcv;
611 paddr_t last = 0;
612 struct uvm_physseg *ps;
613
614 for (lcv = 0; lcv < vm_nphysseg; lcv++) {
615 ps = VM_PHYSMEM_PTR(lcv);
616 if (last < ps->end)
617 last = ps->end;
618 }
619
620 return last;
621 }
622
623
624 static struct vm_page *
uvm_post_preload_check(void)625 uvm_post_preload_check(void)
626 {
627 int preload, lcv;
628
629 /*
630 * check to see if this is a "preload" (i.e. uvm_page_init hasn't been
631 * called yet, so kmem is not available).
632 */
633
634 for (lcv = 0 ; lcv < vm_nphysmem ; lcv++) {
635 if (VM_PHYSMEM_PTR(lcv)->pgs)
636 break;
637 }
638 preload = (lcv == vm_nphysmem);
639
640 /*
641 * if VM is already running, attempt to kmem_alloc vm_page structures
642 */
643
644 if (!preload) {
645 panic("Tried to add RAM after uvm_page_init");
646 }
647
648 return NULL;
649 }
650
651 /*
652 * uvm_page_physunload: unload physical memory and return it to
653 * caller.
654 */
655 bool
uvm_page_physunload(uvm_physseg_t psi,int freelist,paddr_t * paddrp)656 uvm_page_physunload(uvm_physseg_t psi, int freelist, paddr_t *paddrp)
657 {
658 int x;
659 struct uvm_physseg *seg;
660
661 uvm_post_preload_check();
662
663 seg = VM_PHYSMEM_PTR(psi);
664
665 if (seg->free_list != freelist) {
666 return false;
667 }
668
669 /* try from front */
670 if (seg->avail_start == seg->start &&
671 seg->avail_start < seg->avail_end) {
672 *paddrp = ctob(seg->avail_start);
673 seg->avail_start++;
674 seg->start++;
675 /* nothing left? nuke it */
676 if (seg->avail_start == seg->end) {
677 if (vm_nphysmem == 1)
678 panic("uvm_page_physget: out of memory!");
679 vm_nphysmem--;
680 for (x = psi ; x < vm_nphysmem ; x++)
681 /* structure copy */
682 VM_PHYSMEM_PTR_SWAP(x, x + 1);
683 }
684 return (true);
685 }
686
687 /* try from rear */
688 if (seg->avail_end == seg->end &&
689 seg->avail_start < seg->avail_end) {
690 *paddrp = ctob(seg->avail_end - 1);
691 seg->avail_end--;
692 seg->end--;
693 /* nothing left? nuke it */
694 if (seg->avail_end == seg->start) {
695 if (vm_nphysmem == 1)
696 panic("uvm_page_physget: out of memory!");
697 vm_nphysmem--;
698 for (x = psi ; x < vm_nphysmem ; x++)
699 /* structure copy */
700 VM_PHYSMEM_PTR_SWAP(x, x + 1);
701 }
702 return (true);
703 }
704
705 return false;
706 }
707
708 bool
uvm_page_physunload_force(uvm_physseg_t psi,int freelist,paddr_t * paddrp)709 uvm_page_physunload_force(uvm_physseg_t psi, int freelist, paddr_t *paddrp)
710 {
711 int x;
712 struct uvm_physseg *seg;
713
714 uvm_post_preload_check();
715
716 seg = VM_PHYSMEM_PTR(psi);
717
718 /* any room in this bank? */
719 if (seg->avail_start >= seg->avail_end) {
720 return false; /* nope */
721 }
722
723 *paddrp = ctob(seg->avail_start);
724 seg->avail_start++;
725 /* truncate! */
726 seg->start = seg->avail_start;
727
728 /* nothing left? nuke it */
729 if (seg->avail_start == seg->end) {
730 if (vm_nphysmem == 1)
731 panic("uvm_page_physget: out of memory!");
732 vm_nphysmem--;
733 for (x = psi ; x < vm_nphysmem ; x++)
734 /* structure copy */
735 VM_PHYSMEM_PTR_SWAP(x, x + 1);
736 }
737 return (true);
738 }
739
740 bool
uvm_physseg_plug(paddr_t pfn,size_t pages,uvm_physseg_t * psp)741 uvm_physseg_plug(paddr_t pfn, size_t pages, uvm_physseg_t *psp)
742 {
743 int lcv;
744 struct vm_page *pgs;
745 struct uvm_physseg *ps;
746
747 #ifdef DEBUG
748 paddr_t off;
749 uvm_physseg_t upm;
750 upm = uvm_physseg_find(pfn, &off);
751
752 if (uvm_physseg_valid_p(upm)) /* XXX; do we allow "update" plugs ? */
753 return false;
754 #endif
755
756 paddr_t start = pfn;
757 paddr_t end = pfn + pages;
758 paddr_t avail_start = start;
759 paddr_t avail_end = end;
760
761 if (uvmexp.pagesize == 0)
762 panic("uvm_page_physload: page size not set!");
763
764 /*
765 * do we have room?
766 */
767
768 if (vm_nphysmem == VM_PHYSSEG_MAX) {
769 printf("uvm_page_physload: unable to load physical memory "
770 "segment\n");
771 printf("\t%d segments allocated, ignoring 0x%llx -> 0x%llx\n",
772 VM_PHYSSEG_MAX, (long long)start, (long long)end);
773 printf("\tincrease VM_PHYSSEG_MAX\n");
774 if (psp != NULL)
775 *psp = UVM_PHYSSEG_TYPE_INVALID_OVERFLOW;
776 return false;
777 }
778
779 /*
780 * check to see if this is a "preload" (i.e. uvm_page_init hasn't been
781 * called yet, so kmem is not available).
782 */
783 pgs = uvm_post_preload_check();
784
785 /*
786 * now insert us in the proper place in vm_physmem[]
787 */
788
789 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_RANDOM)
790 /* random: put it at the end (easy!) */
791 ps = VM_PHYSMEM_PTR(vm_nphysmem);
792 lcv = vm_nphysmem;
793 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
794 {
795 int x;
796 /* sort by address for binary search */
797 for (lcv = 0 ; lcv < vm_nphysmem ; lcv++)
798 if (start < VM_PHYSMEM_PTR(lcv)->start)
799 break;
800 ps = VM_PHYSMEM_PTR(lcv);
801 /* move back other entries, if necessary ... */
802 for (x = vm_nphysmem ; x > lcv ; x--)
803 /* structure copy */
804 VM_PHYSMEM_PTR_SWAP(x, x - 1);
805 }
806 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
807 {
808 int x;
809 /* sort by largest segment first */
810 for (lcv = 0 ; lcv < vm_nphysmem ; lcv++)
811 if ((end - start) >
812 (VM_PHYSMEM_PTR(lcv)->end - VM_PHYSMEM_PTR(lcv)->start))
813 break;
814 ps = VM_PHYSMEM_PTR(lcv);
815 /* move back other entries, if necessary ... */
816 for (x = vm_nphysmem ; x > lcv ; x--)
817 /* structure copy */
818 VM_PHYSMEM_PTR_SWAP(x, x - 1);
819 }
820 #else
821 panic("uvm_page_physload: unknown physseg strategy selected!");
822 #endif
823
824 ps->start = start;
825 ps->end = end;
826 ps->avail_start = avail_start;
827 ps->avail_end = avail_end;
828
829 ps->pgs = pgs;
830
831 vm_nphysmem++;
832
833 if (psp != NULL)
834 *psp = lcv;
835
836 return true;
837 }
838
839 /*
840 * when VM_PHYSSEG_MAX is 1, we can simplify these functions
841 */
842
843 #if VM_PHYSSEG_MAX == 1
844 static inline int vm_physseg_find_contig(struct uvm_physseg *, int, paddr_t, psize_t *);
845 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
846 static inline int vm_physseg_find_bsearch(struct uvm_physseg *, int, paddr_t, psize_t *);
847 #else
848 static inline int vm_physseg_find_linear(struct uvm_physseg *, int, paddr_t, psize_t *);
849 #endif
850
851 /*
852 * vm_physseg_find: find vm_physseg structure that belongs to a PA
853 */
854 int
uvm_physseg_find(paddr_t pframe,psize_t * offp)855 uvm_physseg_find(paddr_t pframe, psize_t *offp)
856 {
857
858 #if VM_PHYSSEG_MAX == 1
859 return vm_physseg_find_contig(vm_physmem, vm_nphysseg, pframe, offp);
860 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
861 return vm_physseg_find_bsearch(vm_physmem, vm_nphysseg, pframe, offp);
862 #else
863 return vm_physseg_find_linear(vm_physmem, vm_nphysseg, pframe, offp);
864 #endif
865 }
866
867 #if VM_PHYSSEG_MAX == 1
868 static inline int
vm_physseg_find_contig(struct uvm_physseg * segs,int nsegs,paddr_t pframe,psize_t * offp)869 vm_physseg_find_contig(struct uvm_physseg *segs, int nsegs, paddr_t pframe, psize_t *offp)
870 {
871
872 /* 'contig' case */
873 if (pframe >= segs[0].start && pframe < segs[0].end) {
874 if (offp)
875 *offp = pframe - segs[0].start;
876 return(0);
877 }
878 return(-1);
879 }
880
881 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
882
883 static inline int
vm_physseg_find_bsearch(struct uvm_physseg * segs,int nsegs,paddr_t pframe,psize_t * offp)884 vm_physseg_find_bsearch(struct uvm_physseg *segs, int nsegs, paddr_t pframe, psize_t *offp)
885 {
886 /* binary search for it */
887 int start, len, guess;
888
889 /*
890 * if try is too large (thus target is less than try) we reduce
891 * the length to trunc(len/2) [i.e. everything smaller than "try"]
892 *
893 * if the try is too small (thus target is greater than try) then
894 * we set the new start to be (try + 1). this means we need to
895 * reduce the length to (round(len/2) - 1).
896 *
897 * note "adjust" below which takes advantage of the fact that
898 * (round(len/2) - 1) == trunc((len - 1) / 2)
899 * for any value of len we may have
900 */
901
902 for (start = 0, len = nsegs ; len != 0 ; len = len / 2) {
903 guess = start + (len / 2); /* try in the middle */
904
905 /* start past our try? */
906 if (pframe >= segs[guess].start) {
907 /* was try correct? */
908 if (pframe < segs[guess].end) {
909 if (offp)
910 *offp = pframe - segs[guess].start;
911 return guess; /* got it */
912 }
913 start = guess + 1; /* next time, start here */
914 len--; /* "adjust" */
915 } else {
916 /*
917 * pframe before try, just reduce length of
918 * region, done in "for" loop
919 */
920 }
921 }
922 return(-1);
923 }
924
925 #else
926
927 static inline int
vm_physseg_find_linear(struct uvm_physseg * segs,int nsegs,paddr_t pframe,psize_t * offp)928 vm_physseg_find_linear(struct uvm_physseg *segs, int nsegs, paddr_t pframe, psize_t *offp)
929 {
930 /* linear search for it */
931 int lcv;
932
933 for (lcv = 0; lcv < nsegs; lcv++) {
934 if (pframe >= segs[lcv].start &&
935 pframe < segs[lcv].end) {
936 if (offp)
937 *offp = pframe - segs[lcv].start;
938 return(lcv); /* got it */
939 }
940 }
941 return(-1);
942 }
943 #endif
944 #endif /* UVM_HOTPLUG */
945
946 bool
uvm_physseg_valid_p(uvm_physseg_t upm)947 uvm_physseg_valid_p(uvm_physseg_t upm)
948 {
949 struct uvm_physseg *ps;
950
951 if (upm == UVM_PHYSSEG_TYPE_INVALID ||
952 upm == UVM_PHYSSEG_TYPE_INVALID_EMPTY ||
953 upm == UVM_PHYSSEG_TYPE_INVALID_OVERFLOW)
954 return false;
955
956 /*
957 * This is the delicate init dance -
958 * needs to go with the dance.
959 */
960 if (uvm.page_init_done != true)
961 return true;
962
963 ps = HANDLE_TO_PHYSSEG_NODE(upm);
964
965 /* Extra checks needed only post uvm_page_init() */
966 if (ps->pgs == NULL)
967 return false;
968
969 /* XXX: etc. */
970
971 return true;
972
973 }
974
975 /*
976 * Boot protocol dictates that these must be able to return partially
977 * initialised segments.
978 */
979 paddr_t
uvm_physseg_get_start(uvm_physseg_t upm)980 uvm_physseg_get_start(uvm_physseg_t upm)
981 {
982 if (uvm_physseg_valid_p(upm) == false)
983 return (paddr_t) -1;
984
985 return HANDLE_TO_PHYSSEG_NODE(upm)->start;
986 }
987
988 paddr_t
uvm_physseg_get_end(uvm_physseg_t upm)989 uvm_physseg_get_end(uvm_physseg_t upm)
990 {
991 if (uvm_physseg_valid_p(upm) == false)
992 return (paddr_t) -1;
993
994 return HANDLE_TO_PHYSSEG_NODE(upm)->end;
995 }
996
997 paddr_t
uvm_physseg_get_avail_start(uvm_physseg_t upm)998 uvm_physseg_get_avail_start(uvm_physseg_t upm)
999 {
1000 if (uvm_physseg_valid_p(upm) == false)
1001 return (paddr_t) -1;
1002
1003 return HANDLE_TO_PHYSSEG_NODE(upm)->avail_start;
1004 }
1005
1006 #if defined(UVM_PHYSSEG_LEGACY)
1007 void
uvm_physseg_set_avail_start(uvm_physseg_t upm,paddr_t avail_start)1008 uvm_physseg_set_avail_start(uvm_physseg_t upm, paddr_t avail_start)
1009 {
1010 struct uvm_physseg *ps = HANDLE_TO_PHYSSEG_NODE(upm);
1011
1012 #if defined(DIAGNOSTIC)
1013 paddr_t avail_end;
1014 avail_end = uvm_physseg_get_avail_end(upm);
1015 KASSERT(uvm_physseg_valid_p(upm));
1016 KASSERT(avail_start < avail_end);
1017 KASSERT(avail_start >= ps->start);
1018 #endif
1019
1020 ps->avail_start = avail_start;
1021 }
1022
1023 void
uvm_physseg_set_avail_end(uvm_physseg_t upm,paddr_t avail_end)1024 uvm_physseg_set_avail_end(uvm_physseg_t upm, paddr_t avail_end)
1025 {
1026 struct uvm_physseg *ps = HANDLE_TO_PHYSSEG_NODE(upm);
1027
1028 #if defined(DIAGNOSTIC)
1029 paddr_t avail_start;
1030 avail_start = uvm_physseg_get_avail_start(upm);
1031 KASSERT(uvm_physseg_valid_p(upm));
1032 KASSERT(avail_end > avail_start);
1033 KASSERT(avail_end <= ps->end);
1034 #endif
1035
1036 ps->avail_end = avail_end;
1037 }
1038
1039 #endif /* UVM_PHYSSEG_LEGACY */
1040
1041 paddr_t
uvm_physseg_get_avail_end(uvm_physseg_t upm)1042 uvm_physseg_get_avail_end(uvm_physseg_t upm)
1043 {
1044 if (uvm_physseg_valid_p(upm) == false)
1045 return (paddr_t) -1;
1046
1047 return HANDLE_TO_PHYSSEG_NODE(upm)->avail_end;
1048 }
1049
1050 struct vm_page *
uvm_physseg_get_pg(uvm_physseg_t upm,paddr_t idx)1051 uvm_physseg_get_pg(uvm_physseg_t upm, paddr_t idx)
1052 {
1053 KASSERT(uvm_physseg_valid_p(upm));
1054 return &HANDLE_TO_PHYSSEG_NODE(upm)->pgs[idx];
1055 }
1056
1057 #ifdef __HAVE_PMAP_PHYSSEG
1058 struct pmap_physseg *
uvm_physseg_get_pmseg(uvm_physseg_t upm)1059 uvm_physseg_get_pmseg(uvm_physseg_t upm)
1060 {
1061 KASSERT(uvm_physseg_valid_p(upm));
1062 return &(HANDLE_TO_PHYSSEG_NODE(upm)->pmseg);
1063 }
1064 #endif
1065
1066 int
uvm_physseg_get_free_list(uvm_physseg_t upm)1067 uvm_physseg_get_free_list(uvm_physseg_t upm)
1068 {
1069 KASSERT(uvm_physseg_valid_p(upm));
1070 return HANDLE_TO_PHYSSEG_NODE(upm)->free_list;
1071 }
1072
1073 u_int
uvm_physseg_get_start_hint(uvm_physseg_t upm)1074 uvm_physseg_get_start_hint(uvm_physseg_t upm)
1075 {
1076 KASSERT(uvm_physseg_valid_p(upm));
1077 return HANDLE_TO_PHYSSEG_NODE(upm)->start_hint;
1078 }
1079
1080 bool
uvm_physseg_set_start_hint(uvm_physseg_t upm,u_int start_hint)1081 uvm_physseg_set_start_hint(uvm_physseg_t upm, u_int start_hint)
1082 {
1083 if (uvm_physseg_valid_p(upm) == false)
1084 return false;
1085
1086 HANDLE_TO_PHYSSEG_NODE(upm)->start_hint = start_hint;
1087 return true;
1088 }
1089
1090 void
uvm_physseg_init_seg(uvm_physseg_t upm,struct vm_page * pgs)1091 uvm_physseg_init_seg(uvm_physseg_t upm, struct vm_page *pgs)
1092 {
1093 psize_t i;
1094 psize_t n;
1095 paddr_t paddr;
1096 struct uvm_physseg *seg;
1097 struct vm_page *pg;
1098
1099 KASSERT(upm != UVM_PHYSSEG_TYPE_INVALID);
1100 KASSERT(pgs != NULL);
1101
1102 seg = HANDLE_TO_PHYSSEG_NODE(upm);
1103 KASSERT(seg != NULL);
1104 KASSERT(seg->pgs == NULL);
1105
1106 n = seg->end - seg->start;
1107 seg->pgs = pgs;
1108
1109 /* init and free vm_pages (we've already zeroed them) */
1110 paddr = ctob(seg->start);
1111 for (i = 0 ; i < n ; i++, paddr += PAGE_SIZE) {
1112 pg = &seg->pgs[i];
1113 pg->phys_addr = paddr;
1114 #ifdef __HAVE_VM_PAGE_MD
1115 VM_MDPAGE_INIT(pg);
1116 #endif
1117 if (atop(paddr) >= seg->avail_start &&
1118 atop(paddr) < seg->avail_end) {
1119 uvmexp.npages++;
1120 /* add page to free pool */
1121 uvm_page_set_freelist(pg,
1122 uvm_page_lookup_freelist(pg));
1123 /* Disable LOCKDEBUG: too many and too early. */
1124 mutex_init(&pg->interlock, MUTEX_NODEBUG, IPL_NONE);
1125 uvm_pagefree(pg);
1126 }
1127 }
1128 }
1129
1130 void
uvm_physseg_seg_chomp_slab(uvm_physseg_t upm,struct vm_page * pgs,size_t n)1131 uvm_physseg_seg_chomp_slab(uvm_physseg_t upm, struct vm_page *pgs, size_t n)
1132 {
1133 struct uvm_physseg *seg = HANDLE_TO_PHYSSEG_NODE(upm);
1134
1135 /* max number of pre-boot unplug()s allowed */
1136 #define UVM_PHYSSEG_BOOT_UNPLUG_MAX VM_PHYSSEG_MAX
1137
1138 static char btslab_ex_storage[EXTENT_FIXED_STORAGE_SIZE(UVM_PHYSSEG_BOOT_UNPLUG_MAX)];
1139
1140 if (__predict_false(uvm.page_init_done == false)) {
1141 seg->ext = extent_create("Boot time slab", (u_long) pgs, (u_long) (pgs + n),
1142 (void *)btslab_ex_storage, sizeof(btslab_ex_storage), 0);
1143 } else {
1144 seg->ext = extent_create("Hotplug slab", (u_long) pgs, (u_long) (pgs + n), NULL, 0, 0);
1145 }
1146
1147 KASSERT(seg->ext != NULL);
1148
1149 }
1150
1151 struct vm_page *
uvm_physseg_seg_alloc_from_slab(uvm_physseg_t upm,size_t pages)1152 uvm_physseg_seg_alloc_from_slab(uvm_physseg_t upm, size_t pages)
1153 {
1154 int err;
1155 struct uvm_physseg *seg;
1156 struct vm_page *pgs = NULL;
1157
1158 KASSERT(pages > 0);
1159
1160 seg = HANDLE_TO_PHYSSEG_NODE(upm);
1161
1162 if (__predict_false(seg->ext == NULL)) {
1163 /*
1164 * This is a situation unique to boot time.
1165 * It shouldn't happen at any point other than from
1166 * the first uvm_page.c:uvm_page_init() call
1167 * Since we're in a loop, we can get away with the
1168 * below.
1169 */
1170 KASSERT(uvm.page_init_done != true);
1171
1172 uvm_physseg_t upmp = uvm_physseg_get_prev(upm);
1173 KASSERT(upmp != UVM_PHYSSEG_TYPE_INVALID);
1174
1175 seg->ext = HANDLE_TO_PHYSSEG_NODE(upmp)->ext;
1176
1177 KASSERT(seg->ext != NULL);
1178 }
1179
1180 /* We allocate enough for this segment */
1181 err = extent_alloc(seg->ext, sizeof(*pgs) * pages, 1, 0, EX_BOUNDZERO, (u_long *)&pgs);
1182
1183 if (err != 0) {
1184 #ifdef DEBUG
1185 printf("%s: extent_alloc failed with error: %d \n",
1186 __func__, err);
1187 #endif
1188 }
1189
1190 return pgs;
1191 }
1192
1193 /*
1194 * uvm_page_physload: load physical memory into VM system
1195 *
1196 * => all args are PFs
1197 * => all pages in start/end get vm_page structures
1198 * => areas marked by avail_start/avail_end get added to the free page pool
1199 * => we are limited to VM_PHYSSEG_MAX physical memory segments
1200 */
1201
1202 uvm_physseg_t
uvm_page_physload(paddr_t start,paddr_t end,paddr_t avail_start,paddr_t avail_end,int free_list)1203 uvm_page_physload(paddr_t start, paddr_t end, paddr_t avail_start,
1204 paddr_t avail_end, int free_list)
1205 {
1206 struct uvm_physseg *ps;
1207 uvm_physseg_t upm;
1208
1209 if (__predict_true(uvm.page_init_done == true))
1210 panic("%s: unload attempted after uvm_page_init()\n", __func__);
1211 if (uvmexp.pagesize == 0)
1212 panic("uvm_page_physload: page size not set!");
1213 if (free_list >= VM_NFREELIST || free_list < VM_FREELIST_DEFAULT)
1214 panic("uvm_page_physload: bad free list %d", free_list);
1215 if (start >= end)
1216 panic("uvm_page_physload: start[%" PRIxPADDR "] >= end[%"
1217 PRIxPADDR "]", start, end);
1218
1219 if (uvm_physseg_plug(start, end - start, &upm) == false) {
1220 panic("uvm_physseg_plug() failed at boot.");
1221 /* NOTREACHED */
1222 return UVM_PHYSSEG_TYPE_INVALID; /* XXX: correct type */
1223 }
1224
1225 ps = HANDLE_TO_PHYSSEG_NODE(upm);
1226
1227 /* Legacy */
1228 ps->avail_start = avail_start;
1229 ps->avail_end = avail_end;
1230
1231 ps->free_list = free_list; /* XXX: */
1232
1233
1234 return upm;
1235 }
1236
1237 bool
uvm_physseg_unplug(paddr_t pfn,size_t pages)1238 uvm_physseg_unplug(paddr_t pfn, size_t pages)
1239 {
1240 uvm_physseg_t upm;
1241 paddr_t off = 0, start __diagused, end;
1242 struct uvm_physseg *seg;
1243
1244 upm = uvm_physseg_find(pfn, &off);
1245
1246 if (!uvm_physseg_valid_p(upm)) {
1247 printf("%s: Tried to unplug from unknown offset\n", __func__);
1248 return false;
1249 }
1250
1251 seg = HANDLE_TO_PHYSSEG_NODE(upm);
1252
1253 start = uvm_physseg_get_start(upm);
1254 end = uvm_physseg_get_end(upm);
1255
1256 if (end < (pfn + pages)) {
1257 printf("%s: Tried to unplug oversized span \n", __func__);
1258 return false;
1259 }
1260
1261 KASSERT(pfn == start + off); /* sanity */
1262
1263 if (__predict_true(uvm.page_init_done == true)) {
1264 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */
1265 if (extent_free(seg->ext, (u_long)(seg->pgs + off), sizeof(struct vm_page) * pages, EX_MALLOCOK | EX_NOWAIT) != 0)
1266 return false;
1267 }
1268
1269 if (off == 0 && (pfn + pages) == end) {
1270 #if defined(UVM_HOTPLUG) /* rbtree implementation */
1271 int segcount = 0;
1272 struct uvm_physseg *current_ps;
1273 /* Complete segment */
1274 if (uvm_physseg_graph.nentries == 1)
1275 panic("%s: out of memory!", __func__);
1276
1277 if (__predict_true(uvm.page_init_done == true)) {
1278 RB_TREE_FOREACH(current_ps, &(uvm_physseg_graph.rb_tree)) {
1279 if (seg->ext == current_ps->ext)
1280 segcount++;
1281 }
1282 KASSERT(segcount > 0);
1283
1284 if (segcount == 1) {
1285 extent_destroy(seg->ext);
1286 }
1287
1288 /*
1289 * We assume that the unplug will succeed from
1290 * this point onwards
1291 */
1292 uvmexp.npages -= (int) pages;
1293 }
1294
1295 rb_tree_remove_node(&(uvm_physseg_graph.rb_tree), upm);
1296 memset(seg, 0, sizeof(struct uvm_physseg));
1297 uvm_physseg_free(seg, sizeof(struct uvm_physseg));
1298 uvm_physseg_graph.nentries--;
1299 #else /* UVM_HOTPLUG */
1300 int x;
1301 if (vm_nphysmem == 1)
1302 panic("uvm_page_physget: out of memory!");
1303 vm_nphysmem--;
1304 for (x = upm ; x < vm_nphysmem ; x++)
1305 /* structure copy */
1306 VM_PHYSMEM_PTR_SWAP(x, x + 1);
1307 #endif /* UVM_HOTPLUG */
1308 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */
1309 return true;
1310 }
1311
1312 if (off > 0 &&
1313 (pfn + pages) < end) {
1314 #if defined(UVM_HOTPLUG) /* rbtree implementation */
1315 /* middle chunk - need a new segment */
1316 struct uvm_physseg *ps, *current_ps;
1317 ps = uvm_physseg_alloc(sizeof (struct uvm_physseg));
1318 if (ps == NULL) {
1319 printf("%s: Unable to allocated new fragment vm_physseg \n",
1320 __func__);
1321 return false;
1322 }
1323
1324 /* Remove middle chunk */
1325 if (__predict_true(uvm.page_init_done == true)) {
1326 KASSERT(seg->ext != NULL);
1327 ps->ext = seg->ext;
1328
1329 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */
1330 /*
1331 * We assume that the unplug will succeed from
1332 * this point onwards
1333 */
1334 uvmexp.npages -= (int) pages;
1335 }
1336
1337 ps->start = pfn + pages;
1338 ps->avail_start = ps->start; /* XXX: Legacy */
1339
1340 ps->end = seg->end;
1341 ps->avail_end = ps->end; /* XXX: Legacy */
1342
1343 seg->end = pfn;
1344 seg->avail_end = seg->end; /* XXX: Legacy */
1345
1346
1347 /*
1348 * The new pgs array points to the beginning of the
1349 * tail fragment.
1350 */
1351 if (__predict_true(uvm.page_init_done == true))
1352 ps->pgs = seg->pgs + off + pages;
1353
1354 current_ps = rb_tree_insert_node(&(uvm_physseg_graph.rb_tree), ps);
1355 if (current_ps != ps) {
1356 panic("uvm_page_physload: Duplicate address range detected!");
1357 }
1358 uvm_physseg_graph.nentries++;
1359 #else /* UVM_HOTPLUG */
1360 panic("%s: can't unplug() from the middle of a segment without"
1361 " UVM_HOTPLUG\n", __func__);
1362 /* NOTREACHED */
1363 #endif /* UVM_HOTPLUG */
1364 return true;
1365 }
1366
1367 if (off == 0 && (pfn + pages) < end) {
1368 /* Remove front chunk */
1369 if (__predict_true(uvm.page_init_done == true)) {
1370 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */
1371 /*
1372 * We assume that the unplug will succeed from
1373 * this point onwards
1374 */
1375 uvmexp.npages -= (int) pages;
1376 }
1377
1378 /* Truncate */
1379 seg->start = pfn + pages;
1380 seg->avail_start = seg->start; /* XXX: Legacy */
1381
1382 /*
1383 * Move the pgs array start to the beginning of the
1384 * tail end.
1385 */
1386 if (__predict_true(uvm.page_init_done == true))
1387 seg->pgs += pages;
1388
1389 return true;
1390 }
1391
1392 if (off > 0 && (pfn + pages) == end) {
1393 /* back chunk */
1394
1395
1396 /* Truncate! */
1397 seg->end = pfn;
1398 seg->avail_end = seg->end; /* XXX: Legacy */
1399
1400 uvmexp.npages -= (int) pages;
1401
1402 return true;
1403 }
1404
1405 printf("%s: Tried to unplug unknown range \n", __func__);
1406
1407 return false;
1408 }
1409