17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
5ad23a2dbSjohansen * Common Development and Distribution License (the "License").
6ad23a2dbSjohansen * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate /*
2223a80de1SStan Studzinski * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
23e57e118bSJohn Levon * Copyright 2019 Joyent, Inc.
247c478bd9Sstevel@tonic-gate */
257c478bd9Sstevel@tonic-gate
267c478bd9Sstevel@tonic-gate #include <sys/types.h>
277c478bd9Sstevel@tonic-gate #include <sys/t_lock.h>
287c478bd9Sstevel@tonic-gate #include <sys/param.h>
297c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
307c478bd9Sstevel@tonic-gate #include <sys/tuneable.h>
317c478bd9Sstevel@tonic-gate #include <sys/systm.h>
327c478bd9Sstevel@tonic-gate #include <sys/vm.h>
337c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
347c478bd9Sstevel@tonic-gate #include <sys/vmem.h>
357c478bd9Sstevel@tonic-gate #include <sys/mman.h>
367c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
377c478bd9Sstevel@tonic-gate #include <sys/debug.h>
387c478bd9Sstevel@tonic-gate #include <sys/dumphdr.h>
397c478bd9Sstevel@tonic-gate #include <sys/bootconf.h>
407c478bd9Sstevel@tonic-gate #include <sys/lgrp.h>
417c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
427c478bd9Sstevel@tonic-gate #include <vm/hat.h>
437c478bd9Sstevel@tonic-gate #include <vm/page.h>
447c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h>
457c478bd9Sstevel@tonic-gate #include <vm/faultcode.h>
467c478bd9Sstevel@tonic-gate #include <sys/promif.h>
477c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h>
487c478bd9Sstevel@tonic-gate #include <sys/bitmap.h>
497c478bd9Sstevel@tonic-gate #include <sys/mem_cage.h>
507c478bd9Sstevel@tonic-gate
51b48b4544SGangadhar Mylapuram #ifdef __sparc
52b48b4544SGangadhar Mylapuram #include <sys/ivintr.h>
53b48b4544SGangadhar Mylapuram #include <sys/panic.h>
54b48b4544SGangadhar Mylapuram #endif
55b48b4544SGangadhar Mylapuram
567c478bd9Sstevel@tonic-gate /*
577c478bd9Sstevel@tonic-gate * seg_kmem is the primary kernel memory segment driver. It
587c478bd9Sstevel@tonic-gate * maps the kernel heap [kernelheap, ekernelheap), module text,
597c478bd9Sstevel@tonic-gate * and all memory which was allocated before the VM was initialized
607c478bd9Sstevel@tonic-gate * into kas.
617c478bd9Sstevel@tonic-gate *
627c478bd9Sstevel@tonic-gate * Pages which belong to seg_kmem are hashed into &kvp vnode at
637c478bd9Sstevel@tonic-gate * an offset equal to (u_offset_t)virt_addr, and have p_lckcnt >= 1.
647c478bd9Sstevel@tonic-gate * They must never be paged out since segkmem_fault() is a no-op to
657c478bd9Sstevel@tonic-gate * prevent recursive faults.
667c478bd9Sstevel@tonic-gate *
677c478bd9Sstevel@tonic-gate * Currently, seg_kmem pages are sharelocked (p_sharelock == 1) on
687c478bd9Sstevel@tonic-gate * __x86 and are unlocked (p_sharelock == 0) on __sparc. Once __x86
697c478bd9Sstevel@tonic-gate * supports relocation the #ifdef kludges can be removed.
707c478bd9Sstevel@tonic-gate *
717c478bd9Sstevel@tonic-gate * seg_kmem pages may be subject to relocation by page_relocate(),
727c478bd9Sstevel@tonic-gate * provided that the HAT supports it; if this is so, segkmem_reloc
737c478bd9Sstevel@tonic-gate * will be set to a nonzero value. All boot time allocated memory as
747c478bd9Sstevel@tonic-gate * well as static memory is considered off limits to relocation.
757c478bd9Sstevel@tonic-gate * Pages are "relocatable" if p_state does not have P_NORELOC set, so
767c478bd9Sstevel@tonic-gate * we request P_NORELOC pages for memory that isn't safe to relocate.
777c478bd9Sstevel@tonic-gate *
787c478bd9Sstevel@tonic-gate * The kernel heap is logically divided up into four pieces:
797c478bd9Sstevel@tonic-gate *
807c478bd9Sstevel@tonic-gate * heap32_arena is for allocations that require 32-bit absolute
817c478bd9Sstevel@tonic-gate * virtual addresses (e.g. code that uses 32-bit pointers/offsets).
827c478bd9Sstevel@tonic-gate *
837c478bd9Sstevel@tonic-gate * heap_core is for allocations that require 2GB *relative*
847c478bd9Sstevel@tonic-gate * offsets; in other words all memory from heap_core is within
857c478bd9Sstevel@tonic-gate * 2GB of all other memory from the same arena. This is a requirement
867c478bd9Sstevel@tonic-gate * of the addressing modes of some processors in supervisor code.
877c478bd9Sstevel@tonic-gate *
887c478bd9Sstevel@tonic-gate * heap_arena is the general heap arena.
897c478bd9Sstevel@tonic-gate *
907c478bd9Sstevel@tonic-gate * static_arena is the static memory arena. Allocations from it
917c478bd9Sstevel@tonic-gate * are not subject to relocation so it is safe to use the memory
927c478bd9Sstevel@tonic-gate * physical address as well as the virtual address (e.g. the VA to
937c478bd9Sstevel@tonic-gate * PA translations are static). Caches may import from static_arena;
947c478bd9Sstevel@tonic-gate * all other static memory allocations should use static_alloc_arena.
957c478bd9Sstevel@tonic-gate *
967c478bd9Sstevel@tonic-gate * On some platforms which have limited virtual address space, seg_kmem
977c478bd9Sstevel@tonic-gate * may share [kernelheap, ekernelheap) with seg_kp; if this is so,
987c478bd9Sstevel@tonic-gate * segkp_bitmap is non-NULL, and each bit represents a page of virtual
997c478bd9Sstevel@tonic-gate * address space which is actually seg_kp mapped.
1007c478bd9Sstevel@tonic-gate */
1017c478bd9Sstevel@tonic-gate
1027c478bd9Sstevel@tonic-gate extern ulong_t *segkp_bitmap; /* Is set if segkp is from the kernel heap */
1037c478bd9Sstevel@tonic-gate
1047c478bd9Sstevel@tonic-gate char *kernelheap; /* start of primary kernel heap */
1057c478bd9Sstevel@tonic-gate char *ekernelheap; /* end of primary kernel heap */
1067c478bd9Sstevel@tonic-gate struct seg kvseg; /* primary kernel heap segment */
1077c478bd9Sstevel@tonic-gate struct seg kvseg_core; /* "core" kernel heap segment */
108ad23a2dbSjohansen struct seg kzioseg; /* Segment for zio mappings */
1097c478bd9Sstevel@tonic-gate vmem_t *heap_arena; /* primary kernel heap arena */
1107c478bd9Sstevel@tonic-gate vmem_t *heap_core_arena; /* core kernel heap arena */
1117c478bd9Sstevel@tonic-gate char *heap_core_base; /* start of core kernel heap arena */
1127c478bd9Sstevel@tonic-gate char *heap_lp_base; /* start of kernel large page heap arena */
1137c478bd9Sstevel@tonic-gate char *heap_lp_end; /* end of kernel large page heap arena */
1147c478bd9Sstevel@tonic-gate vmem_t *hat_memload_arena; /* HAT translation data */
1157c478bd9Sstevel@tonic-gate struct seg kvseg32; /* 32-bit kernel heap segment */
1167c478bd9Sstevel@tonic-gate vmem_t *heap32_arena; /* 32-bit kernel heap arena */
1177c478bd9Sstevel@tonic-gate vmem_t *heaptext_arena; /* heaptext arena */
1187c478bd9Sstevel@tonic-gate struct as kas; /* kernel address space */
1197c478bd9Sstevel@tonic-gate int segkmem_reloc; /* enable/disable relocatable segkmem pages */
1207c478bd9Sstevel@tonic-gate vmem_t *static_arena; /* arena for caches to import static memory */
1217c478bd9Sstevel@tonic-gate vmem_t *static_alloc_arena; /* arena for allocating static memory */
122ad23a2dbSjohansen vmem_t *zio_arena = NULL; /* arena for allocating zio memory */
123ad23a2dbSjohansen vmem_t *zio_alloc_arena = NULL; /* arena for allocating zio memory */
1247c478bd9Sstevel@tonic-gate
12504909c8cSJohn Levon #if defined(__amd64)
12604909c8cSJohn Levon vmem_t *kvmm_arena; /* arena for vmm VA */
12704909c8cSJohn Levon struct seg kvmmseg; /* Segment for vmm memory */
12804909c8cSJohn Levon #endif
12904909c8cSJohn Levon
1307c478bd9Sstevel@tonic-gate /*
1317c478bd9Sstevel@tonic-gate * seg_kmem driver can map part of the kernel heap with large pages.
1327c478bd9Sstevel@tonic-gate * Currently this functionality is implemented for sparc platforms only.
1337c478bd9Sstevel@tonic-gate *
1347c478bd9Sstevel@tonic-gate * The large page size "segkmem_lpsize" for kernel heap is selected in the
1357c478bd9Sstevel@tonic-gate * platform specific code. It can also be modified via /etc/system file.
1367c478bd9Sstevel@tonic-gate * Setting segkmem_lpsize to PAGESIZE in /etc/system disables usage of large
1377c478bd9Sstevel@tonic-gate * pages for kernel heap. "segkmem_lpshift" is adjusted appropriately to
1387c478bd9Sstevel@tonic-gate * match segkmem_lpsize.
1397c478bd9Sstevel@tonic-gate *
1407c478bd9Sstevel@tonic-gate * At boot time we carve from kernel heap arena a range of virtual addresses
1417c478bd9Sstevel@tonic-gate * that will be used for large page mappings. This range [heap_lp_base,
1427c478bd9Sstevel@tonic-gate * heap_lp_end) is set up as a separate vmem arena - "heap_lp_arena". We also
1437c478bd9Sstevel@tonic-gate * create "kmem_lp_arena" that caches memory already backed up by large
1447c478bd9Sstevel@tonic-gate * pages. kmem_lp_arena imports virtual segments from heap_lp_arena.
1457c478bd9Sstevel@tonic-gate */
1467c478bd9Sstevel@tonic-gate
1477c478bd9Sstevel@tonic-gate size_t segkmem_lpsize;
148081a94b0Saguzovsk int segkmem_lpszc = 0;
1497c478bd9Sstevel@tonic-gate
1507c478bd9Sstevel@tonic-gate size_t segkmem_kmemlp_quantum = 0x400000; /* 4MB */
1517c478bd9Sstevel@tonic-gate size_t segkmem_heaplp_quantum;
1527bc98a2eSeg155566 vmem_t *heap_lp_arena;
1537c478bd9Sstevel@tonic-gate static vmem_t *kmem_lp_arena;
1547c478bd9Sstevel@tonic-gate static segkmem_lpcb_t segkmem_lpcb;
1557c478bd9Sstevel@tonic-gate
156*ae5a8bedSAndy Fiddaman #ifdef __sparc
157*ae5a8bedSAndy Fiddaman static uint_t segkmem_lpshift = PAGESHIFT;
158*ae5a8bedSAndy Fiddaman static vmem_t *segkmem_ppa_arena;
159*ae5a8bedSAndy Fiddaman #endif
160*ae5a8bedSAndy Fiddaman
1617c478bd9Sstevel@tonic-gate /*
1627c478bd9Sstevel@tonic-gate * We use "segkmem_kmemlp_max" to limit the total amount of physical memory
163501f3284Seg155566 * consumed by the large page heap. By default this parameter is set to 1/8 of
1647c478bd9Sstevel@tonic-gate * physmem but can be adjusted through /etc/system either directly or
1657c478bd9Sstevel@tonic-gate * indirectly by setting "segkmem_kmemlp_pcnt" to the percent of physmem
1667c478bd9Sstevel@tonic-gate * we allow for large page heap.
1677c478bd9Sstevel@tonic-gate */
1687c478bd9Sstevel@tonic-gate size_t segkmem_kmemlp_max;
1693df2e8b2SRobert Mustacchi uint_t segkmem_kmemlp_pcnt;
1707c478bd9Sstevel@tonic-gate
1717c478bd9Sstevel@tonic-gate /*
1727c478bd9Sstevel@tonic-gate * Getting large pages for kernel heap could be problematic due to
1737c478bd9Sstevel@tonic-gate * physical memory fragmentation. That's why we allow to preallocate
1747c478bd9Sstevel@tonic-gate * "segkmem_kmemlp_min" bytes at boot time.
1757c478bd9Sstevel@tonic-gate */
1763df2e8b2SRobert Mustacchi size_t segkmem_kmemlp_min;
1777c478bd9Sstevel@tonic-gate
1787c478bd9Sstevel@tonic-gate /*
1797c478bd9Sstevel@tonic-gate * Throttling is used to avoid expensive tries to allocate large pages
1807c478bd9Sstevel@tonic-gate * for kernel heap when a lot of succesive attempts to do so fail.
1817c478bd9Sstevel@tonic-gate */
1827c478bd9Sstevel@tonic-gate static ulong_t segkmem_lpthrottle_max = 0x400000;
1837c478bd9Sstevel@tonic-gate static ulong_t segkmem_lpthrottle_start = 0x40;
1847c478bd9Sstevel@tonic-gate static ulong_t segkmem_use_lpthrottle = 1;
1857c478bd9Sstevel@tonic-gate
1867c478bd9Sstevel@tonic-gate /*
1877c478bd9Sstevel@tonic-gate * Freed pages accumulate on a garbage list until segkmem is ready,
1887c478bd9Sstevel@tonic-gate * at which point we call segkmem_gc() to free it all.
1897c478bd9Sstevel@tonic-gate */
1907c478bd9Sstevel@tonic-gate typedef struct segkmem_gc_list {
1917c478bd9Sstevel@tonic-gate struct segkmem_gc_list *gc_next;
1927c478bd9Sstevel@tonic-gate vmem_t *gc_arena;
1937c478bd9Sstevel@tonic-gate size_t gc_size;
1947c478bd9Sstevel@tonic-gate } segkmem_gc_list_t;
1957c478bd9Sstevel@tonic-gate
1967c478bd9Sstevel@tonic-gate static segkmem_gc_list_t *segkmem_gc_list;
1977c478bd9Sstevel@tonic-gate
1987c478bd9Sstevel@tonic-gate /*
1997c478bd9Sstevel@tonic-gate * Allocations from the hat_memload arena add VM_MEMLOAD to their
2007c478bd9Sstevel@tonic-gate * vmflags so that segkmem_xalloc() can inform the hat layer that it needs
2017c478bd9Sstevel@tonic-gate * to take steps to prevent infinite recursion. HAT allocations also
2027c478bd9Sstevel@tonic-gate * must be non-relocatable to prevent recursive page faults.
2037c478bd9Sstevel@tonic-gate */
2047c478bd9Sstevel@tonic-gate static void *
hat_memload_alloc(vmem_t * vmp,size_t size,int flags)2057c478bd9Sstevel@tonic-gate hat_memload_alloc(vmem_t *vmp, size_t size, int flags)
2067c478bd9Sstevel@tonic-gate {
2077c478bd9Sstevel@tonic-gate flags |= (VM_MEMLOAD | VM_NORELOC);
2087c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, flags));
2097c478bd9Sstevel@tonic-gate }
2107c478bd9Sstevel@tonic-gate
2117c478bd9Sstevel@tonic-gate /*
2127c478bd9Sstevel@tonic-gate * Allocations from static_arena arena (or any other arena that uses
2137c478bd9Sstevel@tonic-gate * segkmem_alloc_permanent()) require non-relocatable (permanently
2147c478bd9Sstevel@tonic-gate * wired) memory pages, since these pages are referenced by physical
2157c478bd9Sstevel@tonic-gate * as well as virtual address.
2167c478bd9Sstevel@tonic-gate */
2177c478bd9Sstevel@tonic-gate void *
segkmem_alloc_permanent(vmem_t * vmp,size_t size,int flags)2187c478bd9Sstevel@tonic-gate segkmem_alloc_permanent(vmem_t *vmp, size_t size, int flags)
2197c478bd9Sstevel@tonic-gate {
2207c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, flags | VM_NORELOC));
2217c478bd9Sstevel@tonic-gate }
2227c478bd9Sstevel@tonic-gate
2237c478bd9Sstevel@tonic-gate /*
2247c478bd9Sstevel@tonic-gate * Initialize kernel heap boundaries.
2257c478bd9Sstevel@tonic-gate */
2267c478bd9Sstevel@tonic-gate void
kernelheap_init(void * heap_start,void * heap_end,char * first_avail,void * core_start,void * core_end)2277c478bd9Sstevel@tonic-gate kernelheap_init(
2287c478bd9Sstevel@tonic-gate void *heap_start,
2297c478bd9Sstevel@tonic-gate void *heap_end,
2307c478bd9Sstevel@tonic-gate char *first_avail,
2317c478bd9Sstevel@tonic-gate void *core_start,
2327c478bd9Sstevel@tonic-gate void *core_end)
2337c478bd9Sstevel@tonic-gate {
2347c478bd9Sstevel@tonic-gate uintptr_t textbase;
2357c478bd9Sstevel@tonic-gate size_t core_size;
2367c478bd9Sstevel@tonic-gate size_t heap_size;
2377c478bd9Sstevel@tonic-gate vmem_t *heaptext_parent;
2387c478bd9Sstevel@tonic-gate size_t heap_lp_size = 0;
239bb121940Sdp78419 #ifdef __sparc
240bb121940Sdp78419 size_t kmem64_sz = kmem64_aligned_end - kmem64_base;
241bb121940Sdp78419 #endif /* __sparc */
2427c478bd9Sstevel@tonic-gate
2437c478bd9Sstevel@tonic-gate kernelheap = heap_start;
2447c478bd9Sstevel@tonic-gate ekernelheap = heap_end;
2457c478bd9Sstevel@tonic-gate
2467c478bd9Sstevel@tonic-gate #ifdef __sparc
2477c478bd9Sstevel@tonic-gate heap_lp_size = (((uintptr_t)heap_end - (uintptr_t)heap_start) / 4);
248bb121940Sdp78419 /*
249bb121940Sdp78419 * Bias heap_lp start address by kmem64_sz to reduce collisions
250bb121940Sdp78419 * in 4M kernel TSB between kmem64 area and heap_lp
251bb121940Sdp78419 */
252bb121940Sdp78419 kmem64_sz = P2ROUNDUP(kmem64_sz, MMU_PAGESIZE256M);
253bb121940Sdp78419 if (kmem64_sz <= heap_lp_size / 2)
254bb121940Sdp78419 heap_lp_size -= kmem64_sz;
2557c478bd9Sstevel@tonic-gate heap_lp_base = ekernelheap - heap_lp_size;
2567c478bd9Sstevel@tonic-gate heap_lp_end = heap_lp_base + heap_lp_size;
2577c478bd9Sstevel@tonic-gate #endif /* __sparc */
2587c478bd9Sstevel@tonic-gate
2597c478bd9Sstevel@tonic-gate /*
2607c478bd9Sstevel@tonic-gate * If this platform has a 'core' heap area, then the space for
2617c478bd9Sstevel@tonic-gate * overflow module text should be carved out of the end of that
2627c478bd9Sstevel@tonic-gate * heap. Otherwise, it gets carved out of the general purpose
2637c478bd9Sstevel@tonic-gate * heap.
2647c478bd9Sstevel@tonic-gate */
2657c478bd9Sstevel@tonic-gate core_size = (uintptr_t)core_end - (uintptr_t)core_start;
2667c478bd9Sstevel@tonic-gate if (core_size > 0) {
2677c478bd9Sstevel@tonic-gate ASSERT(core_size >= HEAPTEXT_SIZE);
2687c478bd9Sstevel@tonic-gate textbase = (uintptr_t)core_end - HEAPTEXT_SIZE;
2697c478bd9Sstevel@tonic-gate core_size -= HEAPTEXT_SIZE;
2707c478bd9Sstevel@tonic-gate }
2717c478bd9Sstevel@tonic-gate #ifndef __sparc
2727c478bd9Sstevel@tonic-gate else {
2737c478bd9Sstevel@tonic-gate ekernelheap -= HEAPTEXT_SIZE;
2747c478bd9Sstevel@tonic-gate textbase = (uintptr_t)ekernelheap;
2757c478bd9Sstevel@tonic-gate }
2767c478bd9Sstevel@tonic-gate #endif
2777c478bd9Sstevel@tonic-gate
2787c478bd9Sstevel@tonic-gate heap_size = (uintptr_t)ekernelheap - (uintptr_t)kernelheap;
2797c478bd9Sstevel@tonic-gate heap_arena = vmem_init("heap", kernelheap, heap_size, PAGESIZE,
2807c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free);
2817c478bd9Sstevel@tonic-gate
2827c478bd9Sstevel@tonic-gate if (core_size > 0) {
2837c478bd9Sstevel@tonic-gate heap_core_arena = vmem_create("heap_core", core_start,
2847c478bd9Sstevel@tonic-gate core_size, PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP);
2857c478bd9Sstevel@tonic-gate heap_core_base = core_start;
2867c478bd9Sstevel@tonic-gate } else {
2877c478bd9Sstevel@tonic-gate heap_core_arena = heap_arena;
2887c478bd9Sstevel@tonic-gate heap_core_base = kernelheap;
2897c478bd9Sstevel@tonic-gate }
2907c478bd9Sstevel@tonic-gate
2917c478bd9Sstevel@tonic-gate /*
2927c478bd9Sstevel@tonic-gate * reserve space for the large page heap. If large pages for kernel
2937c478bd9Sstevel@tonic-gate * heap is enabled large page heap arean will be created later in the
2947c478bd9Sstevel@tonic-gate * boot sequence in segkmem_heap_lp_init(). Otherwise the allocated
2957c478bd9Sstevel@tonic-gate * range will be returned back to the heap_arena.
2967c478bd9Sstevel@tonic-gate */
2977c478bd9Sstevel@tonic-gate if (heap_lp_size) {
2987c478bd9Sstevel@tonic-gate (void) vmem_xalloc(heap_arena, heap_lp_size, PAGESIZE, 0, 0,
2997c478bd9Sstevel@tonic-gate heap_lp_base, heap_lp_end,
3007c478bd9Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3017c478bd9Sstevel@tonic-gate }
3027c478bd9Sstevel@tonic-gate
3037c478bd9Sstevel@tonic-gate /*
3047c478bd9Sstevel@tonic-gate * Remove the already-spoken-for memory range [kernelheap, first_avail).
3057c478bd9Sstevel@tonic-gate */
3067c478bd9Sstevel@tonic-gate (void) vmem_xalloc(heap_arena, first_avail - kernelheap, PAGESIZE,
3077c478bd9Sstevel@tonic-gate 0, 0, kernelheap, first_avail, VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3087c478bd9Sstevel@tonic-gate
3097c478bd9Sstevel@tonic-gate #ifdef __sparc
3107c478bd9Sstevel@tonic-gate heap32_arena = vmem_create("heap32", (void *)SYSBASE32,
3117c478bd9Sstevel@tonic-gate SYSLIMIT32 - SYSBASE32 - HEAPTEXT_SIZE, PAGESIZE, NULL,
3127c478bd9Sstevel@tonic-gate NULL, NULL, 0, VM_SLEEP);
313b48b4544SGangadhar Mylapuram /*
314b48b4544SGangadhar Mylapuram * Prom claims the physical and virtual resources used by panicbuf
315b48b4544SGangadhar Mylapuram * and inter_vec_table. So reserve space for panicbuf, intr_vec_table,
316b48b4544SGangadhar Mylapuram * reserved interrupt vector data structures from 32-bit heap.
317b48b4544SGangadhar Mylapuram */
318b48b4544SGangadhar Mylapuram (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0,
319b48b4544SGangadhar Mylapuram panicbuf, panicbuf + PANICBUFSIZE,
320b48b4544SGangadhar Mylapuram VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
321b48b4544SGangadhar Mylapuram
322b48b4544SGangadhar Mylapuram (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0,
323b48b4544SGangadhar Mylapuram intr_vec_table, (caddr_t)intr_vec_table + IVSIZE,
324b48b4544SGangadhar Mylapuram VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3257c478bd9Sstevel@tonic-gate
3267c478bd9Sstevel@tonic-gate textbase = SYSLIMIT32 - HEAPTEXT_SIZE;
3277c478bd9Sstevel@tonic-gate heaptext_parent = NULL;
3287c478bd9Sstevel@tonic-gate #else /* __sparc */
3297c478bd9Sstevel@tonic-gate heap32_arena = heap_core_arena;
3307c478bd9Sstevel@tonic-gate heaptext_parent = heap_core_arena;
3317c478bd9Sstevel@tonic-gate #endif /* __sparc */
3327c478bd9Sstevel@tonic-gate
3337c478bd9Sstevel@tonic-gate heaptext_arena = vmem_create("heaptext", (void *)textbase,
3347c478bd9Sstevel@tonic-gate HEAPTEXT_SIZE, PAGESIZE, NULL, NULL, heaptext_parent, 0, VM_SLEEP);
3357c478bd9Sstevel@tonic-gate
3367c478bd9Sstevel@tonic-gate /*
3377c478bd9Sstevel@tonic-gate * Create a set of arenas for memory with static translations
3387c478bd9Sstevel@tonic-gate * (e.g. VA -> PA translations cannot change). Since using
3397c478bd9Sstevel@tonic-gate * kernel pages by physical address implies it isn't safe to
3407c478bd9Sstevel@tonic-gate * walk across page boundaries, the static_arena quantum must
3417c478bd9Sstevel@tonic-gate * be PAGESIZE. Any kmem caches that require static memory
3427c478bd9Sstevel@tonic-gate * should source from static_arena, while direct allocations
3437c478bd9Sstevel@tonic-gate * should only use static_alloc_arena.
3447c478bd9Sstevel@tonic-gate */
3457c478bd9Sstevel@tonic-gate static_arena = vmem_create("static", NULL, 0, PAGESIZE,
3467c478bd9Sstevel@tonic-gate segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP);
3477c478bd9Sstevel@tonic-gate static_alloc_arena = vmem_create("static_alloc", NULL, 0,
3487c478bd9Sstevel@tonic-gate sizeof (uint64_t), vmem_alloc, vmem_free, static_arena,
3497c478bd9Sstevel@tonic-gate 0, VM_SLEEP);
3507c478bd9Sstevel@tonic-gate
3517c478bd9Sstevel@tonic-gate /*
3527c478bd9Sstevel@tonic-gate * Create an arena for translation data (ptes, hmes, or hblks).
3537c478bd9Sstevel@tonic-gate * We need an arena for this because hat_memload() is essential
3547c478bd9Sstevel@tonic-gate * to vmem_populate() (see comments in common/os/vmem.c).
3557c478bd9Sstevel@tonic-gate *
3567c478bd9Sstevel@tonic-gate * Note: any kmem cache that allocates from hat_memload_arena
3577c478bd9Sstevel@tonic-gate * must be created as a KMC_NOHASH cache (i.e. no external slab
3587c478bd9Sstevel@tonic-gate * and bufctl structures to allocate) so that slab creation doesn't
3597c478bd9Sstevel@tonic-gate * require anything more than a single vmem_alloc().
3607c478bd9Sstevel@tonic-gate */
3617c478bd9Sstevel@tonic-gate hat_memload_arena = vmem_create("hat_memload", NULL, 0, PAGESIZE,
3627c478bd9Sstevel@tonic-gate hat_memload_alloc, segkmem_free, heap_arena, 0,
3639dd77bc8SDave Plauger VM_SLEEP | VMC_POPULATOR | VMC_DUMPSAFE);
3647c478bd9Sstevel@tonic-gate }
3657c478bd9Sstevel@tonic-gate
3667c478bd9Sstevel@tonic-gate void
boot_mapin(caddr_t addr,size_t size)3677c478bd9Sstevel@tonic-gate boot_mapin(caddr_t addr, size_t size)
3687c478bd9Sstevel@tonic-gate {
3697c478bd9Sstevel@tonic-gate caddr_t eaddr;
3707c478bd9Sstevel@tonic-gate page_t *pp;
3717c478bd9Sstevel@tonic-gate pfn_t pfnum;
3727c478bd9Sstevel@tonic-gate
3737c478bd9Sstevel@tonic-gate if (page_resv(btop(size), KM_NOSLEEP) == 0)
3747c478bd9Sstevel@tonic-gate panic("boot_mapin: page_resv failed");
3757c478bd9Sstevel@tonic-gate
3767c478bd9Sstevel@tonic-gate for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
3777c478bd9Sstevel@tonic-gate pfnum = va_to_pfn(addr);
3780cfdb603Sjosephb if (pfnum == PFN_INVALID)
3790cfdb603Sjosephb continue;
3807c478bd9Sstevel@tonic-gate if ((pp = page_numtopp_nolock(pfnum)) == NULL)
3817c478bd9Sstevel@tonic-gate panic("boot_mapin(): No pp for pfnum = %lx", pfnum);
3827c478bd9Sstevel@tonic-gate
3837c478bd9Sstevel@tonic-gate /*
3847c478bd9Sstevel@tonic-gate * must break up any large pages that may have constituent
3857c478bd9Sstevel@tonic-gate * pages being utilized for BOP_ALLOC()'s before calling
3867c478bd9Sstevel@tonic-gate * page_numtopp().The locking code (ie. page_reclaim())
3877c478bd9Sstevel@tonic-gate * can't handle them
3887c478bd9Sstevel@tonic-gate */
3897c478bd9Sstevel@tonic-gate if (pp->p_szc != 0)
3907c478bd9Sstevel@tonic-gate page_boot_demote(pp);
3917c478bd9Sstevel@tonic-gate
3927c478bd9Sstevel@tonic-gate pp = page_numtopp(pfnum, SE_EXCL);
3937c478bd9Sstevel@tonic-gate if (pp == NULL || PP_ISFREE(pp))
3947c478bd9Sstevel@tonic-gate panic("boot_alloc: pp is NULL or free");
3957c478bd9Sstevel@tonic-gate
3967c478bd9Sstevel@tonic-gate /*
3977c478bd9Sstevel@tonic-gate * If the cage is on but doesn't yet contain this page,
3987c478bd9Sstevel@tonic-gate * mark it as non-relocatable.
3997c478bd9Sstevel@tonic-gate */
40000e145c7Skchow if (kcage_on && !PP_ISNORELOC(pp)) {
4017c478bd9Sstevel@tonic-gate PP_SETNORELOC(pp);
40200e145c7Skchow PLCNT_XFER_NORELOC(pp);
40300e145c7Skchow }
4047c478bd9Sstevel@tonic-gate
4057c478bd9Sstevel@tonic-gate (void) page_hashin(pp, &kvp, (u_offset_t)(uintptr_t)addr, NULL);
4067c478bd9Sstevel@tonic-gate pp->p_lckcnt = 1;
4077c478bd9Sstevel@tonic-gate #if defined(__x86)
4087c478bd9Sstevel@tonic-gate page_downgrade(pp);
4097c478bd9Sstevel@tonic-gate #else
4107c478bd9Sstevel@tonic-gate page_unlock(pp);
4117c478bd9Sstevel@tonic-gate #endif
4127c478bd9Sstevel@tonic-gate }
4137c478bd9Sstevel@tonic-gate }
4147c478bd9Sstevel@tonic-gate
4157c478bd9Sstevel@tonic-gate /*
4167c478bd9Sstevel@tonic-gate * Get pages from boot and hash them into the kernel's vp.
4177c478bd9Sstevel@tonic-gate * Used after page structs have been allocated, but before segkmem is ready.
4187c478bd9Sstevel@tonic-gate */
4197c478bd9Sstevel@tonic-gate void *
boot_alloc(void * inaddr,size_t size,uint_t align)4207c478bd9Sstevel@tonic-gate boot_alloc(void *inaddr, size_t size, uint_t align)
4217c478bd9Sstevel@tonic-gate {
4227c478bd9Sstevel@tonic-gate caddr_t addr = inaddr;
4237c478bd9Sstevel@tonic-gate
4247c478bd9Sstevel@tonic-gate if (bootops == NULL)
4257c478bd9Sstevel@tonic-gate prom_panic("boot_alloc: attempt to allocate memory after "
4267c478bd9Sstevel@tonic-gate "BOP_GONE");
4277c478bd9Sstevel@tonic-gate
4287c478bd9Sstevel@tonic-gate size = ptob(btopr(size));
429ca622e3aSsvemuri #ifdef __sparc
430ca622e3aSsvemuri if (bop_alloc_chunk(addr, size, align) != (caddr_t)addr)
431ca622e3aSsvemuri panic("boot_alloc: bop_alloc_chunk failed");
432ca622e3aSsvemuri #else
4337c478bd9Sstevel@tonic-gate if (BOP_ALLOC(bootops, addr, size, align) != addr)
4347c478bd9Sstevel@tonic-gate panic("boot_alloc: BOP_ALLOC failed");
435ca622e3aSsvemuri #endif
4367c478bd9Sstevel@tonic-gate boot_mapin((caddr_t)addr, size);
4377c478bd9Sstevel@tonic-gate return (addr);
4387c478bd9Sstevel@tonic-gate }
4397c478bd9Sstevel@tonic-gate
4407c478bd9Sstevel@tonic-gate static void
segkmem_badop()4417c478bd9Sstevel@tonic-gate segkmem_badop()
4427c478bd9Sstevel@tonic-gate {
4437c478bd9Sstevel@tonic-gate panic("segkmem_badop");
4447c478bd9Sstevel@tonic-gate }
4457c478bd9Sstevel@tonic-gate
446027bcc9fSToomas Soome #define SEGKMEM_BADOP(t) (t(*)())(uintptr_t)segkmem_badop
4477c478bd9Sstevel@tonic-gate
4487c478bd9Sstevel@tonic-gate /*ARGSUSED*/
4497c478bd9Sstevel@tonic-gate static faultcode_t
segkmem_fault(struct hat * hat,struct seg * seg,caddr_t addr,size_t size,enum fault_type type,enum seg_rw rw)4507c478bd9Sstevel@tonic-gate segkmem_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t size,
4517c478bd9Sstevel@tonic-gate enum fault_type type, enum seg_rw rw)
4527c478bd9Sstevel@tonic-gate {
4534fc2445aSelowe pgcnt_t npages;
4544fc2445aSelowe spgcnt_t pg;
4554fc2445aSelowe page_t *pp;
456ad23a2dbSjohansen struct vnode *vp = seg->s_data;
4574fc2445aSelowe
4587c478bd9Sstevel@tonic-gate ASSERT(RW_READ_HELD(&seg->s_as->a_lock));
4597c478bd9Sstevel@tonic-gate
4607c478bd9Sstevel@tonic-gate if (seg->s_as != &kas || size > seg->s_size ||
4617c478bd9Sstevel@tonic-gate addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
4627c478bd9Sstevel@tonic-gate panic("segkmem_fault: bad args");
4637c478bd9Sstevel@tonic-gate
4647c478bd9Sstevel@tonic-gate /*
4657c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call segkp_fault.
4667c478bd9Sstevel@tonic-gate */
46735b1ab99Sjosephb if (segkp_bitmap && seg == &kvseg &&
46835b1ab99Sjosephb BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
4697c478bd9Sstevel@tonic-gate return (SEGOP_FAULT(hat, segkp, addr, size, type, rw));
4707c478bd9Sstevel@tonic-gate
4714fc2445aSelowe if (rw != S_READ && rw != S_WRITE && rw != S_OTHER)
4724fc2445aSelowe return (FC_NOSUPPORT);
4734fc2445aSelowe
4744fc2445aSelowe npages = btopr(size);
4754fc2445aSelowe
4767c478bd9Sstevel@tonic-gate switch (type) {
4777c478bd9Sstevel@tonic-gate case F_SOFTLOCK: /* lock down already-loaded translations */
4784fc2445aSelowe for (pg = 0; pg < npages; pg++) {
479ad23a2dbSjohansen pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
4804fc2445aSelowe SE_SHARED);
4814fc2445aSelowe if (pp == NULL) {
4824fc2445aSelowe /*
4834fc2445aSelowe * Hmm, no page. Does a kernel mapping
4844fc2445aSelowe * exist for it?
4854fc2445aSelowe */
4864fc2445aSelowe if (!hat_probe(kas.a_hat, addr)) {
4874fc2445aSelowe addr -= PAGESIZE;
4884fc2445aSelowe while (--pg >= 0) {
48935b1ab99Sjosephb pp = page_find(vp, (u_offset_t)
49035b1ab99Sjosephb (uintptr_t)addr);
4914fc2445aSelowe if (pp)
4924fc2445aSelowe page_unlock(pp);
4934fc2445aSelowe addr -= PAGESIZE;
4944fc2445aSelowe }
4954fc2445aSelowe return (FC_NOMAP);
4964fc2445aSelowe }
4974fc2445aSelowe }
4984fc2445aSelowe addr += PAGESIZE;
4994fc2445aSelowe }
5004fc2445aSelowe if (rw == S_OTHER)
5017c478bd9Sstevel@tonic-gate hat_reserve(seg->s_as, addr, size);
5027c478bd9Sstevel@tonic-gate return (0);
5037c478bd9Sstevel@tonic-gate case F_SOFTUNLOCK:
5044fc2445aSelowe while (npages--) {
505ad23a2dbSjohansen pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
5064fc2445aSelowe if (pp)
5074fc2445aSelowe page_unlock(pp);
5084fc2445aSelowe addr += PAGESIZE;
5097c478bd9Sstevel@tonic-gate }
5104fc2445aSelowe return (0);
5114fc2445aSelowe default:
5127c478bd9Sstevel@tonic-gate return (FC_NOSUPPORT);
5137c478bd9Sstevel@tonic-gate }
5144fc2445aSelowe /*NOTREACHED*/
5154fc2445aSelowe }
5167c478bd9Sstevel@tonic-gate
5177c478bd9Sstevel@tonic-gate static int
segkmem_setprot(struct seg * seg,caddr_t addr,size_t size,uint_t prot)5187c478bd9Sstevel@tonic-gate segkmem_setprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
5197c478bd9Sstevel@tonic-gate {
5207c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
5217c478bd9Sstevel@tonic-gate
5227c478bd9Sstevel@tonic-gate if (seg->s_as != &kas || size > seg->s_size ||
5237c478bd9Sstevel@tonic-gate addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
5247c478bd9Sstevel@tonic-gate panic("segkmem_setprot: bad args");
5257c478bd9Sstevel@tonic-gate
5267c478bd9Sstevel@tonic-gate /*
5277c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call segkp.
5287c478bd9Sstevel@tonic-gate */
52935b1ab99Sjosephb if (segkp_bitmap && seg == &kvseg &&
53035b1ab99Sjosephb BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
5317c478bd9Sstevel@tonic-gate return (SEGOP_SETPROT(segkp, addr, size, prot));
5327c478bd9Sstevel@tonic-gate
5337c478bd9Sstevel@tonic-gate if (prot == 0)
5347c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, addr, size, HAT_UNLOAD);
5357c478bd9Sstevel@tonic-gate else
5367c478bd9Sstevel@tonic-gate hat_chgprot(kas.a_hat, addr, size, prot);
5377c478bd9Sstevel@tonic-gate return (0);
5387c478bd9Sstevel@tonic-gate }
5397c478bd9Sstevel@tonic-gate
5407c478bd9Sstevel@tonic-gate /*
5417c478bd9Sstevel@tonic-gate * This is a dummy segkmem function overloaded to call segkp
5427c478bd9Sstevel@tonic-gate * when segkp is under the heap.
5437c478bd9Sstevel@tonic-gate */
5447c478bd9Sstevel@tonic-gate /* ARGSUSED */
5457c478bd9Sstevel@tonic-gate static int
segkmem_checkprot(struct seg * seg,caddr_t addr,size_t size,uint_t prot)5467c478bd9Sstevel@tonic-gate segkmem_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
5477c478bd9Sstevel@tonic-gate {
5487c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
5497c478bd9Sstevel@tonic-gate
5507c478bd9Sstevel@tonic-gate if (seg->s_as != &kas)
5517c478bd9Sstevel@tonic-gate segkmem_badop();
5527c478bd9Sstevel@tonic-gate
5537c478bd9Sstevel@tonic-gate /*
5547c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp.
5557c478bd9Sstevel@tonic-gate */
55635b1ab99Sjosephb if (segkp_bitmap && seg == &kvseg &&
55735b1ab99Sjosephb BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
5587c478bd9Sstevel@tonic-gate return (SEGOP_CHECKPROT(segkp, addr, size, prot));
55935b1ab99Sjosephb
5607c478bd9Sstevel@tonic-gate segkmem_badop();
5617c478bd9Sstevel@tonic-gate return (0);
5627c478bd9Sstevel@tonic-gate }
5637c478bd9Sstevel@tonic-gate
5647c478bd9Sstevel@tonic-gate /*
5657c478bd9Sstevel@tonic-gate * This is a dummy segkmem function overloaded to call segkp
5667c478bd9Sstevel@tonic-gate * when segkp is under the heap.
5677c478bd9Sstevel@tonic-gate */
5687c478bd9Sstevel@tonic-gate /* ARGSUSED */
5697c478bd9Sstevel@tonic-gate static int
segkmem_kluster(struct seg * seg,caddr_t addr,ssize_t delta)5707c478bd9Sstevel@tonic-gate segkmem_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
5717c478bd9Sstevel@tonic-gate {
5727c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
5737c478bd9Sstevel@tonic-gate
5747c478bd9Sstevel@tonic-gate if (seg->s_as != &kas)
5757c478bd9Sstevel@tonic-gate segkmem_badop();
5767c478bd9Sstevel@tonic-gate
5777c478bd9Sstevel@tonic-gate /*
5787c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp.
5797c478bd9Sstevel@tonic-gate */
58035b1ab99Sjosephb if (segkp_bitmap && seg == &kvseg &&
58135b1ab99Sjosephb BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
5827c478bd9Sstevel@tonic-gate return (SEGOP_KLUSTER(segkp, addr, delta));
58335b1ab99Sjosephb
5847c478bd9Sstevel@tonic-gate segkmem_badop();
5857c478bd9Sstevel@tonic-gate return (0);
5867c478bd9Sstevel@tonic-gate }
5877c478bd9Sstevel@tonic-gate
5887c478bd9Sstevel@tonic-gate static void
segkmem_xdump_range(void * arg,void * start,size_t size)5897c478bd9Sstevel@tonic-gate segkmem_xdump_range(void *arg, void *start, size_t size)
5907c478bd9Sstevel@tonic-gate {
5917c478bd9Sstevel@tonic-gate struct as *as = arg;
5927c478bd9Sstevel@tonic-gate caddr_t addr = start;
5937c478bd9Sstevel@tonic-gate caddr_t addr_end = addr + size;
5947c478bd9Sstevel@tonic-gate
5957c478bd9Sstevel@tonic-gate while (addr < addr_end) {
5967c478bd9Sstevel@tonic-gate pfn_t pfn = hat_getpfnum(kas.a_hat, addr);
5977c478bd9Sstevel@tonic-gate if (pfn != PFN_INVALID && pfn <= physmax && pf_is_memory(pfn))
5987c478bd9Sstevel@tonic-gate dump_addpage(as, addr, pfn);
5997c478bd9Sstevel@tonic-gate addr += PAGESIZE;
6007c478bd9Sstevel@tonic-gate dump_timeleft = dump_timeout;
6017c478bd9Sstevel@tonic-gate }
6027c478bd9Sstevel@tonic-gate }
6037c478bd9Sstevel@tonic-gate
6047c478bd9Sstevel@tonic-gate static void
segkmem_dump_range(void * arg,void * start,size_t size)6057c478bd9Sstevel@tonic-gate segkmem_dump_range(void *arg, void *start, size_t size)
6067c478bd9Sstevel@tonic-gate {
6077c478bd9Sstevel@tonic-gate caddr_t addr = start;
6087c478bd9Sstevel@tonic-gate caddr_t addr_end = addr + size;
6097c478bd9Sstevel@tonic-gate
6107c478bd9Sstevel@tonic-gate /*
6117c478bd9Sstevel@tonic-gate * If we are about to start dumping the range of addresses we
6127c478bd9Sstevel@tonic-gate * carved out of the kernel heap for the large page heap walk
6137c478bd9Sstevel@tonic-gate * heap_lp_arena to find what segments are actually populated
6147c478bd9Sstevel@tonic-gate */
6157c478bd9Sstevel@tonic-gate if (SEGKMEM_USE_LARGEPAGES &&
6167c478bd9Sstevel@tonic-gate addr == heap_lp_base && addr_end == heap_lp_end &&
6177c478bd9Sstevel@tonic-gate vmem_size(heap_lp_arena, VMEM_ALLOC) < size) {
6187c478bd9Sstevel@tonic-gate vmem_walk(heap_lp_arena, VMEM_ALLOC | VMEM_REENTRANT,
6197c478bd9Sstevel@tonic-gate segkmem_xdump_range, arg);
6207c478bd9Sstevel@tonic-gate } else {
6217c478bd9Sstevel@tonic-gate segkmem_xdump_range(arg, start, size);
6227c478bd9Sstevel@tonic-gate }
6237c478bd9Sstevel@tonic-gate }
6247c478bd9Sstevel@tonic-gate
6257c478bd9Sstevel@tonic-gate static void
segkmem_dump(struct seg * seg)6267c478bd9Sstevel@tonic-gate segkmem_dump(struct seg *seg)
6277c478bd9Sstevel@tonic-gate {
6287c478bd9Sstevel@tonic-gate /*
6297c478bd9Sstevel@tonic-gate * The kernel's heap_arena (represented by kvseg) is a very large
6307c478bd9Sstevel@tonic-gate * VA space, most of which is typically unused. To speed up dumping
6317c478bd9Sstevel@tonic-gate * we use vmem_walk() to quickly find the pieces of heap_arena that
6327c478bd9Sstevel@tonic-gate * are actually in use. We do the same for heap32_arena and
6337c478bd9Sstevel@tonic-gate * heap_core.
6347c478bd9Sstevel@tonic-gate *
6357c478bd9Sstevel@tonic-gate * We specify VMEM_REENTRANT to vmem_walk() because dump_addpage()
6367c478bd9Sstevel@tonic-gate * may ultimately need to allocate memory. Reentrant walks are
6377c478bd9Sstevel@tonic-gate * necessarily imperfect snapshots. The kernel heap continues
6387c478bd9Sstevel@tonic-gate * to change during a live crash dump, for example. For a normal
6397c478bd9Sstevel@tonic-gate * crash dump, however, we know that there won't be any other threads
6407c478bd9Sstevel@tonic-gate * messing with the heap. Therefore, at worst, we may fail to dump
6417c478bd9Sstevel@tonic-gate * the pages that get allocated by the act of dumping; but we will
6427c478bd9Sstevel@tonic-gate * always dump every page that was allocated when the walk began.
6437c478bd9Sstevel@tonic-gate *
6447c478bd9Sstevel@tonic-gate * The other segkmem segments are dense (fully populated), so there's
6457c478bd9Sstevel@tonic-gate * no need to use this technique when dumping them.
6467c478bd9Sstevel@tonic-gate *
6477c478bd9Sstevel@tonic-gate * Note: when adding special dump handling for any new sparsely-
6487c478bd9Sstevel@tonic-gate * populated segments, be sure to add similar handling to the ::kgrep
6497c478bd9Sstevel@tonic-gate * code in mdb.
6507c478bd9Sstevel@tonic-gate */
6517c478bd9Sstevel@tonic-gate if (seg == &kvseg) {
6527c478bd9Sstevel@tonic-gate vmem_walk(heap_arena, VMEM_ALLOC | VMEM_REENTRANT,
6537c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as);
6547c478bd9Sstevel@tonic-gate #ifndef __sparc
6557c478bd9Sstevel@tonic-gate vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
6567c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as);
6577c478bd9Sstevel@tonic-gate #endif
6587c478bd9Sstevel@tonic-gate } else if (seg == &kvseg_core) {
6597c478bd9Sstevel@tonic-gate vmem_walk(heap_core_arena, VMEM_ALLOC | VMEM_REENTRANT,
6607c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as);
6617c478bd9Sstevel@tonic-gate } else if (seg == &kvseg32) {
6627c478bd9Sstevel@tonic-gate vmem_walk(heap32_arena, VMEM_ALLOC | VMEM_REENTRANT,
6637c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as);
6647c478bd9Sstevel@tonic-gate vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
6657c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as);
666ad23a2dbSjohansen /*
667ad23a2dbSjohansen * We don't want to dump pages attached to kzioseg since they
668ad23a2dbSjohansen * contain file data from ZFS. If this page's segment is
669ad23a2dbSjohansen * kzioseg return instead of writing it to the dump device.
67004909c8cSJohn Levon *
67104909c8cSJohn Levon * Same applies to VM memory allocations.
672ad23a2dbSjohansen */
67304909c8cSJohn Levon } else if (seg == &kzioseg) {
674ad23a2dbSjohansen return;
67504909c8cSJohn Levon #if defined(__amd64)
67604909c8cSJohn Levon } else if (seg == &kvmmseg) {
67704909c8cSJohn Levon return;
67804909c8cSJohn Levon #endif
6797c478bd9Sstevel@tonic-gate } else {
6807c478bd9Sstevel@tonic-gate segkmem_dump_range(seg->s_as, seg->s_base, seg->s_size);
6817c478bd9Sstevel@tonic-gate }
6827c478bd9Sstevel@tonic-gate }
6837c478bd9Sstevel@tonic-gate
6847c478bd9Sstevel@tonic-gate /*
6857c478bd9Sstevel@tonic-gate * lock/unlock kmem pages over a given range [addr, addr+len).
6864fc2445aSelowe * Returns a shadow list of pages in ppp. If there are holes
6874fc2445aSelowe * in the range (e.g. some of the kernel mappings do not have
6884fc2445aSelowe * underlying page_ts) returns ENOTSUP so that as_pagelock()
6894fc2445aSelowe * will handle the range via as_fault(F_SOFTLOCK).
6907c478bd9Sstevel@tonic-gate */
6917c478bd9Sstevel@tonic-gate /*ARGSUSED*/
6927c478bd9Sstevel@tonic-gate static int
segkmem_pagelock(struct seg * seg,caddr_t addr,size_t len,page_t *** ppp,enum lock_type type,enum seg_rw rw)6937c478bd9Sstevel@tonic-gate segkmem_pagelock(struct seg *seg, caddr_t addr, size_t len,
6947c478bd9Sstevel@tonic-gate page_t ***ppp, enum lock_type type, enum seg_rw rw)
6957c478bd9Sstevel@tonic-gate {
6967c478bd9Sstevel@tonic-gate page_t **pplist, *pp;
6977c478bd9Sstevel@tonic-gate pgcnt_t npages;
6984fc2445aSelowe spgcnt_t pg;
6997c478bd9Sstevel@tonic-gate size_t nb;
700ad23a2dbSjohansen struct vnode *vp = seg->s_data;
7017c478bd9Sstevel@tonic-gate
7024fc2445aSelowe ASSERT(ppp != NULL);
7034fc2445aSelowe
7047c478bd9Sstevel@tonic-gate /*
7057c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp.
7067c478bd9Sstevel@tonic-gate */
70735b1ab99Sjosephb if (segkp_bitmap && seg == &kvseg &&
70835b1ab99Sjosephb BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
70935b1ab99Sjosephb return (SEGOP_PAGELOCK(segkp, addr, len, ppp, type, rw));
7107c478bd9Sstevel@tonic-gate
7117c478bd9Sstevel@tonic-gate npages = btopr(len);
7127c478bd9Sstevel@tonic-gate nb = sizeof (page_t *) * npages;
7137c478bd9Sstevel@tonic-gate
7147c478bd9Sstevel@tonic-gate if (type == L_PAGEUNLOCK) {
7154fc2445aSelowe pplist = *ppp;
7164fc2445aSelowe ASSERT(pplist != NULL);
7177c478bd9Sstevel@tonic-gate
7184fc2445aSelowe for (pg = 0; pg < npages; pg++) {
7194fc2445aSelowe pp = pplist[pg];
7207c478bd9Sstevel@tonic-gate page_unlock(pp);
7217c478bd9Sstevel@tonic-gate }
7224fc2445aSelowe kmem_free(pplist, nb);
7237c478bd9Sstevel@tonic-gate return (0);
7247c478bd9Sstevel@tonic-gate }
7257c478bd9Sstevel@tonic-gate
7267c478bd9Sstevel@tonic-gate ASSERT(type == L_PAGELOCK);
7277c478bd9Sstevel@tonic-gate
7284fc2445aSelowe pplist = kmem_alloc(nb, KM_NOSLEEP);
7294fc2445aSelowe if (pplist == NULL) {
7304fc2445aSelowe *ppp = NULL;
7314fc2445aSelowe return (ENOTSUP); /* take the slow path */
7324fc2445aSelowe }
7337c478bd9Sstevel@tonic-gate
7344fc2445aSelowe for (pg = 0; pg < npages; pg++) {
735ad23a2dbSjohansen pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_SHARED);
7364fc2445aSelowe if (pp == NULL) {
7374fc2445aSelowe while (--pg >= 0)
7384fc2445aSelowe page_unlock(pplist[pg]);
7394fc2445aSelowe kmem_free(pplist, nb);
7404fc2445aSelowe *ppp = NULL;
7414fc2445aSelowe return (ENOTSUP);
7424fc2445aSelowe }
7434fc2445aSelowe pplist[pg] = pp;
7447c478bd9Sstevel@tonic-gate addr += PAGESIZE;
7457c478bd9Sstevel@tonic-gate }
7464fc2445aSelowe
7474fc2445aSelowe *ppp = pplist;
7487c478bd9Sstevel@tonic-gate return (0);
7497c478bd9Sstevel@tonic-gate }
7507c478bd9Sstevel@tonic-gate
7517c478bd9Sstevel@tonic-gate /*
7527c478bd9Sstevel@tonic-gate * This is a dummy segkmem function overloaded to call segkp
7537c478bd9Sstevel@tonic-gate * when segkp is under the heap.
7547c478bd9Sstevel@tonic-gate */
7557c478bd9Sstevel@tonic-gate /* ARGSUSED */
7567c478bd9Sstevel@tonic-gate static int
segkmem_getmemid(struct seg * seg,caddr_t addr,memid_t * memidp)7577c478bd9Sstevel@tonic-gate segkmem_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
7587c478bd9Sstevel@tonic-gate {
7597c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
7607c478bd9Sstevel@tonic-gate
7617c478bd9Sstevel@tonic-gate if (seg->s_as != &kas)
7627c478bd9Sstevel@tonic-gate segkmem_badop();
7637c478bd9Sstevel@tonic-gate
7647c478bd9Sstevel@tonic-gate /*
7657c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp.
7667c478bd9Sstevel@tonic-gate */
76735b1ab99Sjosephb if (segkp_bitmap && seg == &kvseg &&
76835b1ab99Sjosephb BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
7697c478bd9Sstevel@tonic-gate return (SEGOP_GETMEMID(segkp, addr, memidp));
77035b1ab99Sjosephb
7717c478bd9Sstevel@tonic-gate segkmem_badop();
7727c478bd9Sstevel@tonic-gate return (0);
7737c478bd9Sstevel@tonic-gate }
7747c478bd9Sstevel@tonic-gate
7757c478bd9Sstevel@tonic-gate /*ARGSUSED*/
7767c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t *
segkmem_getpolicy(struct seg * seg,caddr_t addr)7777c478bd9Sstevel@tonic-gate segkmem_getpolicy(struct seg *seg, caddr_t addr)
7787c478bd9Sstevel@tonic-gate {
7797c478bd9Sstevel@tonic-gate return (NULL);
7807c478bd9Sstevel@tonic-gate }
7817c478bd9Sstevel@tonic-gate
7821bd5c35fSelowe /*ARGSUSED*/
7831bd5c35fSelowe static int
segkmem_capable(struct seg * seg,segcapability_t capability)7841bd5c35fSelowe segkmem_capable(struct seg *seg, segcapability_t capability)
7851bd5c35fSelowe {
7861bd5c35fSelowe if (capability == S_CAPABILITY_NOMINFLT)
7871bd5c35fSelowe return (1);
7881bd5c35fSelowe return (0);
7891bd5c35fSelowe }
7907c478bd9Sstevel@tonic-gate
7912428aad8SPatrick Mooney struct seg_ops segkmem_ops = {
7927c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* dup */
7937c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* unmap */
7947c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(void), /* free */
7957c478bd9Sstevel@tonic-gate segkmem_fault,
7967c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(faultcode_t), /* faulta */
7977c478bd9Sstevel@tonic-gate segkmem_setprot,
7987c478bd9Sstevel@tonic-gate segkmem_checkprot,
7997c478bd9Sstevel@tonic-gate segkmem_kluster,
8007c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(size_t), /* swapout */
8017c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* sync */
8027c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(size_t), /* incore */
8037c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* lockop */
8047c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* getprot */
8057c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(u_offset_t), /* getoffset */
8067c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* gettype */
8077c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* getvp */
8087c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* advise */
8097c478bd9Sstevel@tonic-gate segkmem_dump,
8107c478bd9Sstevel@tonic-gate segkmem_pagelock,
8117c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* setpgsz */
8127c478bd9Sstevel@tonic-gate segkmem_getmemid,
8137c478bd9Sstevel@tonic-gate segkmem_getpolicy, /* getpolicy */
8141bd5c35fSelowe segkmem_capable, /* capable */
8159d12795fSRobert Mustacchi seg_inherit_notsup /* inherit */
8167c478bd9Sstevel@tonic-gate };
8177c478bd9Sstevel@tonic-gate
8187c478bd9Sstevel@tonic-gate int
segkmem_create(struct seg * seg)8197c478bd9Sstevel@tonic-gate segkmem_create(struct seg *seg)
8207c478bd9Sstevel@tonic-gate {
8217c478bd9Sstevel@tonic-gate ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
8227c478bd9Sstevel@tonic-gate seg->s_ops = &segkmem_ops;
82304909c8cSJohn Levon if (seg == &kzioseg)
82404909c8cSJohn Levon seg->s_data = &kvps[KV_ZVP];
82504909c8cSJohn Levon #if defined(__amd64)
82604909c8cSJohn Levon else if (seg == &kvmmseg)
82704909c8cSJohn Levon seg->s_data = &kvps[KV_VVP];
82804909c8cSJohn Levon #endif
82904909c8cSJohn Levon else
83004909c8cSJohn Levon seg->s_data = &kvps[KV_KVP];
8317c478bd9Sstevel@tonic-gate kas.a_size += seg->s_size;
8327c478bd9Sstevel@tonic-gate return (0);
8337c478bd9Sstevel@tonic-gate }
8347c478bd9Sstevel@tonic-gate
8357c478bd9Sstevel@tonic-gate /*ARGSUSED*/
8367c478bd9Sstevel@tonic-gate page_t *
segkmem_page_create(void * addr,size_t size,int vmflag,void * arg)8377c478bd9Sstevel@tonic-gate segkmem_page_create(void *addr, size_t size, int vmflag, void *arg)
8387c478bd9Sstevel@tonic-gate {
839e57e118bSJohn Levon struct seg kseg = { 0 };
840e57e118bSJohn Levon int pgflags = PG_EXCL;
841ad23a2dbSjohansen struct vnode *vp = arg;
842ad23a2dbSjohansen
843ad23a2dbSjohansen if (vp == NULL)
844ad23a2dbSjohansen vp = &kvp;
8457c478bd9Sstevel@tonic-gate
8467c478bd9Sstevel@tonic-gate kseg.s_as = &kas;
8477c478bd9Sstevel@tonic-gate
8487c478bd9Sstevel@tonic-gate if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
8497c478bd9Sstevel@tonic-gate pgflags |= PG_NORELOC;
8507c478bd9Sstevel@tonic-gate if ((vmflag & VM_NOSLEEP) == 0)
8517c478bd9Sstevel@tonic-gate pgflags |= PG_WAIT;
8527c478bd9Sstevel@tonic-gate if (vmflag & VM_PANIC)
8537c478bd9Sstevel@tonic-gate pgflags |= PG_PANIC;
8547c478bd9Sstevel@tonic-gate if (vmflag & VM_PUSHPAGE)
8557c478bd9Sstevel@tonic-gate pgflags |= PG_PUSHPAGE;
85623a80de1SStan Studzinski if (vmflag & VM_NORMALPRI) {
85723a80de1SStan Studzinski ASSERT(vmflag & VM_NOSLEEP);
85823a80de1SStan Studzinski pgflags |= PG_NORMALPRI;
85923a80de1SStan Studzinski }
8607c478bd9Sstevel@tonic-gate
861ad23a2dbSjohansen return (page_create_va(vp, (u_offset_t)(uintptr_t)addr, size,
8627c478bd9Sstevel@tonic-gate pgflags, &kseg, addr));
8637c478bd9Sstevel@tonic-gate }
8647c478bd9Sstevel@tonic-gate
8657c478bd9Sstevel@tonic-gate /*
8667c478bd9Sstevel@tonic-gate * Allocate pages to back the virtual address range [addr, addr + size).
8677c478bd9Sstevel@tonic-gate * If addr is NULL, allocate the virtual address space as well.
8687c478bd9Sstevel@tonic-gate */
8697c478bd9Sstevel@tonic-gate void *
segkmem_xalloc(vmem_t * vmp,void * inaddr,size_t size,int vmflag,uint_t attr,page_t * (* page_create_func)(void *,size_t,int,void *),void * pcarg)8707c478bd9Sstevel@tonic-gate segkmem_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag, uint_t attr,
8717c478bd9Sstevel@tonic-gate page_t *(*page_create_func)(void *, size_t, int, void *), void *pcarg)
8727c478bd9Sstevel@tonic-gate {
8737c478bd9Sstevel@tonic-gate page_t *ppl;
8747c478bd9Sstevel@tonic-gate caddr_t addr = inaddr;
8757c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size);
8767c478bd9Sstevel@tonic-gate int allocflag;
8777c478bd9Sstevel@tonic-gate
8787c478bd9Sstevel@tonic-gate if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
8797c478bd9Sstevel@tonic-gate return (NULL);
8807c478bd9Sstevel@tonic-gate
8817c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
8827c478bd9Sstevel@tonic-gate
8837c478bd9Sstevel@tonic-gate if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
8847c478bd9Sstevel@tonic-gate if (inaddr == NULL)
8857c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size);
8867c478bd9Sstevel@tonic-gate return (NULL);
8877c478bd9Sstevel@tonic-gate }
8887c478bd9Sstevel@tonic-gate
8897c478bd9Sstevel@tonic-gate ppl = page_create_func(addr, size, vmflag, pcarg);
8907c478bd9Sstevel@tonic-gate if (ppl == NULL) {
8917c478bd9Sstevel@tonic-gate if (inaddr == NULL)
8927c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size);
8937c478bd9Sstevel@tonic-gate page_unresv(npages);
8947c478bd9Sstevel@tonic-gate return (NULL);
8957c478bd9Sstevel@tonic-gate }
8967c478bd9Sstevel@tonic-gate
8977c478bd9Sstevel@tonic-gate /*
8987c478bd9Sstevel@tonic-gate * Under certain conditions, we need to let the HAT layer know
8997c478bd9Sstevel@tonic-gate * that it cannot safely allocate memory. Allocations from
9007c478bd9Sstevel@tonic-gate * the hat_memload vmem arena always need this, to prevent
9017c478bd9Sstevel@tonic-gate * infinite recursion.
9027c478bd9Sstevel@tonic-gate *
9037c478bd9Sstevel@tonic-gate * In addition, the x86 hat cannot safely do memory
9047c478bd9Sstevel@tonic-gate * allocations while in vmem_populate(), because there
9057c478bd9Sstevel@tonic-gate * is no simple bound on its usage.
9067c478bd9Sstevel@tonic-gate */
9077c478bd9Sstevel@tonic-gate if (vmflag & VM_MEMLOAD)
9087c478bd9Sstevel@tonic-gate allocflag = HAT_NO_KALLOC;
9097c478bd9Sstevel@tonic-gate #if defined(__x86)
9107c478bd9Sstevel@tonic-gate else if (vmem_is_populator())
9117c478bd9Sstevel@tonic-gate allocflag = HAT_NO_KALLOC;
9127c478bd9Sstevel@tonic-gate #endif
9137c478bd9Sstevel@tonic-gate else
9147c478bd9Sstevel@tonic-gate allocflag = 0;
9157c478bd9Sstevel@tonic-gate
9167c478bd9Sstevel@tonic-gate while (ppl != NULL) {
9177c478bd9Sstevel@tonic-gate page_t *pp = ppl;
9187c478bd9Sstevel@tonic-gate page_sub(&ppl, pp);
9197c478bd9Sstevel@tonic-gate ASSERT(page_iolock_assert(pp));
9207c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp));
9217c478bd9Sstevel@tonic-gate page_io_unlock(pp);
9227c478bd9Sstevel@tonic-gate hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset, pp,
9237c478bd9Sstevel@tonic-gate (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
9247c478bd9Sstevel@tonic-gate HAT_LOAD_LOCK | allocflag);
9257c478bd9Sstevel@tonic-gate pp->p_lckcnt = 1;
9267c478bd9Sstevel@tonic-gate #if defined(__x86)
9277c478bd9Sstevel@tonic-gate page_downgrade(pp);
9287c478bd9Sstevel@tonic-gate #else
9297c478bd9Sstevel@tonic-gate if (vmflag & SEGKMEM_SHARELOCKED)
9307c478bd9Sstevel@tonic-gate page_downgrade(pp);
9317c478bd9Sstevel@tonic-gate else
9327c478bd9Sstevel@tonic-gate page_unlock(pp);
9337c478bd9Sstevel@tonic-gate #endif
9347c478bd9Sstevel@tonic-gate }
9357c478bd9Sstevel@tonic-gate
9367c478bd9Sstevel@tonic-gate return (addr);
9377c478bd9Sstevel@tonic-gate }
9387c478bd9Sstevel@tonic-gate
939ad23a2dbSjohansen static void *
segkmem_alloc_vn(vmem_t * vmp,size_t size,int vmflag,struct vnode * vp)940ad23a2dbSjohansen segkmem_alloc_vn(vmem_t *vmp, size_t size, int vmflag, struct vnode *vp)
9417c478bd9Sstevel@tonic-gate {
9427c478bd9Sstevel@tonic-gate void *addr;
9437c478bd9Sstevel@tonic-gate segkmem_gc_list_t *gcp, **prev_gcpp;
9447c478bd9Sstevel@tonic-gate
945ad23a2dbSjohansen ASSERT(vp != NULL);
946ad23a2dbSjohansen
9477c478bd9Sstevel@tonic-gate if (kvseg.s_base == NULL) {
9487c478bd9Sstevel@tonic-gate #ifndef __sparc
9497c478bd9Sstevel@tonic-gate if (bootops->bsys_alloc == NULL)
9507c478bd9Sstevel@tonic-gate halt("Memory allocation between bop_alloc() and "
9517c478bd9Sstevel@tonic-gate "kmem_alloc().\n");
9527c478bd9Sstevel@tonic-gate #endif
9537c478bd9Sstevel@tonic-gate
9547c478bd9Sstevel@tonic-gate /*
9557c478bd9Sstevel@tonic-gate * There's not a lot of memory to go around during boot,
9567c478bd9Sstevel@tonic-gate * so recycle it if we can.
9577c478bd9Sstevel@tonic-gate */
9587c478bd9Sstevel@tonic-gate for (prev_gcpp = &segkmem_gc_list; (gcp = *prev_gcpp) != NULL;
9597c478bd9Sstevel@tonic-gate prev_gcpp = &gcp->gc_next) {
9607c478bd9Sstevel@tonic-gate if (gcp->gc_arena == vmp && gcp->gc_size == size) {
9617c478bd9Sstevel@tonic-gate *prev_gcpp = gcp->gc_next;
9627c478bd9Sstevel@tonic-gate return (gcp);
9637c478bd9Sstevel@tonic-gate }
9647c478bd9Sstevel@tonic-gate }
9657c478bd9Sstevel@tonic-gate
9667c478bd9Sstevel@tonic-gate addr = vmem_alloc(vmp, size, vmflag | VM_PANIC);
9677c478bd9Sstevel@tonic-gate if (boot_alloc(addr, size, BO_NO_ALIGN) != addr)
9687c478bd9Sstevel@tonic-gate panic("segkmem_alloc: boot_alloc failed");
9697c478bd9Sstevel@tonic-gate return (addr);
9707c478bd9Sstevel@tonic-gate }
9717c478bd9Sstevel@tonic-gate return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
972ad23a2dbSjohansen segkmem_page_create, vp));
973ad23a2dbSjohansen }
974ad23a2dbSjohansen
975ad23a2dbSjohansen void *
segkmem_alloc(vmem_t * vmp,size_t size,int vmflag)976ad23a2dbSjohansen segkmem_alloc(vmem_t *vmp, size_t size, int vmflag)
977ad23a2dbSjohansen {
978ad23a2dbSjohansen return (segkmem_alloc_vn(vmp, size, vmflag, &kvp));
979ad23a2dbSjohansen }
980ad23a2dbSjohansen
98104909c8cSJohn Levon static void *
segkmem_zio_alloc(vmem_t * vmp,size_t size,int vmflag)982ad23a2dbSjohansen segkmem_zio_alloc(vmem_t *vmp, size_t size, int vmflag)
983ad23a2dbSjohansen {
98404909c8cSJohn Levon return (segkmem_alloc_vn(vmp, size, vmflag, &kvps[KV_ZVP]));
9857c478bd9Sstevel@tonic-gate }
9867c478bd9Sstevel@tonic-gate
9877c478bd9Sstevel@tonic-gate /*
9887c478bd9Sstevel@tonic-gate * Any changes to this routine must also be carried over to
9897c478bd9Sstevel@tonic-gate * devmap_free_pages() in the seg_dev driver. This is because
9907c478bd9Sstevel@tonic-gate * we currently don't have a special kernel segment for non-paged
9917c478bd9Sstevel@tonic-gate * kernel memory that is exported by drivers to user space.
9927c478bd9Sstevel@tonic-gate */
99304909c8cSJohn Levon void
segkmem_xfree(vmem_t * vmp,void * inaddr,size_t size,struct vnode * vp,void (* func)(page_t *))99404909c8cSJohn Levon segkmem_xfree(vmem_t *vmp, void *inaddr, size_t size, struct vnode *vp,
995843e1988Sjohnlev void (*func)(page_t *))
9967c478bd9Sstevel@tonic-gate {
9977c478bd9Sstevel@tonic-gate page_t *pp;
9987c478bd9Sstevel@tonic-gate caddr_t addr = inaddr;
9997c478bd9Sstevel@tonic-gate caddr_t eaddr;
10007c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size);
10017c478bd9Sstevel@tonic-gate
10027c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
1003ad23a2dbSjohansen ASSERT(vp != NULL);
10047c478bd9Sstevel@tonic-gate
10057c478bd9Sstevel@tonic-gate if (kvseg.s_base == NULL) {
10067c478bd9Sstevel@tonic-gate segkmem_gc_list_t *gc = inaddr;
10077c478bd9Sstevel@tonic-gate gc->gc_arena = vmp;
10087c478bd9Sstevel@tonic-gate gc->gc_size = size;
10097c478bd9Sstevel@tonic-gate gc->gc_next = segkmem_gc_list;
10107c478bd9Sstevel@tonic-gate segkmem_gc_list = gc;
10117c478bd9Sstevel@tonic-gate return;
10127c478bd9Sstevel@tonic-gate }
10137c478bd9Sstevel@tonic-gate
10147c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
10157c478bd9Sstevel@tonic-gate
10167c478bd9Sstevel@tonic-gate for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
10177c478bd9Sstevel@tonic-gate #if defined(__x86)
1018ad23a2dbSjohansen pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
10197c478bd9Sstevel@tonic-gate if (pp == NULL)
10207c478bd9Sstevel@tonic-gate panic("segkmem_free: page not found");
10217c478bd9Sstevel@tonic-gate if (!page_tryupgrade(pp)) {
10227c478bd9Sstevel@tonic-gate /*
10237c478bd9Sstevel@tonic-gate * Some other thread has a sharelock. Wait for
10247c478bd9Sstevel@tonic-gate * it to drop the lock so we can free this page.
10257c478bd9Sstevel@tonic-gate */
10267c478bd9Sstevel@tonic-gate page_unlock(pp);
1027ad23a2dbSjohansen pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
10287c478bd9Sstevel@tonic-gate SE_EXCL);
10297c478bd9Sstevel@tonic-gate }
10307c478bd9Sstevel@tonic-gate #else
1031ad23a2dbSjohansen pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
10327c478bd9Sstevel@tonic-gate #endif
10337c478bd9Sstevel@tonic-gate if (pp == NULL)
10347c478bd9Sstevel@tonic-gate panic("segkmem_free: page not found");
10357c478bd9Sstevel@tonic-gate /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
10367c478bd9Sstevel@tonic-gate pp->p_lckcnt = 0;
1037843e1988Sjohnlev if (func)
1038843e1988Sjohnlev func(pp);
1039843e1988Sjohnlev else
10407c478bd9Sstevel@tonic-gate page_destroy(pp, 0);
10417c478bd9Sstevel@tonic-gate }
1042843e1988Sjohnlev if (func == NULL)
10437c478bd9Sstevel@tonic-gate page_unresv(npages);
10447c478bd9Sstevel@tonic-gate
10457c478bd9Sstevel@tonic-gate if (vmp != NULL)
10467c478bd9Sstevel@tonic-gate vmem_free(vmp, inaddr, size);
1047ad23a2dbSjohansen
1048ad23a2dbSjohansen }
1049ad23a2dbSjohansen
1050ad23a2dbSjohansen void
segkmem_free(vmem_t * vmp,void * inaddr,size_t size)1051ad23a2dbSjohansen segkmem_free(vmem_t *vmp, void *inaddr, size_t size)
1052ad23a2dbSjohansen {
105304909c8cSJohn Levon segkmem_xfree(vmp, inaddr, size, &kvp, NULL);
1054ad23a2dbSjohansen }
1055ad23a2dbSjohansen
105604909c8cSJohn Levon static void
segkmem_zio_free(vmem_t * vmp,void * inaddr,size_t size)1057ad23a2dbSjohansen segkmem_zio_free(vmem_t *vmp, void *inaddr, size_t size)
1058ad23a2dbSjohansen {
105904909c8cSJohn Levon segkmem_xfree(vmp, inaddr, size, &kvps[KV_ZVP], NULL);
10607c478bd9Sstevel@tonic-gate }
10617c478bd9Sstevel@tonic-gate
10627c478bd9Sstevel@tonic-gate void
segkmem_gc(void)10637c478bd9Sstevel@tonic-gate segkmem_gc(void)
10647c478bd9Sstevel@tonic-gate {
10657c478bd9Sstevel@tonic-gate ASSERT(kvseg.s_base != NULL);
10667c478bd9Sstevel@tonic-gate while (segkmem_gc_list != NULL) {
10677c478bd9Sstevel@tonic-gate segkmem_gc_list_t *gc = segkmem_gc_list;
10687c478bd9Sstevel@tonic-gate segkmem_gc_list = gc->gc_next;
10697c478bd9Sstevel@tonic-gate segkmem_free(gc->gc_arena, gc, gc->gc_size);
10707c478bd9Sstevel@tonic-gate }
10717c478bd9Sstevel@tonic-gate }
10727c478bd9Sstevel@tonic-gate
10737c478bd9Sstevel@tonic-gate /*
10747c478bd9Sstevel@tonic-gate * Legacy entry points from here to end of file.
10757c478bd9Sstevel@tonic-gate */
10767c478bd9Sstevel@tonic-gate void
segkmem_mapin(struct seg * seg,void * addr,size_t size,uint_t vprot,pfn_t pfn,uint_t flags)10777c478bd9Sstevel@tonic-gate segkmem_mapin(struct seg *seg, void *addr, size_t size, uint_t vprot,
10787c478bd9Sstevel@tonic-gate pfn_t pfn, uint_t flags)
10797c478bd9Sstevel@tonic-gate {
10807c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
10817c478bd9Sstevel@tonic-gate hat_devload(seg->s_as->a_hat, addr, size, pfn, vprot,
10827c478bd9Sstevel@tonic-gate flags | HAT_LOAD_LOCK);
10837c478bd9Sstevel@tonic-gate }
10847c478bd9Sstevel@tonic-gate
10857c478bd9Sstevel@tonic-gate void
segkmem_mapout(struct seg * seg,void * addr,size_t size)10867c478bd9Sstevel@tonic-gate segkmem_mapout(struct seg *seg, void *addr, size_t size)
10877c478bd9Sstevel@tonic-gate {
10887c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
10897c478bd9Sstevel@tonic-gate }
10907c478bd9Sstevel@tonic-gate
10917c478bd9Sstevel@tonic-gate void *
kmem_getpages(pgcnt_t npages,int kmflag)10927c478bd9Sstevel@tonic-gate kmem_getpages(pgcnt_t npages, int kmflag)
10937c478bd9Sstevel@tonic-gate {
10947c478bd9Sstevel@tonic-gate return (kmem_alloc(ptob(npages), kmflag));
10957c478bd9Sstevel@tonic-gate }
10967c478bd9Sstevel@tonic-gate
10977c478bd9Sstevel@tonic-gate void
kmem_freepages(void * addr,pgcnt_t npages)10987c478bd9Sstevel@tonic-gate kmem_freepages(void *addr, pgcnt_t npages)
10997c478bd9Sstevel@tonic-gate {
11007c478bd9Sstevel@tonic-gate kmem_free(addr, ptob(npages));
11017c478bd9Sstevel@tonic-gate }
11027c478bd9Sstevel@tonic-gate
1103*ae5a8bedSAndy Fiddaman #ifdef __sparc
11047c478bd9Sstevel@tonic-gate /*
11057c478bd9Sstevel@tonic-gate * segkmem_page_create_large() allocates a large page to be used for the kmem
11067c478bd9Sstevel@tonic-gate * caches. If kpr is enabled we ask for a relocatable page unless requested
11077c478bd9Sstevel@tonic-gate * otherwise. If kpr is disabled we have to ask for a non-reloc page
11087c478bd9Sstevel@tonic-gate */
11097c478bd9Sstevel@tonic-gate static page_t *
segkmem_page_create_large(void * addr,size_t size,int vmflag,void * arg)11107c478bd9Sstevel@tonic-gate segkmem_page_create_large(void *addr, size_t size, int vmflag, void *arg)
11117c478bd9Sstevel@tonic-gate {
11127c478bd9Sstevel@tonic-gate int pgflags;
11137c478bd9Sstevel@tonic-gate
11147c478bd9Sstevel@tonic-gate pgflags = PG_EXCL;
11157c478bd9Sstevel@tonic-gate
11167c478bd9Sstevel@tonic-gate if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
11177c478bd9Sstevel@tonic-gate pgflags |= PG_NORELOC;
11187c478bd9Sstevel@tonic-gate if (!(vmflag & VM_NOSLEEP))
11197c478bd9Sstevel@tonic-gate pgflags |= PG_WAIT;
11207c478bd9Sstevel@tonic-gate if (vmflag & VM_PUSHPAGE)
11217c478bd9Sstevel@tonic-gate pgflags |= PG_PUSHPAGE;
112223a80de1SStan Studzinski if (vmflag & VM_NORMALPRI)
112323a80de1SStan Studzinski pgflags |= PG_NORMALPRI;
11247c478bd9Sstevel@tonic-gate
11257c478bd9Sstevel@tonic-gate return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size,
11267c478bd9Sstevel@tonic-gate pgflags, &kvseg, addr, arg));
11277c478bd9Sstevel@tonic-gate }
11287c478bd9Sstevel@tonic-gate
11297c478bd9Sstevel@tonic-gate /*
11307c478bd9Sstevel@tonic-gate * Allocate a large page to back the virtual address range
11317c478bd9Sstevel@tonic-gate * [addr, addr + size). If addr is NULL, allocate the virtual address
11327c478bd9Sstevel@tonic-gate * space as well.
11337c478bd9Sstevel@tonic-gate */
11347c478bd9Sstevel@tonic-gate static void *
segkmem_xalloc_lp(vmem_t * vmp,void * inaddr,size_t size,int vmflag,uint_t attr,page_t * (* page_create_func)(void *,size_t,int,void *),void * pcarg)11357c478bd9Sstevel@tonic-gate segkmem_xalloc_lp(vmem_t *vmp, void *inaddr, size_t size, int vmflag,
11367c478bd9Sstevel@tonic-gate uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *),
11377c478bd9Sstevel@tonic-gate void *pcarg)
11387c478bd9Sstevel@tonic-gate {
11397c478bd9Sstevel@tonic-gate caddr_t addr = inaddr, pa;
11407c478bd9Sstevel@tonic-gate size_t lpsize = segkmem_lpsize;
11417c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size);
11427c478bd9Sstevel@tonic-gate pgcnt_t nbpages = btop(lpsize);
11437c478bd9Sstevel@tonic-gate pgcnt_t nlpages = size >> segkmem_lpshift;
11447c478bd9Sstevel@tonic-gate size_t ppasize = nbpages * sizeof (page_t *);
11457c478bd9Sstevel@tonic-gate page_t *pp, *rootpp, **ppa, *pplist = NULL;
11467c478bd9Sstevel@tonic-gate int i;
11477c478bd9Sstevel@tonic-gate
1148423908e1Sdp78419 vmflag |= VM_NOSLEEP;
1149423908e1Sdp78419
11507c478bd9Sstevel@tonic-gate if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
11517c478bd9Sstevel@tonic-gate return (NULL);
11527c478bd9Sstevel@tonic-gate }
11537c478bd9Sstevel@tonic-gate
11547c478bd9Sstevel@tonic-gate /*
11557c478bd9Sstevel@tonic-gate * allocate an array we need for hat_memload_array.
11567c478bd9Sstevel@tonic-gate * we use a separate arena to avoid recursion.
11577c478bd9Sstevel@tonic-gate * we will not need this array when hat_memload_array learns pp++
11587c478bd9Sstevel@tonic-gate */
11597c478bd9Sstevel@tonic-gate if ((ppa = vmem_alloc(segkmem_ppa_arena, ppasize, vmflag)) == NULL) {
11607c478bd9Sstevel@tonic-gate goto fail_array_alloc;
11617c478bd9Sstevel@tonic-gate }
11627c478bd9Sstevel@tonic-gate
11637c478bd9Sstevel@tonic-gate if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
11647c478bd9Sstevel@tonic-gate goto fail_vmem_alloc;
11657c478bd9Sstevel@tonic-gate
11667c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & (lpsize - 1)) == 0);
11677c478bd9Sstevel@tonic-gate
11687c478bd9Sstevel@tonic-gate /* create all the pages */
11697c478bd9Sstevel@tonic-gate for (pa = addr, i = 0; i < nlpages; i++, pa += lpsize) {
11707c478bd9Sstevel@tonic-gate if ((pp = page_create_func(pa, lpsize, vmflag, pcarg)) == NULL)
11717c478bd9Sstevel@tonic-gate goto fail_page_create;
11727c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &pp);
11737c478bd9Sstevel@tonic-gate }
11747c478bd9Sstevel@tonic-gate
11757c478bd9Sstevel@tonic-gate /* at this point we have all the resource to complete the request */
11767c478bd9Sstevel@tonic-gate while ((rootpp = pplist) != NULL) {
11777c478bd9Sstevel@tonic-gate for (i = 0; i < nbpages; i++) {
11787c478bd9Sstevel@tonic-gate ASSERT(pplist != NULL);
11797c478bd9Sstevel@tonic-gate pp = pplist;
11807c478bd9Sstevel@tonic-gate page_sub(&pplist, pp);
11817c478bd9Sstevel@tonic-gate ASSERT(page_iolock_assert(pp));
11827c478bd9Sstevel@tonic-gate page_io_unlock(pp);
11837c478bd9Sstevel@tonic-gate ppa[i] = pp;
11847c478bd9Sstevel@tonic-gate }
11857c478bd9Sstevel@tonic-gate /*
11867c478bd9Sstevel@tonic-gate * Load the locked entry. It's OK to preload the entry into the
11877c478bd9Sstevel@tonic-gate * TSB since we now support large mappings in the kernel TSB.
11887c478bd9Sstevel@tonic-gate */
11897c478bd9Sstevel@tonic-gate hat_memload_array(kas.a_hat,
11907c478bd9Sstevel@tonic-gate (caddr_t)(uintptr_t)rootpp->p_offset, lpsize,
11917c478bd9Sstevel@tonic-gate ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
11927c478bd9Sstevel@tonic-gate HAT_LOAD_LOCK);
11937c478bd9Sstevel@tonic-gate
11947c478bd9Sstevel@tonic-gate for (--i; i >= 0; --i) {
11957c478bd9Sstevel@tonic-gate ppa[i]->p_lckcnt = 1;
11967c478bd9Sstevel@tonic-gate page_unlock(ppa[i]);
11977c478bd9Sstevel@tonic-gate }
11987c478bd9Sstevel@tonic-gate }
11997c478bd9Sstevel@tonic-gate
12007c478bd9Sstevel@tonic-gate vmem_free(segkmem_ppa_arena, ppa, ppasize);
12017c478bd9Sstevel@tonic-gate return (addr);
12027c478bd9Sstevel@tonic-gate
12037c478bd9Sstevel@tonic-gate fail_page_create:
12047c478bd9Sstevel@tonic-gate while ((rootpp = pplist) != NULL) {
12057c478bd9Sstevel@tonic-gate for (i = 0, pp = pplist; i < nbpages; i++, pp = pplist) {
12067c478bd9Sstevel@tonic-gate ASSERT(pp != NULL);
12077c478bd9Sstevel@tonic-gate page_sub(&pplist, pp);
12087c478bd9Sstevel@tonic-gate ASSERT(page_iolock_assert(pp));
12097c478bd9Sstevel@tonic-gate page_io_unlock(pp);
12107c478bd9Sstevel@tonic-gate }
12117c478bd9Sstevel@tonic-gate page_destroy_pages(rootpp);
12127c478bd9Sstevel@tonic-gate }
12137c478bd9Sstevel@tonic-gate
12147c478bd9Sstevel@tonic-gate if (inaddr == NULL)
12157c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size);
12167c478bd9Sstevel@tonic-gate
12177c478bd9Sstevel@tonic-gate fail_vmem_alloc:
12187c478bd9Sstevel@tonic-gate vmem_free(segkmem_ppa_arena, ppa, ppasize);
12197c478bd9Sstevel@tonic-gate
12207c478bd9Sstevel@tonic-gate fail_array_alloc:
12217c478bd9Sstevel@tonic-gate page_unresv(npages);
12227c478bd9Sstevel@tonic-gate
12237c478bd9Sstevel@tonic-gate return (NULL);
12247c478bd9Sstevel@tonic-gate }
12257c478bd9Sstevel@tonic-gate
12267c478bd9Sstevel@tonic-gate static void
segkmem_free_one_lp(caddr_t addr,size_t size)12277c478bd9Sstevel@tonic-gate segkmem_free_one_lp(caddr_t addr, size_t size)
12287c478bd9Sstevel@tonic-gate {
12297c478bd9Sstevel@tonic-gate page_t *pp, *rootpp = NULL;
12307c478bd9Sstevel@tonic-gate pgcnt_t pgs_left = btopr(size);
12317c478bd9Sstevel@tonic-gate
12327c478bd9Sstevel@tonic-gate ASSERT(size == segkmem_lpsize);
12337c478bd9Sstevel@tonic-gate
12347c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
12357c478bd9Sstevel@tonic-gate
12367c478bd9Sstevel@tonic-gate for (; pgs_left > 0; addr += PAGESIZE, pgs_left--) {
12377c478bd9Sstevel@tonic-gate pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
12387c478bd9Sstevel@tonic-gate if (pp == NULL)
12397c478bd9Sstevel@tonic-gate panic("segkmem_free_one_lp: page not found");
12407c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp));
12417c478bd9Sstevel@tonic-gate pp->p_lckcnt = 0;
12427c478bd9Sstevel@tonic-gate if (rootpp == NULL)
12437c478bd9Sstevel@tonic-gate rootpp = pp;
12447c478bd9Sstevel@tonic-gate }
12457c478bd9Sstevel@tonic-gate ASSERT(rootpp != NULL);
12467c478bd9Sstevel@tonic-gate page_destroy_pages(rootpp);
12477c478bd9Sstevel@tonic-gate
12487c478bd9Sstevel@tonic-gate /* page_unresv() is done by the caller */
12497c478bd9Sstevel@tonic-gate }
1250*ae5a8bedSAndy Fiddaman #endif /* __sparc */
12517c478bd9Sstevel@tonic-gate
12527c478bd9Sstevel@tonic-gate /*
12537c478bd9Sstevel@tonic-gate * This function is called to import new spans into the vmem arenas like
12547c478bd9Sstevel@tonic-gate * kmem_default_arena and kmem_oversize_arena. It first tries to import
12557c478bd9Sstevel@tonic-gate * spans from large page arena - kmem_lp_arena. In order to do this it might
12567c478bd9Sstevel@tonic-gate * have to "upgrade the requested size" to kmem_lp_arena quantum. If
12577c478bd9Sstevel@tonic-gate * it was not able to satisfy the upgraded request it then calls regular
12587c478bd9Sstevel@tonic-gate * segkmem_alloc() that satisfies the request by importing from "*vmp" arena
12597c478bd9Sstevel@tonic-gate */
1260aaa10e67Sha137994 /*ARGSUSED*/
12617c478bd9Sstevel@tonic-gate void *
segkmem_alloc_lp(vmem_t * vmp,size_t * sizep,size_t align,int vmflag)1262aaa10e67Sha137994 segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, size_t align, int vmflag)
12637c478bd9Sstevel@tonic-gate {
12647c478bd9Sstevel@tonic-gate size_t size;
12657c478bd9Sstevel@tonic-gate kthread_t *t = curthread;
12667c478bd9Sstevel@tonic-gate segkmem_lpcb_t *lpcb = &segkmem_lpcb;
12677c478bd9Sstevel@tonic-gate
12687c478bd9Sstevel@tonic-gate ASSERT(sizep != NULL);
12697c478bd9Sstevel@tonic-gate
12707c478bd9Sstevel@tonic-gate size = *sizep;
12717c478bd9Sstevel@tonic-gate
12727c478bd9Sstevel@tonic-gate if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) &&
12737c478bd9Sstevel@tonic-gate !(vmflag & SEGKMEM_SHARELOCKED)) {
12747c478bd9Sstevel@tonic-gate
12757c478bd9Sstevel@tonic-gate size_t kmemlp_qnt = segkmem_kmemlp_quantum;
12767c478bd9Sstevel@tonic-gate size_t asize = P2ROUNDUP(size, kmemlp_qnt);
12777c478bd9Sstevel@tonic-gate void *addr = NULL;
12787c478bd9Sstevel@tonic-gate ulong_t *lpthrtp = &lpcb->lp_throttle;
12797c478bd9Sstevel@tonic-gate ulong_t lpthrt = *lpthrtp;
12807c478bd9Sstevel@tonic-gate int dowakeup = 0;
12817c478bd9Sstevel@tonic-gate int doalloc = 1;
12827c478bd9Sstevel@tonic-gate
12837c478bd9Sstevel@tonic-gate ASSERT(kmem_lp_arena != NULL);
12847c478bd9Sstevel@tonic-gate ASSERT(asize >= size);
12857c478bd9Sstevel@tonic-gate
12867c478bd9Sstevel@tonic-gate if (lpthrt != 0) {
12877c478bd9Sstevel@tonic-gate /* try to update the throttle value */
12881a5e258fSJosef 'Jeff' Sipek lpthrt = atomic_inc_ulong_nv(lpthrtp);
12897c478bd9Sstevel@tonic-gate if (lpthrt >= segkmem_lpthrottle_max) {
12907c478bd9Sstevel@tonic-gate lpthrt = atomic_cas_ulong(lpthrtp, lpthrt,
12917c478bd9Sstevel@tonic-gate segkmem_lpthrottle_max / 4);
12927c478bd9Sstevel@tonic-gate }
12937c478bd9Sstevel@tonic-gate
12947c478bd9Sstevel@tonic-gate /*
12957c478bd9Sstevel@tonic-gate * when we get above throttle start do an exponential
12967c478bd9Sstevel@tonic-gate * backoff at trying large pages and reaping
12977c478bd9Sstevel@tonic-gate */
12987c478bd9Sstevel@tonic-gate if (lpthrt > segkmem_lpthrottle_start &&
1299de710d24SJosef 'Jeff' Sipek !ISP2(lpthrt)) {
1300501f3284Seg155566 lpcb->allocs_throttled++;
13017c478bd9Sstevel@tonic-gate lpthrt--;
1302de710d24SJosef 'Jeff' Sipek if (ISP2(lpthrt))
13037c478bd9Sstevel@tonic-gate kmem_reap();
13047c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, vmflag));
13057c478bd9Sstevel@tonic-gate }
13067c478bd9Sstevel@tonic-gate }
13077c478bd9Sstevel@tonic-gate
13087c478bd9Sstevel@tonic-gate if (!(vmflag & VM_NOSLEEP) &&
13097c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum >= (8 * kmemlp_qnt) &&
13107c478bd9Sstevel@tonic-gate vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt &&
13117c478bd9Sstevel@tonic-gate asize < (segkmem_heaplp_quantum - kmemlp_qnt)) {
13127c478bd9Sstevel@tonic-gate
13137c478bd9Sstevel@tonic-gate /*
13147c478bd9Sstevel@tonic-gate * we are low on free memory in kmem_lp_arena
13157c478bd9Sstevel@tonic-gate * we let only one guy to allocate heap_lp
13167c478bd9Sstevel@tonic-gate * quantum size chunk that everybody is going to
13177c478bd9Sstevel@tonic-gate * share
13187c478bd9Sstevel@tonic-gate */
13197c478bd9Sstevel@tonic-gate mutex_enter(&lpcb->lp_lock);
13207c478bd9Sstevel@tonic-gate
13217c478bd9Sstevel@tonic-gate if (lpcb->lp_wait) {
13227c478bd9Sstevel@tonic-gate
13237c478bd9Sstevel@tonic-gate /* we are not the first one - wait */
13247c478bd9Sstevel@tonic-gate cv_wait(&lpcb->lp_cv, &lpcb->lp_lock);
13257c478bd9Sstevel@tonic-gate if (vmem_size(kmem_lp_arena, VMEM_FREE) <
13267c478bd9Sstevel@tonic-gate kmemlp_qnt) {
13277c478bd9Sstevel@tonic-gate doalloc = 0;
13287c478bd9Sstevel@tonic-gate }
13297c478bd9Sstevel@tonic-gate } else if (vmem_size(kmem_lp_arena, VMEM_FREE) <=
13307c478bd9Sstevel@tonic-gate kmemlp_qnt) {
13317c478bd9Sstevel@tonic-gate
13327c478bd9Sstevel@tonic-gate /*
13337c478bd9Sstevel@tonic-gate * we are the first one, make sure we import
13347c478bd9Sstevel@tonic-gate * a large page
13357c478bd9Sstevel@tonic-gate */
13367c478bd9Sstevel@tonic-gate if (asize == kmemlp_qnt)
13377c478bd9Sstevel@tonic-gate asize += kmemlp_qnt;
13387c478bd9Sstevel@tonic-gate dowakeup = 1;
13397c478bd9Sstevel@tonic-gate lpcb->lp_wait = 1;
13407c478bd9Sstevel@tonic-gate }
13417c478bd9Sstevel@tonic-gate
13427c478bd9Sstevel@tonic-gate mutex_exit(&lpcb->lp_lock);
13437c478bd9Sstevel@tonic-gate }
13447c478bd9Sstevel@tonic-gate
13457c478bd9Sstevel@tonic-gate /*
13467c478bd9Sstevel@tonic-gate * VM_ABORT flag prevents sleeps in vmem_xalloc when
13477c478bd9Sstevel@tonic-gate * large pages are not available. In that case this allocation
13487c478bd9Sstevel@tonic-gate * attempt will fail and we will retry allocation with small
13497c478bd9Sstevel@tonic-gate * pages. We also do not want to panic if this allocation fails
13507c478bd9Sstevel@tonic-gate * because we are going to retry.
13517c478bd9Sstevel@tonic-gate */
13527c478bd9Sstevel@tonic-gate if (doalloc) {
13537c478bd9Sstevel@tonic-gate addr = vmem_alloc(kmem_lp_arena, asize,
13547c478bd9Sstevel@tonic-gate (vmflag | VM_ABORT) & ~VM_PANIC);
13557c478bd9Sstevel@tonic-gate
13567c478bd9Sstevel@tonic-gate if (dowakeup) {
13577c478bd9Sstevel@tonic-gate mutex_enter(&lpcb->lp_lock);
13587c478bd9Sstevel@tonic-gate ASSERT(lpcb->lp_wait != 0);
13597c478bd9Sstevel@tonic-gate lpcb->lp_wait = 0;
13607c478bd9Sstevel@tonic-gate cv_broadcast(&lpcb->lp_cv);
13617c478bd9Sstevel@tonic-gate mutex_exit(&lpcb->lp_lock);
13627c478bd9Sstevel@tonic-gate }
13637c478bd9Sstevel@tonic-gate }
13647c478bd9Sstevel@tonic-gate
13657c478bd9Sstevel@tonic-gate if (addr != NULL) {
13667c478bd9Sstevel@tonic-gate *sizep = asize;
13677c478bd9Sstevel@tonic-gate *lpthrtp = 0;
13687c478bd9Sstevel@tonic-gate return (addr);
13697c478bd9Sstevel@tonic-gate }
13707c478bd9Sstevel@tonic-gate
13717c478bd9Sstevel@tonic-gate if (vmflag & VM_NOSLEEP)
1372501f3284Seg155566 lpcb->nosleep_allocs_failed++;
13737c478bd9Sstevel@tonic-gate else
1374501f3284Seg155566 lpcb->sleep_allocs_failed++;
1375501f3284Seg155566 lpcb->alloc_bytes_failed += size;
13767c478bd9Sstevel@tonic-gate
13777c478bd9Sstevel@tonic-gate /* if large page throttling is not started yet do it */
13787c478bd9Sstevel@tonic-gate if (segkmem_use_lpthrottle && lpthrt == 0) {
13797c478bd9Sstevel@tonic-gate lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1);
13807c478bd9Sstevel@tonic-gate }
13817c478bd9Sstevel@tonic-gate }
13827c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, vmflag));
13837c478bd9Sstevel@tonic-gate }
13847c478bd9Sstevel@tonic-gate
13857c478bd9Sstevel@tonic-gate void
segkmem_free_lp(vmem_t * vmp,void * inaddr,size_t size)13867c478bd9Sstevel@tonic-gate segkmem_free_lp(vmem_t *vmp, void *inaddr, size_t size)
13877c478bd9Sstevel@tonic-gate {
13887c478bd9Sstevel@tonic-gate if (kmem_lp_arena == NULL || !IS_KMEM_VA_LARGEPAGE((caddr_t)inaddr)) {
13897c478bd9Sstevel@tonic-gate segkmem_free(vmp, inaddr, size);
13907c478bd9Sstevel@tonic-gate } else {
13917c478bd9Sstevel@tonic-gate vmem_free(kmem_lp_arena, inaddr, size);
13927c478bd9Sstevel@tonic-gate }
13937c478bd9Sstevel@tonic-gate }
13947c478bd9Sstevel@tonic-gate
1395*ae5a8bedSAndy Fiddaman #ifdef __sparc
13967c478bd9Sstevel@tonic-gate /*
13977c478bd9Sstevel@tonic-gate * segkmem_alloc_lpi() imports virtual memory from large page heap arena
13987c478bd9Sstevel@tonic-gate * into kmem_lp arena. In the process it maps the imported segment with
13997c478bd9Sstevel@tonic-gate * large pages
14007c478bd9Sstevel@tonic-gate */
14017c478bd9Sstevel@tonic-gate static void *
segkmem_alloc_lpi(vmem_t * vmp,size_t size,int vmflag)14027c478bd9Sstevel@tonic-gate segkmem_alloc_lpi(vmem_t *vmp, size_t size, int vmflag)
14037c478bd9Sstevel@tonic-gate {
14047c478bd9Sstevel@tonic-gate segkmem_lpcb_t *lpcb = &segkmem_lpcb;
14057c478bd9Sstevel@tonic-gate void *addr;
14067c478bd9Sstevel@tonic-gate
14077c478bd9Sstevel@tonic-gate ASSERT(size != 0);
14087c478bd9Sstevel@tonic-gate ASSERT(vmp == heap_lp_arena);
14097c478bd9Sstevel@tonic-gate
14107c478bd9Sstevel@tonic-gate /* do not allow large page heap grow beyound limits */
14117c478bd9Sstevel@tonic-gate if (vmem_size(vmp, VMEM_ALLOC) >= segkmem_kmemlp_max) {
1412501f3284Seg155566 lpcb->allocs_limited++;
14137c478bd9Sstevel@tonic-gate return (NULL);
14147c478bd9Sstevel@tonic-gate }
14157c478bd9Sstevel@tonic-gate
14167c478bd9Sstevel@tonic-gate addr = segkmem_xalloc_lp(vmp, NULL, size, vmflag, 0,
14177c478bd9Sstevel@tonic-gate segkmem_page_create_large, NULL);
14187c478bd9Sstevel@tonic-gate return (addr);
14197c478bd9Sstevel@tonic-gate }
14207c478bd9Sstevel@tonic-gate
14217c478bd9Sstevel@tonic-gate /*
14227c478bd9Sstevel@tonic-gate * segkmem_free_lpi() returns virtual memory back into large page heap arena
14237c478bd9Sstevel@tonic-gate * from kmem_lp arena. Beore doing this it unmaps the segment and frees
14247c478bd9Sstevel@tonic-gate * large pages used to map it.
14257c478bd9Sstevel@tonic-gate */
14267c478bd9Sstevel@tonic-gate static void
segkmem_free_lpi(vmem_t * vmp,void * inaddr,size_t size)14277c478bd9Sstevel@tonic-gate segkmem_free_lpi(vmem_t *vmp, void *inaddr, size_t size)
14287c478bd9Sstevel@tonic-gate {
14297c478bd9Sstevel@tonic-gate pgcnt_t nlpages = size >> segkmem_lpshift;
14307c478bd9Sstevel@tonic-gate size_t lpsize = segkmem_lpsize;
14317c478bd9Sstevel@tonic-gate caddr_t addr = inaddr;
14327c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size);
14337c478bd9Sstevel@tonic-gate int i;
14347c478bd9Sstevel@tonic-gate
14357c478bd9Sstevel@tonic-gate ASSERT(vmp == heap_lp_arena);
14367c478bd9Sstevel@tonic-gate ASSERT(IS_KMEM_VA_LARGEPAGE(addr));
14377c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)inaddr & (lpsize - 1)) == 0);
14387c478bd9Sstevel@tonic-gate
14397c478bd9Sstevel@tonic-gate for (i = 0; i < nlpages; i++) {
14407c478bd9Sstevel@tonic-gate segkmem_free_one_lp(addr, lpsize);
14417c478bd9Sstevel@tonic-gate addr += lpsize;
14427c478bd9Sstevel@tonic-gate }
14437c478bd9Sstevel@tonic-gate
14447c478bd9Sstevel@tonic-gate page_unresv(npages);
14457c478bd9Sstevel@tonic-gate
14467c478bd9Sstevel@tonic-gate vmem_free(vmp, inaddr, size);
14477c478bd9Sstevel@tonic-gate }
1448*ae5a8bedSAndy Fiddaman #endif /* __sparc */
14497c478bd9Sstevel@tonic-gate
14507c478bd9Sstevel@tonic-gate /*
14517c478bd9Sstevel@tonic-gate * This function is called at system boot time by kmem_init right after
14527c478bd9Sstevel@tonic-gate * /etc/system file has been read. It checks based on hardware configuration
14537c478bd9Sstevel@tonic-gate * and /etc/system settings if system is going to use large pages. The
14547c478bd9Sstevel@tonic-gate * initialiazation necessary to actually start using large pages
14557c478bd9Sstevel@tonic-gate * happens later in the process after segkmem_heap_lp_init() is called.
14567c478bd9Sstevel@tonic-gate */
14577c478bd9Sstevel@tonic-gate int
segkmem_lpsetup()14587c478bd9Sstevel@tonic-gate segkmem_lpsetup()
14597c478bd9Sstevel@tonic-gate {
14607c478bd9Sstevel@tonic-gate int use_large_pages = 0;
14617c478bd9Sstevel@tonic-gate
14627c478bd9Sstevel@tonic-gate #ifdef __sparc
14637c478bd9Sstevel@tonic-gate
14647c478bd9Sstevel@tonic-gate size_t memtotal = physmem * PAGESIZE;
14657c478bd9Sstevel@tonic-gate
14667c478bd9Sstevel@tonic-gate if (heap_lp_base == NULL) {
14677c478bd9Sstevel@tonic-gate segkmem_lpsize = PAGESIZE;
14687c478bd9Sstevel@tonic-gate return (0);
14697c478bd9Sstevel@tonic-gate }
14707c478bd9Sstevel@tonic-gate
14717c478bd9Sstevel@tonic-gate /* get a platform dependent value of large page size for kernel heap */
14727c478bd9Sstevel@tonic-gate segkmem_lpsize = get_segkmem_lpsize(segkmem_lpsize);
14737c478bd9Sstevel@tonic-gate
14747c478bd9Sstevel@tonic-gate if (segkmem_lpsize <= PAGESIZE) {
14757c478bd9Sstevel@tonic-gate /*
14767c478bd9Sstevel@tonic-gate * put virtual space reserved for the large page kernel
14777c478bd9Sstevel@tonic-gate * back to the regular heap
14787c478bd9Sstevel@tonic-gate */
14797c478bd9Sstevel@tonic-gate vmem_xfree(heap_arena, heap_lp_base,
14807c478bd9Sstevel@tonic-gate heap_lp_end - heap_lp_base);
14817c478bd9Sstevel@tonic-gate heap_lp_base = NULL;
14827c478bd9Sstevel@tonic-gate heap_lp_end = NULL;
14837c478bd9Sstevel@tonic-gate segkmem_lpsize = PAGESIZE;
14847c478bd9Sstevel@tonic-gate return (0);
14857c478bd9Sstevel@tonic-gate }
14867c478bd9Sstevel@tonic-gate
14877c478bd9Sstevel@tonic-gate /* set heap_lp quantum if necessary */
1488de710d24SJosef 'Jeff' Sipek if (segkmem_heaplp_quantum == 0 || !ISP2(segkmem_heaplp_quantum) ||
14897c478bd9Sstevel@tonic-gate P2PHASE(segkmem_heaplp_quantum, segkmem_lpsize)) {
14907c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum = segkmem_lpsize;
14917c478bd9Sstevel@tonic-gate }
14927c478bd9Sstevel@tonic-gate
14937c478bd9Sstevel@tonic-gate /* set kmem_lp quantum if necessary */
1494de710d24SJosef 'Jeff' Sipek if (segkmem_kmemlp_quantum == 0 || !ISP2(segkmem_kmemlp_quantum) ||
14957c478bd9Sstevel@tonic-gate segkmem_kmemlp_quantum > segkmem_heaplp_quantum) {
14967c478bd9Sstevel@tonic-gate segkmem_kmemlp_quantum = segkmem_heaplp_quantum;
14977c478bd9Sstevel@tonic-gate }
14987c478bd9Sstevel@tonic-gate
14997c478bd9Sstevel@tonic-gate /* set total amount of memory allowed for large page kernel heap */
15007c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_max == 0) {
15017c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_pcnt == 0 || segkmem_kmemlp_pcnt > 100)
1502501f3284Seg155566 segkmem_kmemlp_pcnt = 12;
1503501f3284Seg155566 segkmem_kmemlp_max = (memtotal * segkmem_kmemlp_pcnt) / 100;
15047c478bd9Sstevel@tonic-gate }
15057c478bd9Sstevel@tonic-gate segkmem_kmemlp_max = P2ROUNDUP(segkmem_kmemlp_max,
15067c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum);
15077c478bd9Sstevel@tonic-gate
15087c478bd9Sstevel@tonic-gate /* fix lp kmem preallocation request if necesssary */
15097c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_min) {
15107c478bd9Sstevel@tonic-gate segkmem_kmemlp_min = P2ROUNDUP(segkmem_kmemlp_min,
15117c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum);
15127c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_min > segkmem_kmemlp_max)
15137c478bd9Sstevel@tonic-gate segkmem_kmemlp_min = segkmem_kmemlp_max;
15147c478bd9Sstevel@tonic-gate }
15157c478bd9Sstevel@tonic-gate
15167c478bd9Sstevel@tonic-gate use_large_pages = 1;
1517081a94b0Saguzovsk segkmem_lpszc = page_szc(segkmem_lpsize);
1518081a94b0Saguzovsk segkmem_lpshift = page_get_shift(segkmem_lpszc);
15197c478bd9Sstevel@tonic-gate
15207c478bd9Sstevel@tonic-gate #endif
15217c478bd9Sstevel@tonic-gate return (use_large_pages);
15227c478bd9Sstevel@tonic-gate }
15237c478bd9Sstevel@tonic-gate
1524ad23a2dbSjohansen void
segkmem_zio_init(void * zio_mem_base,size_t zio_mem_size)1525ad23a2dbSjohansen segkmem_zio_init(void *zio_mem_base, size_t zio_mem_size)
1526ad23a2dbSjohansen {
1527ad23a2dbSjohansen ASSERT(zio_mem_base != NULL);
1528ad23a2dbSjohansen ASSERT(zio_mem_size != 0);
1529ad23a2dbSjohansen
1530e291592aSJonathan Adams /*
1531e291592aSJonathan Adams * To reduce VA space fragmentation, we set up quantum caches for the
1532e291592aSJonathan Adams * smaller sizes; we chose 32k because that translates to 128k VA
1533e291592aSJonathan Adams * slabs, which matches nicely with the common 128k zio_data bufs.
1534e291592aSJonathan Adams */
15350fab61baSJonathan W Adams zio_arena = vmem_create("zfs_file_data", zio_mem_base, zio_mem_size,
1536e291592aSJonathan Adams PAGESIZE, NULL, NULL, NULL, 32 * 1024, VM_SLEEP);
1537ad23a2dbSjohansen
15380fab61baSJonathan W Adams zio_alloc_arena = vmem_create("zfs_file_data_buf", NULL, 0, PAGESIZE,
1539ad23a2dbSjohansen segkmem_zio_alloc, segkmem_zio_free, zio_arena, 0, VM_SLEEP);
1540ad23a2dbSjohansen
1541ad23a2dbSjohansen ASSERT(zio_arena != NULL);
1542ad23a2dbSjohansen ASSERT(zio_alloc_arena != NULL);
1543ad23a2dbSjohansen }
1544ad23a2dbSjohansen
154504909c8cSJohn Levon #if defined(__amd64)
15467c478bd9Sstevel@tonic-gate
154704909c8cSJohn Levon void
segkmem_kvmm_init(void * base,size_t size)154804909c8cSJohn Levon segkmem_kvmm_init(void *base, size_t size)
154904909c8cSJohn Levon {
155004909c8cSJohn Levon ASSERT(base != NULL);
155104909c8cSJohn Levon ASSERT(size != 0);
155204909c8cSJohn Levon
155304909c8cSJohn Levon kvmm_arena = vmem_create("kvmm_arena", base, size, 1024 * 1024,
155404909c8cSJohn Levon NULL, NULL, NULL, 0, VM_SLEEP);
155504909c8cSJohn Levon
155604909c8cSJohn Levon ASSERT(kvmm_arena != NULL);
155704909c8cSJohn Levon }
155804909c8cSJohn Levon
155904909c8cSJohn Levon #elif defined(__sparc)
15607c478bd9Sstevel@tonic-gate
15617c478bd9Sstevel@tonic-gate static void *
segkmem_alloc_ppa(vmem_t * vmp,size_t size,int vmflag)15627c478bd9Sstevel@tonic-gate segkmem_alloc_ppa(vmem_t *vmp, size_t size, int vmflag)
15637c478bd9Sstevel@tonic-gate {
15647c478bd9Sstevel@tonic-gate size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
15657c478bd9Sstevel@tonic-gate void *addr;
15667c478bd9Sstevel@tonic-gate
15677c478bd9Sstevel@tonic-gate if (ppaquantum <= PAGESIZE)
15687c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, vmflag));
15697c478bd9Sstevel@tonic-gate
15707c478bd9Sstevel@tonic-gate ASSERT((size & (ppaquantum - 1)) == 0);
15717c478bd9Sstevel@tonic-gate
15727c478bd9Sstevel@tonic-gate addr = vmem_xalloc(vmp, size, ppaquantum, 0, 0, NULL, NULL, vmflag);
15737c478bd9Sstevel@tonic-gate if (addr != NULL && segkmem_xalloc(vmp, addr, size, vmflag, 0,
15747c478bd9Sstevel@tonic-gate segkmem_page_create, NULL) == NULL) {
15757c478bd9Sstevel@tonic-gate vmem_xfree(vmp, addr, size);
15767c478bd9Sstevel@tonic-gate addr = NULL;
15777c478bd9Sstevel@tonic-gate }
15787c478bd9Sstevel@tonic-gate
15797c478bd9Sstevel@tonic-gate return (addr);
15807c478bd9Sstevel@tonic-gate }
15817c478bd9Sstevel@tonic-gate
15827c478bd9Sstevel@tonic-gate static void
segkmem_free_ppa(vmem_t * vmp,void * addr,size_t size)15837c478bd9Sstevel@tonic-gate segkmem_free_ppa(vmem_t *vmp, void *addr, size_t size)
15847c478bd9Sstevel@tonic-gate {
15857c478bd9Sstevel@tonic-gate size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
15867c478bd9Sstevel@tonic-gate
15877c478bd9Sstevel@tonic-gate ASSERT(addr != NULL);
15887c478bd9Sstevel@tonic-gate
15897c478bd9Sstevel@tonic-gate if (ppaquantum <= PAGESIZE) {
15907c478bd9Sstevel@tonic-gate segkmem_free(vmp, addr, size);
15917c478bd9Sstevel@tonic-gate } else {
15927c478bd9Sstevel@tonic-gate segkmem_free(NULL, addr, size);
15937c478bd9Sstevel@tonic-gate vmem_xfree(vmp, addr, size);
15947c478bd9Sstevel@tonic-gate }
15957c478bd9Sstevel@tonic-gate }
15967c478bd9Sstevel@tonic-gate
15977c478bd9Sstevel@tonic-gate void
segkmem_heap_lp_init()15987c478bd9Sstevel@tonic-gate segkmem_heap_lp_init()
15997c478bd9Sstevel@tonic-gate {
16007c478bd9Sstevel@tonic-gate segkmem_lpcb_t *lpcb = &segkmem_lpcb;
16017c478bd9Sstevel@tonic-gate size_t heap_lp_size = heap_lp_end - heap_lp_base;
16027c478bd9Sstevel@tonic-gate size_t lpsize = segkmem_lpsize;
16037c478bd9Sstevel@tonic-gate size_t ppaquantum;
16047c478bd9Sstevel@tonic-gate void *addr;
16057c478bd9Sstevel@tonic-gate
16067c478bd9Sstevel@tonic-gate if (segkmem_lpsize <= PAGESIZE) {
16077c478bd9Sstevel@tonic-gate ASSERT(heap_lp_base == NULL);
16087c478bd9Sstevel@tonic-gate ASSERT(heap_lp_end == NULL);
16097c478bd9Sstevel@tonic-gate return;
16107c478bd9Sstevel@tonic-gate }
16117c478bd9Sstevel@tonic-gate
16127c478bd9Sstevel@tonic-gate ASSERT(segkmem_heaplp_quantum >= lpsize);
16137c478bd9Sstevel@tonic-gate ASSERT((segkmem_heaplp_quantum & (lpsize - 1)) == 0);
16147c478bd9Sstevel@tonic-gate ASSERT(lpcb->lp_uselp == 0);
16157c478bd9Sstevel@tonic-gate ASSERT(heap_lp_base != NULL);
16167c478bd9Sstevel@tonic-gate ASSERT(heap_lp_end != NULL);
16177c478bd9Sstevel@tonic-gate ASSERT(heap_lp_base < heap_lp_end);
16187c478bd9Sstevel@tonic-gate ASSERT(heap_lp_arena == NULL);
16197c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)heap_lp_base & (lpsize - 1)) == 0);
16207c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)heap_lp_end & (lpsize - 1)) == 0);
16217c478bd9Sstevel@tonic-gate
16227c478bd9Sstevel@tonic-gate /* create large page heap arena */
16237c478bd9Sstevel@tonic-gate heap_lp_arena = vmem_create("heap_lp", heap_lp_base, heap_lp_size,
16247c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
16257c478bd9Sstevel@tonic-gate
16267c478bd9Sstevel@tonic-gate ASSERT(heap_lp_arena != NULL);
16277c478bd9Sstevel@tonic-gate
16287c478bd9Sstevel@tonic-gate /* This arena caches memory already mapped by large pages */
16297c478bd9Sstevel@tonic-gate kmem_lp_arena = vmem_create("kmem_lp", NULL, 0, segkmem_kmemlp_quantum,
16307c478bd9Sstevel@tonic-gate segkmem_alloc_lpi, segkmem_free_lpi, heap_lp_arena, 0, VM_SLEEP);
16317c478bd9Sstevel@tonic-gate
16327c478bd9Sstevel@tonic-gate ASSERT(kmem_lp_arena != NULL);
16337c478bd9Sstevel@tonic-gate
16347c478bd9Sstevel@tonic-gate mutex_init(&lpcb->lp_lock, NULL, MUTEX_DEFAULT, NULL);
16357c478bd9Sstevel@tonic-gate cv_init(&lpcb->lp_cv, NULL, CV_DEFAULT, NULL);
16367c478bd9Sstevel@tonic-gate
16377c478bd9Sstevel@tonic-gate /*
16387c478bd9Sstevel@tonic-gate * this arena is used for the array of page_t pointers necessary
16397c478bd9Sstevel@tonic-gate * to call hat_mem_load_array
16407c478bd9Sstevel@tonic-gate */
16417c478bd9Sstevel@tonic-gate ppaquantum = btopr(lpsize) * sizeof (page_t *);
16427c478bd9Sstevel@tonic-gate segkmem_ppa_arena = vmem_create("segkmem_ppa", NULL, 0, ppaquantum,
16437c478bd9Sstevel@tonic-gate segkmem_alloc_ppa, segkmem_free_ppa, heap_arena, ppaquantum,
16447c478bd9Sstevel@tonic-gate VM_SLEEP);
16457c478bd9Sstevel@tonic-gate
16467c478bd9Sstevel@tonic-gate ASSERT(segkmem_ppa_arena != NULL);
16477c478bd9Sstevel@tonic-gate
16487c478bd9Sstevel@tonic-gate /* prealloacate some memory for the lp kernel heap */
16497c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_min) {
16507c478bd9Sstevel@tonic-gate
16517c478bd9Sstevel@tonic-gate ASSERT(P2PHASE(segkmem_kmemlp_min,
16527c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum) == 0);
16537c478bd9Sstevel@tonic-gate
16547c478bd9Sstevel@tonic-gate if ((addr = segkmem_alloc_lpi(heap_lp_arena,
16557c478bd9Sstevel@tonic-gate segkmem_kmemlp_min, VM_SLEEP)) != NULL) {
16567c478bd9Sstevel@tonic-gate
16577c478bd9Sstevel@tonic-gate addr = vmem_add(kmem_lp_arena, addr,
16587c478bd9Sstevel@tonic-gate segkmem_kmemlp_min, VM_SLEEP);
16597c478bd9Sstevel@tonic-gate ASSERT(addr != NULL);
16607c478bd9Sstevel@tonic-gate }
16617c478bd9Sstevel@tonic-gate }
16627c478bd9Sstevel@tonic-gate
16637c478bd9Sstevel@tonic-gate lpcb->lp_uselp = 1;
16647c478bd9Sstevel@tonic-gate }
16657c478bd9Sstevel@tonic-gate
16667c478bd9Sstevel@tonic-gate #endif
1667