17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5ad23a2dbSjohansen * Common Development and Distribution License (the "License"). 6ad23a2dbSjohansen * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22a98e9dbfSaguzovsk * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #include <sys/types.h> 277c478bd9Sstevel@tonic-gate #include <sys/t_lock.h> 287c478bd9Sstevel@tonic-gate #include <sys/param.h> 297c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 307c478bd9Sstevel@tonic-gate #include <sys/tuneable.h> 317c478bd9Sstevel@tonic-gate #include <sys/systm.h> 327c478bd9Sstevel@tonic-gate #include <sys/vm.h> 337c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 347c478bd9Sstevel@tonic-gate #include <sys/vmem.h> 357c478bd9Sstevel@tonic-gate #include <sys/mman.h> 367c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 377c478bd9Sstevel@tonic-gate #include <sys/debug.h> 387c478bd9Sstevel@tonic-gate #include <sys/dumphdr.h> 397c478bd9Sstevel@tonic-gate #include <sys/bootconf.h> 407c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 417c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 427c478bd9Sstevel@tonic-gate #include <vm/hat.h> 437c478bd9Sstevel@tonic-gate #include <vm/page.h> 447c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h> 457c478bd9Sstevel@tonic-gate #include <vm/faultcode.h> 467c478bd9Sstevel@tonic-gate #include <sys/promif.h> 477c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h> 487c478bd9Sstevel@tonic-gate #include <sys/bitmap.h> 497c478bd9Sstevel@tonic-gate #include <sys/mem_cage.h> 507c478bd9Sstevel@tonic-gate 517c478bd9Sstevel@tonic-gate /* 527c478bd9Sstevel@tonic-gate * seg_kmem is the primary kernel memory segment driver. It 537c478bd9Sstevel@tonic-gate * maps the kernel heap [kernelheap, ekernelheap), module text, 547c478bd9Sstevel@tonic-gate * and all memory which was allocated before the VM was initialized 557c478bd9Sstevel@tonic-gate * into kas. 567c478bd9Sstevel@tonic-gate * 577c478bd9Sstevel@tonic-gate * Pages which belong to seg_kmem are hashed into &kvp vnode at 587c478bd9Sstevel@tonic-gate * an offset equal to (u_offset_t)virt_addr, and have p_lckcnt >= 1. 597c478bd9Sstevel@tonic-gate * They must never be paged out since segkmem_fault() is a no-op to 607c478bd9Sstevel@tonic-gate * prevent recursive faults. 617c478bd9Sstevel@tonic-gate * 627c478bd9Sstevel@tonic-gate * Currently, seg_kmem pages are sharelocked (p_sharelock == 1) on 637c478bd9Sstevel@tonic-gate * __x86 and are unlocked (p_sharelock == 0) on __sparc. Once __x86 647c478bd9Sstevel@tonic-gate * supports relocation the #ifdef kludges can be removed. 657c478bd9Sstevel@tonic-gate * 667c478bd9Sstevel@tonic-gate * seg_kmem pages may be subject to relocation by page_relocate(), 677c478bd9Sstevel@tonic-gate * provided that the HAT supports it; if this is so, segkmem_reloc 687c478bd9Sstevel@tonic-gate * will be set to a nonzero value. All boot time allocated memory as 697c478bd9Sstevel@tonic-gate * well as static memory is considered off limits to relocation. 707c478bd9Sstevel@tonic-gate * Pages are "relocatable" if p_state does not have P_NORELOC set, so 717c478bd9Sstevel@tonic-gate * we request P_NORELOC pages for memory that isn't safe to relocate. 727c478bd9Sstevel@tonic-gate * 737c478bd9Sstevel@tonic-gate * The kernel heap is logically divided up into four pieces: 747c478bd9Sstevel@tonic-gate * 757c478bd9Sstevel@tonic-gate * heap32_arena is for allocations that require 32-bit absolute 767c478bd9Sstevel@tonic-gate * virtual addresses (e.g. code that uses 32-bit pointers/offsets). 777c478bd9Sstevel@tonic-gate * 787c478bd9Sstevel@tonic-gate * heap_core is for allocations that require 2GB *relative* 797c478bd9Sstevel@tonic-gate * offsets; in other words all memory from heap_core is within 807c478bd9Sstevel@tonic-gate * 2GB of all other memory from the same arena. This is a requirement 817c478bd9Sstevel@tonic-gate * of the addressing modes of some processors in supervisor code. 827c478bd9Sstevel@tonic-gate * 837c478bd9Sstevel@tonic-gate * heap_arena is the general heap arena. 847c478bd9Sstevel@tonic-gate * 857c478bd9Sstevel@tonic-gate * static_arena is the static memory arena. Allocations from it 867c478bd9Sstevel@tonic-gate * are not subject to relocation so it is safe to use the memory 877c478bd9Sstevel@tonic-gate * physical address as well as the virtual address (e.g. the VA to 887c478bd9Sstevel@tonic-gate * PA translations are static). Caches may import from static_arena; 897c478bd9Sstevel@tonic-gate * all other static memory allocations should use static_alloc_arena. 907c478bd9Sstevel@tonic-gate * 917c478bd9Sstevel@tonic-gate * On some platforms which have limited virtual address space, seg_kmem 927c478bd9Sstevel@tonic-gate * may share [kernelheap, ekernelheap) with seg_kp; if this is so, 937c478bd9Sstevel@tonic-gate * segkp_bitmap is non-NULL, and each bit represents a page of virtual 947c478bd9Sstevel@tonic-gate * address space which is actually seg_kp mapped. 957c478bd9Sstevel@tonic-gate */ 967c478bd9Sstevel@tonic-gate 977c478bd9Sstevel@tonic-gate extern ulong_t *segkp_bitmap; /* Is set if segkp is from the kernel heap */ 987c478bd9Sstevel@tonic-gate 997c478bd9Sstevel@tonic-gate char *kernelheap; /* start of primary kernel heap */ 1007c478bd9Sstevel@tonic-gate char *ekernelheap; /* end of primary kernel heap */ 1017c478bd9Sstevel@tonic-gate struct seg kvseg; /* primary kernel heap segment */ 1027c478bd9Sstevel@tonic-gate struct seg kvseg_core; /* "core" kernel heap segment */ 103ad23a2dbSjohansen struct seg kzioseg; /* Segment for zio mappings */ 1047c478bd9Sstevel@tonic-gate vmem_t *heap_arena; /* primary kernel heap arena */ 1057c478bd9Sstevel@tonic-gate vmem_t *heap_core_arena; /* core kernel heap arena */ 1067c478bd9Sstevel@tonic-gate char *heap_core_base; /* start of core kernel heap arena */ 1077c478bd9Sstevel@tonic-gate char *heap_lp_base; /* start of kernel large page heap arena */ 1087c478bd9Sstevel@tonic-gate char *heap_lp_end; /* end of kernel large page heap arena */ 1097c478bd9Sstevel@tonic-gate vmem_t *hat_memload_arena; /* HAT translation data */ 1107c478bd9Sstevel@tonic-gate struct seg kvseg32; /* 32-bit kernel heap segment */ 1117c478bd9Sstevel@tonic-gate vmem_t *heap32_arena; /* 32-bit kernel heap arena */ 1127c478bd9Sstevel@tonic-gate vmem_t *heaptext_arena; /* heaptext arena */ 1137c478bd9Sstevel@tonic-gate struct as kas; /* kernel address space */ 1147c478bd9Sstevel@tonic-gate struct vnode kvp; /* vnode for all segkmem pages */ 115ad23a2dbSjohansen struct vnode zvp; /* vnode for zfs pages */ 1167c478bd9Sstevel@tonic-gate int segkmem_reloc; /* enable/disable relocatable segkmem pages */ 1177c478bd9Sstevel@tonic-gate vmem_t *static_arena; /* arena for caches to import static memory */ 1187c478bd9Sstevel@tonic-gate vmem_t *static_alloc_arena; /* arena for allocating static memory */ 119ad23a2dbSjohansen vmem_t *zio_arena = NULL; /* arena for allocating zio memory */ 120ad23a2dbSjohansen vmem_t *zio_alloc_arena = NULL; /* arena for allocating zio memory */ 1217c478bd9Sstevel@tonic-gate 1227c478bd9Sstevel@tonic-gate /* 1237c478bd9Sstevel@tonic-gate * seg_kmem driver can map part of the kernel heap with large pages. 1247c478bd9Sstevel@tonic-gate * Currently this functionality is implemented for sparc platforms only. 1257c478bd9Sstevel@tonic-gate * 1267c478bd9Sstevel@tonic-gate * The large page size "segkmem_lpsize" for kernel heap is selected in the 1277c478bd9Sstevel@tonic-gate * platform specific code. It can also be modified via /etc/system file. 1287c478bd9Sstevel@tonic-gate * Setting segkmem_lpsize to PAGESIZE in /etc/system disables usage of large 1297c478bd9Sstevel@tonic-gate * pages for kernel heap. "segkmem_lpshift" is adjusted appropriately to 1307c478bd9Sstevel@tonic-gate * match segkmem_lpsize. 1317c478bd9Sstevel@tonic-gate * 1327c478bd9Sstevel@tonic-gate * At boot time we carve from kernel heap arena a range of virtual addresses 1337c478bd9Sstevel@tonic-gate * that will be used for large page mappings. This range [heap_lp_base, 1347c478bd9Sstevel@tonic-gate * heap_lp_end) is set up as a separate vmem arena - "heap_lp_arena". We also 1357c478bd9Sstevel@tonic-gate * create "kmem_lp_arena" that caches memory already backed up by large 1367c478bd9Sstevel@tonic-gate * pages. kmem_lp_arena imports virtual segments from heap_lp_arena. 1377c478bd9Sstevel@tonic-gate */ 1387c478bd9Sstevel@tonic-gate 1397c478bd9Sstevel@tonic-gate size_t segkmem_lpsize; 1407c478bd9Sstevel@tonic-gate static uint_t segkmem_lpshift = PAGESHIFT; 141081a94b0Saguzovsk int segkmem_lpszc = 0; 1427c478bd9Sstevel@tonic-gate 1437c478bd9Sstevel@tonic-gate size_t segkmem_kmemlp_quantum = 0x400000; /* 4MB */ 1447c478bd9Sstevel@tonic-gate size_t segkmem_heaplp_quantum; 1457bc98a2eSeg155566 vmem_t *heap_lp_arena; 1467c478bd9Sstevel@tonic-gate static vmem_t *kmem_lp_arena; 1477c478bd9Sstevel@tonic-gate static vmem_t *segkmem_ppa_arena; 1487c478bd9Sstevel@tonic-gate static segkmem_lpcb_t segkmem_lpcb; 1497c478bd9Sstevel@tonic-gate 1507c478bd9Sstevel@tonic-gate /* 1517c478bd9Sstevel@tonic-gate * We use "segkmem_kmemlp_max" to limit the total amount of physical memory 152501f3284Seg155566 * consumed by the large page heap. By default this parameter is set to 1/8 of 1537c478bd9Sstevel@tonic-gate * physmem but can be adjusted through /etc/system either directly or 1547c478bd9Sstevel@tonic-gate * indirectly by setting "segkmem_kmemlp_pcnt" to the percent of physmem 1557c478bd9Sstevel@tonic-gate * we allow for large page heap. 1567c478bd9Sstevel@tonic-gate */ 1577c478bd9Sstevel@tonic-gate size_t segkmem_kmemlp_max; 1587c478bd9Sstevel@tonic-gate static uint_t segkmem_kmemlp_pcnt; 1597c478bd9Sstevel@tonic-gate 1607c478bd9Sstevel@tonic-gate /* 1617c478bd9Sstevel@tonic-gate * Getting large pages for kernel heap could be problematic due to 1627c478bd9Sstevel@tonic-gate * physical memory fragmentation. That's why we allow to preallocate 1637c478bd9Sstevel@tonic-gate * "segkmem_kmemlp_min" bytes at boot time. 1647c478bd9Sstevel@tonic-gate */ 1657c478bd9Sstevel@tonic-gate static size_t segkmem_kmemlp_min; 1667c478bd9Sstevel@tonic-gate 1677c478bd9Sstevel@tonic-gate /* 1687c478bd9Sstevel@tonic-gate * Throttling is used to avoid expensive tries to allocate large pages 1697c478bd9Sstevel@tonic-gate * for kernel heap when a lot of succesive attempts to do so fail. 1707c478bd9Sstevel@tonic-gate */ 1717c478bd9Sstevel@tonic-gate static ulong_t segkmem_lpthrottle_max = 0x400000; 1727c478bd9Sstevel@tonic-gate static ulong_t segkmem_lpthrottle_start = 0x40; 1737c478bd9Sstevel@tonic-gate static ulong_t segkmem_use_lpthrottle = 1; 1747c478bd9Sstevel@tonic-gate 1757c478bd9Sstevel@tonic-gate /* 1767c478bd9Sstevel@tonic-gate * Freed pages accumulate on a garbage list until segkmem is ready, 1777c478bd9Sstevel@tonic-gate * at which point we call segkmem_gc() to free it all. 1787c478bd9Sstevel@tonic-gate */ 1797c478bd9Sstevel@tonic-gate typedef struct segkmem_gc_list { 1807c478bd9Sstevel@tonic-gate struct segkmem_gc_list *gc_next; 1817c478bd9Sstevel@tonic-gate vmem_t *gc_arena; 1827c478bd9Sstevel@tonic-gate size_t gc_size; 1837c478bd9Sstevel@tonic-gate } segkmem_gc_list_t; 1847c478bd9Sstevel@tonic-gate 1857c478bd9Sstevel@tonic-gate static segkmem_gc_list_t *segkmem_gc_list; 1867c478bd9Sstevel@tonic-gate 1877c478bd9Sstevel@tonic-gate /* 1887c478bd9Sstevel@tonic-gate * Allocations from the hat_memload arena add VM_MEMLOAD to their 1897c478bd9Sstevel@tonic-gate * vmflags so that segkmem_xalloc() can inform the hat layer that it needs 1907c478bd9Sstevel@tonic-gate * to take steps to prevent infinite recursion. HAT allocations also 1917c478bd9Sstevel@tonic-gate * must be non-relocatable to prevent recursive page faults. 1927c478bd9Sstevel@tonic-gate */ 1937c478bd9Sstevel@tonic-gate static void * 1947c478bd9Sstevel@tonic-gate hat_memload_alloc(vmem_t *vmp, size_t size, int flags) 1957c478bd9Sstevel@tonic-gate { 1967c478bd9Sstevel@tonic-gate flags |= (VM_MEMLOAD | VM_NORELOC); 1977c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, flags)); 1987c478bd9Sstevel@tonic-gate } 1997c478bd9Sstevel@tonic-gate 2007c478bd9Sstevel@tonic-gate /* 2017c478bd9Sstevel@tonic-gate * Allocations from static_arena arena (or any other arena that uses 2027c478bd9Sstevel@tonic-gate * segkmem_alloc_permanent()) require non-relocatable (permanently 2037c478bd9Sstevel@tonic-gate * wired) memory pages, since these pages are referenced by physical 2047c478bd9Sstevel@tonic-gate * as well as virtual address. 2057c478bd9Sstevel@tonic-gate */ 2067c478bd9Sstevel@tonic-gate void * 2077c478bd9Sstevel@tonic-gate segkmem_alloc_permanent(vmem_t *vmp, size_t size, int flags) 2087c478bd9Sstevel@tonic-gate { 2097c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, flags | VM_NORELOC)); 2107c478bd9Sstevel@tonic-gate } 2117c478bd9Sstevel@tonic-gate 2127c478bd9Sstevel@tonic-gate /* 2137c478bd9Sstevel@tonic-gate * Initialize kernel heap boundaries. 2147c478bd9Sstevel@tonic-gate */ 2157c478bd9Sstevel@tonic-gate void 2167c478bd9Sstevel@tonic-gate kernelheap_init( 2177c478bd9Sstevel@tonic-gate void *heap_start, 2187c478bd9Sstevel@tonic-gate void *heap_end, 2197c478bd9Sstevel@tonic-gate char *first_avail, 2207c478bd9Sstevel@tonic-gate void *core_start, 2217c478bd9Sstevel@tonic-gate void *core_end) 2227c478bd9Sstevel@tonic-gate { 2237c478bd9Sstevel@tonic-gate uintptr_t textbase; 2247c478bd9Sstevel@tonic-gate size_t core_size; 2257c478bd9Sstevel@tonic-gate size_t heap_size; 2267c478bd9Sstevel@tonic-gate vmem_t *heaptext_parent; 2277c478bd9Sstevel@tonic-gate size_t heap_lp_size = 0; 228bb121940Sdp78419 #ifdef __sparc 229bb121940Sdp78419 size_t kmem64_sz = kmem64_aligned_end - kmem64_base; 230bb121940Sdp78419 #endif /* __sparc */ 2317c478bd9Sstevel@tonic-gate 2327c478bd9Sstevel@tonic-gate kernelheap = heap_start; 2337c478bd9Sstevel@tonic-gate ekernelheap = heap_end; 2347c478bd9Sstevel@tonic-gate 2357c478bd9Sstevel@tonic-gate #ifdef __sparc 2367c478bd9Sstevel@tonic-gate heap_lp_size = (((uintptr_t)heap_end - (uintptr_t)heap_start) / 4); 237bb121940Sdp78419 /* 238bb121940Sdp78419 * Bias heap_lp start address by kmem64_sz to reduce collisions 239bb121940Sdp78419 * in 4M kernel TSB between kmem64 area and heap_lp 240bb121940Sdp78419 */ 241bb121940Sdp78419 kmem64_sz = P2ROUNDUP(kmem64_sz, MMU_PAGESIZE256M); 242bb121940Sdp78419 if (kmem64_sz <= heap_lp_size / 2) 243bb121940Sdp78419 heap_lp_size -= kmem64_sz; 2447c478bd9Sstevel@tonic-gate heap_lp_base = ekernelheap - heap_lp_size; 2457c478bd9Sstevel@tonic-gate heap_lp_end = heap_lp_base + heap_lp_size; 2467c478bd9Sstevel@tonic-gate #endif /* __sparc */ 2477c478bd9Sstevel@tonic-gate 2487c478bd9Sstevel@tonic-gate /* 2497c478bd9Sstevel@tonic-gate * If this platform has a 'core' heap area, then the space for 2507c478bd9Sstevel@tonic-gate * overflow module text should be carved out of the end of that 2517c478bd9Sstevel@tonic-gate * heap. Otherwise, it gets carved out of the general purpose 2527c478bd9Sstevel@tonic-gate * heap. 2537c478bd9Sstevel@tonic-gate */ 2547c478bd9Sstevel@tonic-gate core_size = (uintptr_t)core_end - (uintptr_t)core_start; 2557c478bd9Sstevel@tonic-gate if (core_size > 0) { 2567c478bd9Sstevel@tonic-gate ASSERT(core_size >= HEAPTEXT_SIZE); 2577c478bd9Sstevel@tonic-gate textbase = (uintptr_t)core_end - HEAPTEXT_SIZE; 2587c478bd9Sstevel@tonic-gate core_size -= HEAPTEXT_SIZE; 2597c478bd9Sstevel@tonic-gate } 2607c478bd9Sstevel@tonic-gate #ifndef __sparc 2617c478bd9Sstevel@tonic-gate else { 2627c478bd9Sstevel@tonic-gate ekernelheap -= HEAPTEXT_SIZE; 2637c478bd9Sstevel@tonic-gate textbase = (uintptr_t)ekernelheap; 2647c478bd9Sstevel@tonic-gate } 2657c478bd9Sstevel@tonic-gate #endif 2667c478bd9Sstevel@tonic-gate 2677c478bd9Sstevel@tonic-gate heap_size = (uintptr_t)ekernelheap - (uintptr_t)kernelheap; 2687c478bd9Sstevel@tonic-gate heap_arena = vmem_init("heap", kernelheap, heap_size, PAGESIZE, 2697c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free); 2707c478bd9Sstevel@tonic-gate 2717c478bd9Sstevel@tonic-gate if (core_size > 0) { 2727c478bd9Sstevel@tonic-gate heap_core_arena = vmem_create("heap_core", core_start, 2737c478bd9Sstevel@tonic-gate core_size, PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP); 2747c478bd9Sstevel@tonic-gate heap_core_base = core_start; 2757c478bd9Sstevel@tonic-gate } else { 2767c478bd9Sstevel@tonic-gate heap_core_arena = heap_arena; 2777c478bd9Sstevel@tonic-gate heap_core_base = kernelheap; 2787c478bd9Sstevel@tonic-gate } 2797c478bd9Sstevel@tonic-gate 2807c478bd9Sstevel@tonic-gate /* 2817c478bd9Sstevel@tonic-gate * reserve space for the large page heap. If large pages for kernel 2827c478bd9Sstevel@tonic-gate * heap is enabled large page heap arean will be created later in the 2837c478bd9Sstevel@tonic-gate * boot sequence in segkmem_heap_lp_init(). Otherwise the allocated 2847c478bd9Sstevel@tonic-gate * range will be returned back to the heap_arena. 2857c478bd9Sstevel@tonic-gate */ 2867c478bd9Sstevel@tonic-gate if (heap_lp_size) { 2877c478bd9Sstevel@tonic-gate (void) vmem_xalloc(heap_arena, heap_lp_size, PAGESIZE, 0, 0, 2887c478bd9Sstevel@tonic-gate heap_lp_base, heap_lp_end, 2897c478bd9Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 2907c478bd9Sstevel@tonic-gate } 2917c478bd9Sstevel@tonic-gate 2927c478bd9Sstevel@tonic-gate /* 2937c478bd9Sstevel@tonic-gate * Remove the already-spoken-for memory range [kernelheap, first_avail). 2947c478bd9Sstevel@tonic-gate */ 2957c478bd9Sstevel@tonic-gate (void) vmem_xalloc(heap_arena, first_avail - kernelheap, PAGESIZE, 2967c478bd9Sstevel@tonic-gate 0, 0, kernelheap, first_avail, VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 2977c478bd9Sstevel@tonic-gate 2987c478bd9Sstevel@tonic-gate #ifdef __sparc 2997c478bd9Sstevel@tonic-gate heap32_arena = vmem_create("heap32", (void *)SYSBASE32, 3007c478bd9Sstevel@tonic-gate SYSLIMIT32 - SYSBASE32 - HEAPTEXT_SIZE, PAGESIZE, NULL, 3017c478bd9Sstevel@tonic-gate NULL, NULL, 0, VM_SLEEP); 3027c478bd9Sstevel@tonic-gate 3037c478bd9Sstevel@tonic-gate textbase = SYSLIMIT32 - HEAPTEXT_SIZE; 3047c478bd9Sstevel@tonic-gate heaptext_parent = NULL; 3057c478bd9Sstevel@tonic-gate #else /* __sparc */ 3067c478bd9Sstevel@tonic-gate heap32_arena = heap_core_arena; 3077c478bd9Sstevel@tonic-gate heaptext_parent = heap_core_arena; 3087c478bd9Sstevel@tonic-gate #endif /* __sparc */ 3097c478bd9Sstevel@tonic-gate 3107c478bd9Sstevel@tonic-gate heaptext_arena = vmem_create("heaptext", (void *)textbase, 3117c478bd9Sstevel@tonic-gate HEAPTEXT_SIZE, PAGESIZE, NULL, NULL, heaptext_parent, 0, VM_SLEEP); 3127c478bd9Sstevel@tonic-gate 3137c478bd9Sstevel@tonic-gate /* 3147c478bd9Sstevel@tonic-gate * Create a set of arenas for memory with static translations 3157c478bd9Sstevel@tonic-gate * (e.g. VA -> PA translations cannot change). Since using 3167c478bd9Sstevel@tonic-gate * kernel pages by physical address implies it isn't safe to 3177c478bd9Sstevel@tonic-gate * walk across page boundaries, the static_arena quantum must 3187c478bd9Sstevel@tonic-gate * be PAGESIZE. Any kmem caches that require static memory 3197c478bd9Sstevel@tonic-gate * should source from static_arena, while direct allocations 3207c478bd9Sstevel@tonic-gate * should only use static_alloc_arena. 3217c478bd9Sstevel@tonic-gate */ 3227c478bd9Sstevel@tonic-gate static_arena = vmem_create("static", NULL, 0, PAGESIZE, 3237c478bd9Sstevel@tonic-gate segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP); 3247c478bd9Sstevel@tonic-gate static_alloc_arena = vmem_create("static_alloc", NULL, 0, 3257c478bd9Sstevel@tonic-gate sizeof (uint64_t), vmem_alloc, vmem_free, static_arena, 3267c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 3277c478bd9Sstevel@tonic-gate 3287c478bd9Sstevel@tonic-gate /* 3297c478bd9Sstevel@tonic-gate * Create an arena for translation data (ptes, hmes, or hblks). 3307c478bd9Sstevel@tonic-gate * We need an arena for this because hat_memload() is essential 3317c478bd9Sstevel@tonic-gate * to vmem_populate() (see comments in common/os/vmem.c). 3327c478bd9Sstevel@tonic-gate * 3337c478bd9Sstevel@tonic-gate * Note: any kmem cache that allocates from hat_memload_arena 3347c478bd9Sstevel@tonic-gate * must be created as a KMC_NOHASH cache (i.e. no external slab 3357c478bd9Sstevel@tonic-gate * and bufctl structures to allocate) so that slab creation doesn't 3367c478bd9Sstevel@tonic-gate * require anything more than a single vmem_alloc(). 3377c478bd9Sstevel@tonic-gate */ 3387c478bd9Sstevel@tonic-gate hat_memload_arena = vmem_create("hat_memload", NULL, 0, PAGESIZE, 3397c478bd9Sstevel@tonic-gate hat_memload_alloc, segkmem_free, heap_arena, 0, 3407c478bd9Sstevel@tonic-gate VM_SLEEP | VMC_POPULATOR); 3417c478bd9Sstevel@tonic-gate } 3427c478bd9Sstevel@tonic-gate 3437c478bd9Sstevel@tonic-gate void 3447c478bd9Sstevel@tonic-gate boot_mapin(caddr_t addr, size_t size) 3457c478bd9Sstevel@tonic-gate { 3467c478bd9Sstevel@tonic-gate caddr_t eaddr; 3477c478bd9Sstevel@tonic-gate page_t *pp; 3487c478bd9Sstevel@tonic-gate pfn_t pfnum; 3497c478bd9Sstevel@tonic-gate 3507c478bd9Sstevel@tonic-gate if (page_resv(btop(size), KM_NOSLEEP) == 0) 3517c478bd9Sstevel@tonic-gate panic("boot_mapin: page_resv failed"); 3527c478bd9Sstevel@tonic-gate 3537c478bd9Sstevel@tonic-gate for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) { 3547c478bd9Sstevel@tonic-gate pfnum = va_to_pfn(addr); 3550cfdb603Sjosephb if (pfnum == PFN_INVALID) 3560cfdb603Sjosephb continue; 3577c478bd9Sstevel@tonic-gate if ((pp = page_numtopp_nolock(pfnum)) == NULL) 3587c478bd9Sstevel@tonic-gate panic("boot_mapin(): No pp for pfnum = %lx", pfnum); 3597c478bd9Sstevel@tonic-gate 3607c478bd9Sstevel@tonic-gate /* 3617c478bd9Sstevel@tonic-gate * must break up any large pages that may have constituent 3627c478bd9Sstevel@tonic-gate * pages being utilized for BOP_ALLOC()'s before calling 3637c478bd9Sstevel@tonic-gate * page_numtopp().The locking code (ie. page_reclaim()) 3647c478bd9Sstevel@tonic-gate * can't handle them 3657c478bd9Sstevel@tonic-gate */ 3667c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) 3677c478bd9Sstevel@tonic-gate page_boot_demote(pp); 3687c478bd9Sstevel@tonic-gate 3697c478bd9Sstevel@tonic-gate pp = page_numtopp(pfnum, SE_EXCL); 3707c478bd9Sstevel@tonic-gate if (pp == NULL || PP_ISFREE(pp)) 3717c478bd9Sstevel@tonic-gate panic("boot_alloc: pp is NULL or free"); 3727c478bd9Sstevel@tonic-gate 3737c478bd9Sstevel@tonic-gate /* 3747c478bd9Sstevel@tonic-gate * If the cage is on but doesn't yet contain this page, 3757c478bd9Sstevel@tonic-gate * mark it as non-relocatable. 3767c478bd9Sstevel@tonic-gate */ 37700e145c7Skchow if (kcage_on && !PP_ISNORELOC(pp)) { 3787c478bd9Sstevel@tonic-gate PP_SETNORELOC(pp); 37900e145c7Skchow PLCNT_XFER_NORELOC(pp); 38000e145c7Skchow } 3817c478bd9Sstevel@tonic-gate 3827c478bd9Sstevel@tonic-gate (void) page_hashin(pp, &kvp, (u_offset_t)(uintptr_t)addr, NULL); 3837c478bd9Sstevel@tonic-gate pp->p_lckcnt = 1; 3847c478bd9Sstevel@tonic-gate #if defined(__x86) 3857c478bd9Sstevel@tonic-gate page_downgrade(pp); 3867c478bd9Sstevel@tonic-gate #else 3877c478bd9Sstevel@tonic-gate page_unlock(pp); 3887c478bd9Sstevel@tonic-gate #endif 3897c478bd9Sstevel@tonic-gate } 3907c478bd9Sstevel@tonic-gate } 3917c478bd9Sstevel@tonic-gate 3927c478bd9Sstevel@tonic-gate /* 3937c478bd9Sstevel@tonic-gate * Get pages from boot and hash them into the kernel's vp. 3947c478bd9Sstevel@tonic-gate * Used after page structs have been allocated, but before segkmem is ready. 3957c478bd9Sstevel@tonic-gate */ 3967c478bd9Sstevel@tonic-gate void * 3977c478bd9Sstevel@tonic-gate boot_alloc(void *inaddr, size_t size, uint_t align) 3987c478bd9Sstevel@tonic-gate { 3997c478bd9Sstevel@tonic-gate caddr_t addr = inaddr; 4007c478bd9Sstevel@tonic-gate 4017c478bd9Sstevel@tonic-gate if (bootops == NULL) 4027c478bd9Sstevel@tonic-gate prom_panic("boot_alloc: attempt to allocate memory after " 4037c478bd9Sstevel@tonic-gate "BOP_GONE"); 4047c478bd9Sstevel@tonic-gate 4057c478bd9Sstevel@tonic-gate size = ptob(btopr(size)); 406ca622e3aSsvemuri #ifdef __sparc 407ca622e3aSsvemuri if (bop_alloc_chunk(addr, size, align) != (caddr_t)addr) 408ca622e3aSsvemuri panic("boot_alloc: bop_alloc_chunk failed"); 409ca622e3aSsvemuri #else 4107c478bd9Sstevel@tonic-gate if (BOP_ALLOC(bootops, addr, size, align) != addr) 4117c478bd9Sstevel@tonic-gate panic("boot_alloc: BOP_ALLOC failed"); 412ca622e3aSsvemuri #endif 4137c478bd9Sstevel@tonic-gate boot_mapin((caddr_t)addr, size); 4147c478bd9Sstevel@tonic-gate return (addr); 4157c478bd9Sstevel@tonic-gate } 4167c478bd9Sstevel@tonic-gate 4177c478bd9Sstevel@tonic-gate static void 4187c478bd9Sstevel@tonic-gate segkmem_badop() 4197c478bd9Sstevel@tonic-gate { 4207c478bd9Sstevel@tonic-gate panic("segkmem_badop"); 4217c478bd9Sstevel@tonic-gate } 4227c478bd9Sstevel@tonic-gate 4237c478bd9Sstevel@tonic-gate #define SEGKMEM_BADOP(t) (t(*)())segkmem_badop 4247c478bd9Sstevel@tonic-gate 4257c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 4267c478bd9Sstevel@tonic-gate static faultcode_t 4277c478bd9Sstevel@tonic-gate segkmem_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t size, 4287c478bd9Sstevel@tonic-gate enum fault_type type, enum seg_rw rw) 4297c478bd9Sstevel@tonic-gate { 4304fc2445aSelowe pgcnt_t npages; 4314fc2445aSelowe spgcnt_t pg; 4324fc2445aSelowe page_t *pp; 433ad23a2dbSjohansen struct vnode *vp = seg->s_data; 4344fc2445aSelowe 4357c478bd9Sstevel@tonic-gate ASSERT(RW_READ_HELD(&seg->s_as->a_lock)); 4367c478bd9Sstevel@tonic-gate 4377c478bd9Sstevel@tonic-gate if (seg->s_as != &kas || size > seg->s_size || 4387c478bd9Sstevel@tonic-gate addr < seg->s_base || addr + size > seg->s_base + seg->s_size) 4397c478bd9Sstevel@tonic-gate panic("segkmem_fault: bad args"); 4407c478bd9Sstevel@tonic-gate 4417c478bd9Sstevel@tonic-gate /* 4427c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call segkp_fault. 4437c478bd9Sstevel@tonic-gate */ 44435b1ab99Sjosephb if (segkp_bitmap && seg == &kvseg && 44535b1ab99Sjosephb BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base)))) 4467c478bd9Sstevel@tonic-gate return (SEGOP_FAULT(hat, segkp, addr, size, type, rw)); 4477c478bd9Sstevel@tonic-gate 4484fc2445aSelowe if (rw != S_READ && rw != S_WRITE && rw != S_OTHER) 4494fc2445aSelowe return (FC_NOSUPPORT); 4504fc2445aSelowe 4514fc2445aSelowe npages = btopr(size); 4524fc2445aSelowe 4537c478bd9Sstevel@tonic-gate switch (type) { 4547c478bd9Sstevel@tonic-gate case F_SOFTLOCK: /* lock down already-loaded translations */ 4554fc2445aSelowe for (pg = 0; pg < npages; pg++) { 456ad23a2dbSjohansen pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, 4574fc2445aSelowe SE_SHARED); 4584fc2445aSelowe if (pp == NULL) { 4594fc2445aSelowe /* 4604fc2445aSelowe * Hmm, no page. Does a kernel mapping 4614fc2445aSelowe * exist for it? 4624fc2445aSelowe */ 4634fc2445aSelowe if (!hat_probe(kas.a_hat, addr)) { 4644fc2445aSelowe addr -= PAGESIZE; 4654fc2445aSelowe while (--pg >= 0) { 46635b1ab99Sjosephb pp = page_find(vp, (u_offset_t) 46735b1ab99Sjosephb (uintptr_t)addr); 4684fc2445aSelowe if (pp) 4694fc2445aSelowe page_unlock(pp); 4704fc2445aSelowe addr -= PAGESIZE; 4714fc2445aSelowe } 4724fc2445aSelowe return (FC_NOMAP); 4734fc2445aSelowe } 4744fc2445aSelowe } 4754fc2445aSelowe addr += PAGESIZE; 4764fc2445aSelowe } 4774fc2445aSelowe if (rw == S_OTHER) 4787c478bd9Sstevel@tonic-gate hat_reserve(seg->s_as, addr, size); 4797c478bd9Sstevel@tonic-gate return (0); 4807c478bd9Sstevel@tonic-gate case F_SOFTUNLOCK: 4814fc2445aSelowe while (npages--) { 482ad23a2dbSjohansen pp = page_find(vp, (u_offset_t)(uintptr_t)addr); 4834fc2445aSelowe if (pp) 4844fc2445aSelowe page_unlock(pp); 4854fc2445aSelowe addr += PAGESIZE; 4867c478bd9Sstevel@tonic-gate } 4874fc2445aSelowe return (0); 4884fc2445aSelowe default: 4897c478bd9Sstevel@tonic-gate return (FC_NOSUPPORT); 4907c478bd9Sstevel@tonic-gate } 4914fc2445aSelowe /*NOTREACHED*/ 4924fc2445aSelowe } 4937c478bd9Sstevel@tonic-gate 4947c478bd9Sstevel@tonic-gate static int 4957c478bd9Sstevel@tonic-gate segkmem_setprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot) 4967c478bd9Sstevel@tonic-gate { 4977c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); 4987c478bd9Sstevel@tonic-gate 4997c478bd9Sstevel@tonic-gate if (seg->s_as != &kas || size > seg->s_size || 5007c478bd9Sstevel@tonic-gate addr < seg->s_base || addr + size > seg->s_base + seg->s_size) 5017c478bd9Sstevel@tonic-gate panic("segkmem_setprot: bad args"); 5027c478bd9Sstevel@tonic-gate 5037c478bd9Sstevel@tonic-gate /* 5047c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call segkp. 5057c478bd9Sstevel@tonic-gate */ 50635b1ab99Sjosephb if (segkp_bitmap && seg == &kvseg && 50735b1ab99Sjosephb BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base)))) 5087c478bd9Sstevel@tonic-gate return (SEGOP_SETPROT(segkp, addr, size, prot)); 5097c478bd9Sstevel@tonic-gate 5107c478bd9Sstevel@tonic-gate if (prot == 0) 5117c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, addr, size, HAT_UNLOAD); 5127c478bd9Sstevel@tonic-gate else 5137c478bd9Sstevel@tonic-gate hat_chgprot(kas.a_hat, addr, size, prot); 5147c478bd9Sstevel@tonic-gate return (0); 5157c478bd9Sstevel@tonic-gate } 5167c478bd9Sstevel@tonic-gate 5177c478bd9Sstevel@tonic-gate /* 5187c478bd9Sstevel@tonic-gate * This is a dummy segkmem function overloaded to call segkp 5197c478bd9Sstevel@tonic-gate * when segkp is under the heap. 5207c478bd9Sstevel@tonic-gate */ 5217c478bd9Sstevel@tonic-gate /* ARGSUSED */ 5227c478bd9Sstevel@tonic-gate static int 5237c478bd9Sstevel@tonic-gate segkmem_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot) 5247c478bd9Sstevel@tonic-gate { 5257c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); 5267c478bd9Sstevel@tonic-gate 5277c478bd9Sstevel@tonic-gate if (seg->s_as != &kas) 5287c478bd9Sstevel@tonic-gate segkmem_badop(); 5297c478bd9Sstevel@tonic-gate 5307c478bd9Sstevel@tonic-gate /* 5317c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp. 5327c478bd9Sstevel@tonic-gate */ 53335b1ab99Sjosephb if (segkp_bitmap && seg == &kvseg && 53435b1ab99Sjosephb BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base)))) 5357c478bd9Sstevel@tonic-gate return (SEGOP_CHECKPROT(segkp, addr, size, prot)); 53635b1ab99Sjosephb 5377c478bd9Sstevel@tonic-gate segkmem_badop(); 5387c478bd9Sstevel@tonic-gate return (0); 5397c478bd9Sstevel@tonic-gate } 5407c478bd9Sstevel@tonic-gate 5417c478bd9Sstevel@tonic-gate /* 5427c478bd9Sstevel@tonic-gate * This is a dummy segkmem function overloaded to call segkp 5437c478bd9Sstevel@tonic-gate * when segkp is under the heap. 5447c478bd9Sstevel@tonic-gate */ 5457c478bd9Sstevel@tonic-gate /* ARGSUSED */ 5467c478bd9Sstevel@tonic-gate static int 5477c478bd9Sstevel@tonic-gate segkmem_kluster(struct seg *seg, caddr_t addr, ssize_t delta) 5487c478bd9Sstevel@tonic-gate { 5497c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); 5507c478bd9Sstevel@tonic-gate 5517c478bd9Sstevel@tonic-gate if (seg->s_as != &kas) 5527c478bd9Sstevel@tonic-gate segkmem_badop(); 5537c478bd9Sstevel@tonic-gate 5547c478bd9Sstevel@tonic-gate /* 5557c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp. 5567c478bd9Sstevel@tonic-gate */ 55735b1ab99Sjosephb if (segkp_bitmap && seg == &kvseg && 55835b1ab99Sjosephb BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base)))) 5597c478bd9Sstevel@tonic-gate return (SEGOP_KLUSTER(segkp, addr, delta)); 56035b1ab99Sjosephb 5617c478bd9Sstevel@tonic-gate segkmem_badop(); 5627c478bd9Sstevel@tonic-gate return (0); 5637c478bd9Sstevel@tonic-gate } 5647c478bd9Sstevel@tonic-gate 5657c478bd9Sstevel@tonic-gate static void 5667c478bd9Sstevel@tonic-gate segkmem_xdump_range(void *arg, void *start, size_t size) 5677c478bd9Sstevel@tonic-gate { 5687c478bd9Sstevel@tonic-gate struct as *as = arg; 5697c478bd9Sstevel@tonic-gate caddr_t addr = start; 5707c478bd9Sstevel@tonic-gate caddr_t addr_end = addr + size; 5717c478bd9Sstevel@tonic-gate 5727c478bd9Sstevel@tonic-gate while (addr < addr_end) { 5737c478bd9Sstevel@tonic-gate pfn_t pfn = hat_getpfnum(kas.a_hat, addr); 5747c478bd9Sstevel@tonic-gate if (pfn != PFN_INVALID && pfn <= physmax && pf_is_memory(pfn)) 5757c478bd9Sstevel@tonic-gate dump_addpage(as, addr, pfn); 5767c478bd9Sstevel@tonic-gate addr += PAGESIZE; 5777c478bd9Sstevel@tonic-gate dump_timeleft = dump_timeout; 5787c478bd9Sstevel@tonic-gate } 5797c478bd9Sstevel@tonic-gate } 5807c478bd9Sstevel@tonic-gate 5817c478bd9Sstevel@tonic-gate static void 5827c478bd9Sstevel@tonic-gate segkmem_dump_range(void *arg, void *start, size_t size) 5837c478bd9Sstevel@tonic-gate { 5847c478bd9Sstevel@tonic-gate caddr_t addr = start; 5857c478bd9Sstevel@tonic-gate caddr_t addr_end = addr + size; 5867c478bd9Sstevel@tonic-gate 5877c478bd9Sstevel@tonic-gate /* 5887c478bd9Sstevel@tonic-gate * If we are about to start dumping the range of addresses we 5897c478bd9Sstevel@tonic-gate * carved out of the kernel heap for the large page heap walk 5907c478bd9Sstevel@tonic-gate * heap_lp_arena to find what segments are actually populated 5917c478bd9Sstevel@tonic-gate */ 5927c478bd9Sstevel@tonic-gate if (SEGKMEM_USE_LARGEPAGES && 5937c478bd9Sstevel@tonic-gate addr == heap_lp_base && addr_end == heap_lp_end && 5947c478bd9Sstevel@tonic-gate vmem_size(heap_lp_arena, VMEM_ALLOC) < size) { 5957c478bd9Sstevel@tonic-gate vmem_walk(heap_lp_arena, VMEM_ALLOC | VMEM_REENTRANT, 5967c478bd9Sstevel@tonic-gate segkmem_xdump_range, arg); 5977c478bd9Sstevel@tonic-gate } else { 5987c478bd9Sstevel@tonic-gate segkmem_xdump_range(arg, start, size); 5997c478bd9Sstevel@tonic-gate } 6007c478bd9Sstevel@tonic-gate } 6017c478bd9Sstevel@tonic-gate 6027c478bd9Sstevel@tonic-gate static void 6037c478bd9Sstevel@tonic-gate segkmem_dump(struct seg *seg) 6047c478bd9Sstevel@tonic-gate { 6057c478bd9Sstevel@tonic-gate /* 6067c478bd9Sstevel@tonic-gate * The kernel's heap_arena (represented by kvseg) is a very large 6077c478bd9Sstevel@tonic-gate * VA space, most of which is typically unused. To speed up dumping 6087c478bd9Sstevel@tonic-gate * we use vmem_walk() to quickly find the pieces of heap_arena that 6097c478bd9Sstevel@tonic-gate * are actually in use. We do the same for heap32_arena and 6107c478bd9Sstevel@tonic-gate * heap_core. 6117c478bd9Sstevel@tonic-gate * 6127c478bd9Sstevel@tonic-gate * We specify VMEM_REENTRANT to vmem_walk() because dump_addpage() 6137c478bd9Sstevel@tonic-gate * may ultimately need to allocate memory. Reentrant walks are 6147c478bd9Sstevel@tonic-gate * necessarily imperfect snapshots. The kernel heap continues 6157c478bd9Sstevel@tonic-gate * to change during a live crash dump, for example. For a normal 6167c478bd9Sstevel@tonic-gate * crash dump, however, we know that there won't be any other threads 6177c478bd9Sstevel@tonic-gate * messing with the heap. Therefore, at worst, we may fail to dump 6187c478bd9Sstevel@tonic-gate * the pages that get allocated by the act of dumping; but we will 6197c478bd9Sstevel@tonic-gate * always dump every page that was allocated when the walk began. 6207c478bd9Sstevel@tonic-gate * 6217c478bd9Sstevel@tonic-gate * The other segkmem segments are dense (fully populated), so there's 6227c478bd9Sstevel@tonic-gate * no need to use this technique when dumping them. 6237c478bd9Sstevel@tonic-gate * 6247c478bd9Sstevel@tonic-gate * Note: when adding special dump handling for any new sparsely- 6257c478bd9Sstevel@tonic-gate * populated segments, be sure to add similar handling to the ::kgrep 6267c478bd9Sstevel@tonic-gate * code in mdb. 6277c478bd9Sstevel@tonic-gate */ 6287c478bd9Sstevel@tonic-gate if (seg == &kvseg) { 6297c478bd9Sstevel@tonic-gate vmem_walk(heap_arena, VMEM_ALLOC | VMEM_REENTRANT, 6307c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 6317c478bd9Sstevel@tonic-gate #ifndef __sparc 6327c478bd9Sstevel@tonic-gate vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT, 6337c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 6347c478bd9Sstevel@tonic-gate #endif 6357c478bd9Sstevel@tonic-gate } else if (seg == &kvseg_core) { 6367c478bd9Sstevel@tonic-gate vmem_walk(heap_core_arena, VMEM_ALLOC | VMEM_REENTRANT, 6377c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 6387c478bd9Sstevel@tonic-gate } else if (seg == &kvseg32) { 6397c478bd9Sstevel@tonic-gate vmem_walk(heap32_arena, VMEM_ALLOC | VMEM_REENTRANT, 6407c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 6417c478bd9Sstevel@tonic-gate vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT, 6427c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 643ad23a2dbSjohansen } else if (seg == &kzioseg) { 644ad23a2dbSjohansen /* 645ad23a2dbSjohansen * We don't want to dump pages attached to kzioseg since they 646ad23a2dbSjohansen * contain file data from ZFS. If this page's segment is 647ad23a2dbSjohansen * kzioseg return instead of writing it to the dump device. 648ad23a2dbSjohansen */ 649ad23a2dbSjohansen return; 6507c478bd9Sstevel@tonic-gate } else { 6517c478bd9Sstevel@tonic-gate segkmem_dump_range(seg->s_as, seg->s_base, seg->s_size); 6527c478bd9Sstevel@tonic-gate } 6537c478bd9Sstevel@tonic-gate } 6547c478bd9Sstevel@tonic-gate 6557c478bd9Sstevel@tonic-gate /* 6567c478bd9Sstevel@tonic-gate * lock/unlock kmem pages over a given range [addr, addr+len). 6574fc2445aSelowe * Returns a shadow list of pages in ppp. If there are holes 6584fc2445aSelowe * in the range (e.g. some of the kernel mappings do not have 6594fc2445aSelowe * underlying page_ts) returns ENOTSUP so that as_pagelock() 6604fc2445aSelowe * will handle the range via as_fault(F_SOFTLOCK). 6617c478bd9Sstevel@tonic-gate */ 6627c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 6637c478bd9Sstevel@tonic-gate static int 6647c478bd9Sstevel@tonic-gate segkmem_pagelock(struct seg *seg, caddr_t addr, size_t len, 6657c478bd9Sstevel@tonic-gate page_t ***ppp, enum lock_type type, enum seg_rw rw) 6667c478bd9Sstevel@tonic-gate { 6677c478bd9Sstevel@tonic-gate page_t **pplist, *pp; 6687c478bd9Sstevel@tonic-gate pgcnt_t npages; 6694fc2445aSelowe spgcnt_t pg; 6707c478bd9Sstevel@tonic-gate size_t nb; 671ad23a2dbSjohansen struct vnode *vp = seg->s_data; 6727c478bd9Sstevel@tonic-gate 6734fc2445aSelowe ASSERT(ppp != NULL); 6744fc2445aSelowe 6757c478bd9Sstevel@tonic-gate /* 6767c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp. 6777c478bd9Sstevel@tonic-gate */ 67835b1ab99Sjosephb if (segkp_bitmap && seg == &kvseg && 67935b1ab99Sjosephb BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base)))) 68035b1ab99Sjosephb return (SEGOP_PAGELOCK(segkp, addr, len, ppp, type, rw)); 6817c478bd9Sstevel@tonic-gate 6827c478bd9Sstevel@tonic-gate npages = btopr(len); 6837c478bd9Sstevel@tonic-gate nb = sizeof (page_t *) * npages; 6847c478bd9Sstevel@tonic-gate 6857c478bd9Sstevel@tonic-gate if (type == L_PAGEUNLOCK) { 6864fc2445aSelowe pplist = *ppp; 6874fc2445aSelowe ASSERT(pplist != NULL); 6887c478bd9Sstevel@tonic-gate 6894fc2445aSelowe for (pg = 0; pg < npages; pg++) { 6904fc2445aSelowe pp = pplist[pg]; 6917c478bd9Sstevel@tonic-gate page_unlock(pp); 6927c478bd9Sstevel@tonic-gate } 6934fc2445aSelowe kmem_free(pplist, nb); 6947c478bd9Sstevel@tonic-gate return (0); 6957c478bd9Sstevel@tonic-gate } 6967c478bd9Sstevel@tonic-gate 6977c478bd9Sstevel@tonic-gate ASSERT(type == L_PAGELOCK); 6987c478bd9Sstevel@tonic-gate 6994fc2445aSelowe pplist = kmem_alloc(nb, KM_NOSLEEP); 7004fc2445aSelowe if (pplist == NULL) { 7014fc2445aSelowe *ppp = NULL; 7024fc2445aSelowe return (ENOTSUP); /* take the slow path */ 7034fc2445aSelowe } 7047c478bd9Sstevel@tonic-gate 7054fc2445aSelowe for (pg = 0; pg < npages; pg++) { 706ad23a2dbSjohansen pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_SHARED); 7074fc2445aSelowe if (pp == NULL) { 7084fc2445aSelowe while (--pg >= 0) 7094fc2445aSelowe page_unlock(pplist[pg]); 7104fc2445aSelowe kmem_free(pplist, nb); 7114fc2445aSelowe *ppp = NULL; 7124fc2445aSelowe return (ENOTSUP); 7134fc2445aSelowe } 7144fc2445aSelowe pplist[pg] = pp; 7157c478bd9Sstevel@tonic-gate addr += PAGESIZE; 7167c478bd9Sstevel@tonic-gate } 7174fc2445aSelowe 7184fc2445aSelowe *ppp = pplist; 7197c478bd9Sstevel@tonic-gate return (0); 7207c478bd9Sstevel@tonic-gate } 7217c478bd9Sstevel@tonic-gate 7227c478bd9Sstevel@tonic-gate /* 7237c478bd9Sstevel@tonic-gate * This is a dummy segkmem function overloaded to call segkp 7247c478bd9Sstevel@tonic-gate * when segkp is under the heap. 7257c478bd9Sstevel@tonic-gate */ 7267c478bd9Sstevel@tonic-gate /* ARGSUSED */ 7277c478bd9Sstevel@tonic-gate static int 7287c478bd9Sstevel@tonic-gate segkmem_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 7297c478bd9Sstevel@tonic-gate { 7307c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); 7317c478bd9Sstevel@tonic-gate 7327c478bd9Sstevel@tonic-gate if (seg->s_as != &kas) 7337c478bd9Sstevel@tonic-gate segkmem_badop(); 7347c478bd9Sstevel@tonic-gate 7357c478bd9Sstevel@tonic-gate /* 7367c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp. 7377c478bd9Sstevel@tonic-gate */ 73835b1ab99Sjosephb if (segkp_bitmap && seg == &kvseg && 73935b1ab99Sjosephb BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base)))) 7407c478bd9Sstevel@tonic-gate return (SEGOP_GETMEMID(segkp, addr, memidp)); 74135b1ab99Sjosephb 7427c478bd9Sstevel@tonic-gate segkmem_badop(); 7437c478bd9Sstevel@tonic-gate return (0); 7447c478bd9Sstevel@tonic-gate } 7457c478bd9Sstevel@tonic-gate 7467c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 7477c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t * 7487c478bd9Sstevel@tonic-gate segkmem_getpolicy(struct seg *seg, caddr_t addr) 7497c478bd9Sstevel@tonic-gate { 7507c478bd9Sstevel@tonic-gate return (NULL); 7517c478bd9Sstevel@tonic-gate } 7527c478bd9Sstevel@tonic-gate 7531bd5c35fSelowe /*ARGSUSED*/ 7541bd5c35fSelowe static int 7551bd5c35fSelowe segkmem_capable(struct seg *seg, segcapability_t capability) 7561bd5c35fSelowe { 7571bd5c35fSelowe if (capability == S_CAPABILITY_NOMINFLT) 7581bd5c35fSelowe return (1); 7591bd5c35fSelowe return (0); 7601bd5c35fSelowe } 7617c478bd9Sstevel@tonic-gate 7627c478bd9Sstevel@tonic-gate static struct seg_ops segkmem_ops = { 7637c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* dup */ 7647c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* unmap */ 7657c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(void), /* free */ 7667c478bd9Sstevel@tonic-gate segkmem_fault, 7677c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(faultcode_t), /* faulta */ 7687c478bd9Sstevel@tonic-gate segkmem_setprot, 7697c478bd9Sstevel@tonic-gate segkmem_checkprot, 7707c478bd9Sstevel@tonic-gate segkmem_kluster, 7717c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(size_t), /* swapout */ 7727c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* sync */ 7737c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(size_t), /* incore */ 7747c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* lockop */ 7757c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* getprot */ 7767c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(u_offset_t), /* getoffset */ 7777c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* gettype */ 7787c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* getvp */ 7797c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* advise */ 7807c478bd9Sstevel@tonic-gate segkmem_dump, 7817c478bd9Sstevel@tonic-gate segkmem_pagelock, 7827c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* setpgsz */ 7837c478bd9Sstevel@tonic-gate segkmem_getmemid, 7847c478bd9Sstevel@tonic-gate segkmem_getpolicy, /* getpolicy */ 7851bd5c35fSelowe segkmem_capable, /* capable */ 7867c478bd9Sstevel@tonic-gate }; 7877c478bd9Sstevel@tonic-gate 7887c478bd9Sstevel@tonic-gate int 789ad23a2dbSjohansen segkmem_zio_create(struct seg *seg) 790ad23a2dbSjohansen { 791ad23a2dbSjohansen ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock)); 792ad23a2dbSjohansen seg->s_ops = &segkmem_ops; 793ad23a2dbSjohansen seg->s_data = &zvp; 794ad23a2dbSjohansen kas.a_size += seg->s_size; 795ad23a2dbSjohansen return (0); 796ad23a2dbSjohansen } 797ad23a2dbSjohansen 798ad23a2dbSjohansen int 7997c478bd9Sstevel@tonic-gate segkmem_create(struct seg *seg) 8007c478bd9Sstevel@tonic-gate { 8017c478bd9Sstevel@tonic-gate ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock)); 8027c478bd9Sstevel@tonic-gate seg->s_ops = &segkmem_ops; 803ad23a2dbSjohansen seg->s_data = &kvp; 8047c478bd9Sstevel@tonic-gate kas.a_size += seg->s_size; 8057c478bd9Sstevel@tonic-gate return (0); 8067c478bd9Sstevel@tonic-gate } 8077c478bd9Sstevel@tonic-gate 8087c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 8097c478bd9Sstevel@tonic-gate page_t * 8107c478bd9Sstevel@tonic-gate segkmem_page_create(void *addr, size_t size, int vmflag, void *arg) 8117c478bd9Sstevel@tonic-gate { 8127c478bd9Sstevel@tonic-gate struct seg kseg; 8137c478bd9Sstevel@tonic-gate int pgflags; 814ad23a2dbSjohansen struct vnode *vp = arg; 815ad23a2dbSjohansen 816ad23a2dbSjohansen if (vp == NULL) 817ad23a2dbSjohansen vp = &kvp; 8187c478bd9Sstevel@tonic-gate 8197c478bd9Sstevel@tonic-gate kseg.s_as = &kas; 8207c478bd9Sstevel@tonic-gate pgflags = PG_EXCL; 8217c478bd9Sstevel@tonic-gate 8227c478bd9Sstevel@tonic-gate if (segkmem_reloc == 0 || (vmflag & VM_NORELOC)) 8237c478bd9Sstevel@tonic-gate pgflags |= PG_NORELOC; 8247c478bd9Sstevel@tonic-gate if ((vmflag & VM_NOSLEEP) == 0) 8257c478bd9Sstevel@tonic-gate pgflags |= PG_WAIT; 8267c478bd9Sstevel@tonic-gate if (vmflag & VM_PANIC) 8277c478bd9Sstevel@tonic-gate pgflags |= PG_PANIC; 8287c478bd9Sstevel@tonic-gate if (vmflag & VM_PUSHPAGE) 8297c478bd9Sstevel@tonic-gate pgflags |= PG_PUSHPAGE; 8307c478bd9Sstevel@tonic-gate 831ad23a2dbSjohansen return (page_create_va(vp, (u_offset_t)(uintptr_t)addr, size, 8327c478bd9Sstevel@tonic-gate pgflags, &kseg, addr)); 8337c478bd9Sstevel@tonic-gate } 8347c478bd9Sstevel@tonic-gate 8357c478bd9Sstevel@tonic-gate /* 8367c478bd9Sstevel@tonic-gate * Allocate pages to back the virtual address range [addr, addr + size). 8377c478bd9Sstevel@tonic-gate * If addr is NULL, allocate the virtual address space as well. 8387c478bd9Sstevel@tonic-gate */ 8397c478bd9Sstevel@tonic-gate void * 8407c478bd9Sstevel@tonic-gate segkmem_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag, uint_t attr, 8417c478bd9Sstevel@tonic-gate page_t *(*page_create_func)(void *, size_t, int, void *), void *pcarg) 8427c478bd9Sstevel@tonic-gate { 8437c478bd9Sstevel@tonic-gate page_t *ppl; 8447c478bd9Sstevel@tonic-gate caddr_t addr = inaddr; 8457c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size); 8467c478bd9Sstevel@tonic-gate int allocflag; 8477c478bd9Sstevel@tonic-gate 8487c478bd9Sstevel@tonic-gate if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL) 8497c478bd9Sstevel@tonic-gate return (NULL); 8507c478bd9Sstevel@tonic-gate 8517c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0); 8527c478bd9Sstevel@tonic-gate 8537c478bd9Sstevel@tonic-gate if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) { 8547c478bd9Sstevel@tonic-gate if (inaddr == NULL) 8557c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size); 8567c478bd9Sstevel@tonic-gate return (NULL); 8577c478bd9Sstevel@tonic-gate } 8587c478bd9Sstevel@tonic-gate 8597c478bd9Sstevel@tonic-gate ppl = page_create_func(addr, size, vmflag, pcarg); 8607c478bd9Sstevel@tonic-gate if (ppl == NULL) { 8617c478bd9Sstevel@tonic-gate if (inaddr == NULL) 8627c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size); 8637c478bd9Sstevel@tonic-gate page_unresv(npages); 8647c478bd9Sstevel@tonic-gate return (NULL); 8657c478bd9Sstevel@tonic-gate } 8667c478bd9Sstevel@tonic-gate 8677c478bd9Sstevel@tonic-gate /* 8687c478bd9Sstevel@tonic-gate * Under certain conditions, we need to let the HAT layer know 8697c478bd9Sstevel@tonic-gate * that it cannot safely allocate memory. Allocations from 8707c478bd9Sstevel@tonic-gate * the hat_memload vmem arena always need this, to prevent 8717c478bd9Sstevel@tonic-gate * infinite recursion. 8727c478bd9Sstevel@tonic-gate * 8737c478bd9Sstevel@tonic-gate * In addition, the x86 hat cannot safely do memory 8747c478bd9Sstevel@tonic-gate * allocations while in vmem_populate(), because there 8757c478bd9Sstevel@tonic-gate * is no simple bound on its usage. 8767c478bd9Sstevel@tonic-gate */ 8777c478bd9Sstevel@tonic-gate if (vmflag & VM_MEMLOAD) 8787c478bd9Sstevel@tonic-gate allocflag = HAT_NO_KALLOC; 8797c478bd9Sstevel@tonic-gate #if defined(__x86) 8807c478bd9Sstevel@tonic-gate else if (vmem_is_populator()) 8817c478bd9Sstevel@tonic-gate allocflag = HAT_NO_KALLOC; 8827c478bd9Sstevel@tonic-gate #endif 8837c478bd9Sstevel@tonic-gate else 8847c478bd9Sstevel@tonic-gate allocflag = 0; 8857c478bd9Sstevel@tonic-gate 8867c478bd9Sstevel@tonic-gate while (ppl != NULL) { 8877c478bd9Sstevel@tonic-gate page_t *pp = ppl; 8887c478bd9Sstevel@tonic-gate page_sub(&ppl, pp); 8897c478bd9Sstevel@tonic-gate ASSERT(page_iolock_assert(pp)); 8907c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 8917c478bd9Sstevel@tonic-gate page_io_unlock(pp); 8927c478bd9Sstevel@tonic-gate hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset, pp, 8937c478bd9Sstevel@tonic-gate (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr, 8947c478bd9Sstevel@tonic-gate HAT_LOAD_LOCK | allocflag); 8957c478bd9Sstevel@tonic-gate pp->p_lckcnt = 1; 8967c478bd9Sstevel@tonic-gate #if defined(__x86) 8977c478bd9Sstevel@tonic-gate page_downgrade(pp); 8987c478bd9Sstevel@tonic-gate #else 8997c478bd9Sstevel@tonic-gate if (vmflag & SEGKMEM_SHARELOCKED) 9007c478bd9Sstevel@tonic-gate page_downgrade(pp); 9017c478bd9Sstevel@tonic-gate else 9027c478bd9Sstevel@tonic-gate page_unlock(pp); 9037c478bd9Sstevel@tonic-gate #endif 9047c478bd9Sstevel@tonic-gate } 9057c478bd9Sstevel@tonic-gate 9067c478bd9Sstevel@tonic-gate return (addr); 9077c478bd9Sstevel@tonic-gate } 9087c478bd9Sstevel@tonic-gate 909ad23a2dbSjohansen static void * 910ad23a2dbSjohansen segkmem_alloc_vn(vmem_t *vmp, size_t size, int vmflag, struct vnode *vp) 9117c478bd9Sstevel@tonic-gate { 9127c478bd9Sstevel@tonic-gate void *addr; 9137c478bd9Sstevel@tonic-gate segkmem_gc_list_t *gcp, **prev_gcpp; 9147c478bd9Sstevel@tonic-gate 915ad23a2dbSjohansen ASSERT(vp != NULL); 916ad23a2dbSjohansen 9177c478bd9Sstevel@tonic-gate if (kvseg.s_base == NULL) { 9187c478bd9Sstevel@tonic-gate #ifndef __sparc 9197c478bd9Sstevel@tonic-gate if (bootops->bsys_alloc == NULL) 9207c478bd9Sstevel@tonic-gate halt("Memory allocation between bop_alloc() and " 9217c478bd9Sstevel@tonic-gate "kmem_alloc().\n"); 9227c478bd9Sstevel@tonic-gate #endif 9237c478bd9Sstevel@tonic-gate 9247c478bd9Sstevel@tonic-gate /* 9257c478bd9Sstevel@tonic-gate * There's not a lot of memory to go around during boot, 9267c478bd9Sstevel@tonic-gate * so recycle it if we can. 9277c478bd9Sstevel@tonic-gate */ 9287c478bd9Sstevel@tonic-gate for (prev_gcpp = &segkmem_gc_list; (gcp = *prev_gcpp) != NULL; 9297c478bd9Sstevel@tonic-gate prev_gcpp = &gcp->gc_next) { 9307c478bd9Sstevel@tonic-gate if (gcp->gc_arena == vmp && gcp->gc_size == size) { 9317c478bd9Sstevel@tonic-gate *prev_gcpp = gcp->gc_next; 9327c478bd9Sstevel@tonic-gate return (gcp); 9337c478bd9Sstevel@tonic-gate } 9347c478bd9Sstevel@tonic-gate } 9357c478bd9Sstevel@tonic-gate 9367c478bd9Sstevel@tonic-gate addr = vmem_alloc(vmp, size, vmflag | VM_PANIC); 9377c478bd9Sstevel@tonic-gate if (boot_alloc(addr, size, BO_NO_ALIGN) != addr) 9387c478bd9Sstevel@tonic-gate panic("segkmem_alloc: boot_alloc failed"); 9397c478bd9Sstevel@tonic-gate return (addr); 9407c478bd9Sstevel@tonic-gate } 9417c478bd9Sstevel@tonic-gate return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, 942ad23a2dbSjohansen segkmem_page_create, vp)); 943ad23a2dbSjohansen } 944ad23a2dbSjohansen 945ad23a2dbSjohansen void * 946ad23a2dbSjohansen segkmem_alloc(vmem_t *vmp, size_t size, int vmflag) 947ad23a2dbSjohansen { 948ad23a2dbSjohansen return (segkmem_alloc_vn(vmp, size, vmflag, &kvp)); 949ad23a2dbSjohansen } 950ad23a2dbSjohansen 951ad23a2dbSjohansen void * 952ad23a2dbSjohansen segkmem_zio_alloc(vmem_t *vmp, size_t size, int vmflag) 953ad23a2dbSjohansen { 954ad23a2dbSjohansen return (segkmem_alloc_vn(vmp, size, vmflag, &zvp)); 9557c478bd9Sstevel@tonic-gate } 9567c478bd9Sstevel@tonic-gate 9577c478bd9Sstevel@tonic-gate /* 9587c478bd9Sstevel@tonic-gate * Any changes to this routine must also be carried over to 9597c478bd9Sstevel@tonic-gate * devmap_free_pages() in the seg_dev driver. This is because 9607c478bd9Sstevel@tonic-gate * we currently don't have a special kernel segment for non-paged 9617c478bd9Sstevel@tonic-gate * kernel memory that is exported by drivers to user space. 9627c478bd9Sstevel@tonic-gate */ 963ad23a2dbSjohansen static void 964843e1988Sjohnlev segkmem_free_vn(vmem_t *vmp, void *inaddr, size_t size, struct vnode *vp, 965843e1988Sjohnlev void (*func)(page_t *)) 9667c478bd9Sstevel@tonic-gate { 9677c478bd9Sstevel@tonic-gate page_t *pp; 9687c478bd9Sstevel@tonic-gate caddr_t addr = inaddr; 9697c478bd9Sstevel@tonic-gate caddr_t eaddr; 9707c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size); 9717c478bd9Sstevel@tonic-gate 9727c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0); 973ad23a2dbSjohansen ASSERT(vp != NULL); 9747c478bd9Sstevel@tonic-gate 9757c478bd9Sstevel@tonic-gate if (kvseg.s_base == NULL) { 9767c478bd9Sstevel@tonic-gate segkmem_gc_list_t *gc = inaddr; 9777c478bd9Sstevel@tonic-gate gc->gc_arena = vmp; 9787c478bd9Sstevel@tonic-gate gc->gc_size = size; 9797c478bd9Sstevel@tonic-gate gc->gc_next = segkmem_gc_list; 9807c478bd9Sstevel@tonic-gate segkmem_gc_list = gc; 9817c478bd9Sstevel@tonic-gate return; 9827c478bd9Sstevel@tonic-gate } 9837c478bd9Sstevel@tonic-gate 9847c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK); 9857c478bd9Sstevel@tonic-gate 9867c478bd9Sstevel@tonic-gate for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) { 9877c478bd9Sstevel@tonic-gate #if defined(__x86) 988ad23a2dbSjohansen pp = page_find(vp, (u_offset_t)(uintptr_t)addr); 9897c478bd9Sstevel@tonic-gate if (pp == NULL) 9907c478bd9Sstevel@tonic-gate panic("segkmem_free: page not found"); 9917c478bd9Sstevel@tonic-gate if (!page_tryupgrade(pp)) { 9927c478bd9Sstevel@tonic-gate /* 9937c478bd9Sstevel@tonic-gate * Some other thread has a sharelock. Wait for 9947c478bd9Sstevel@tonic-gate * it to drop the lock so we can free this page. 9957c478bd9Sstevel@tonic-gate */ 9967c478bd9Sstevel@tonic-gate page_unlock(pp); 997ad23a2dbSjohansen pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, 9987c478bd9Sstevel@tonic-gate SE_EXCL); 9997c478bd9Sstevel@tonic-gate } 10007c478bd9Sstevel@tonic-gate #else 1001ad23a2dbSjohansen pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_EXCL); 10027c478bd9Sstevel@tonic-gate #endif 10037c478bd9Sstevel@tonic-gate if (pp == NULL) 10047c478bd9Sstevel@tonic-gate panic("segkmem_free: page not found"); 10057c478bd9Sstevel@tonic-gate /* Clear p_lckcnt so page_destroy() doesn't update availrmem */ 10067c478bd9Sstevel@tonic-gate pp->p_lckcnt = 0; 1007843e1988Sjohnlev if (func) 1008843e1988Sjohnlev func(pp); 1009843e1988Sjohnlev else 10107c478bd9Sstevel@tonic-gate page_destroy(pp, 0); 10117c478bd9Sstevel@tonic-gate } 1012843e1988Sjohnlev if (func == NULL) 10137c478bd9Sstevel@tonic-gate page_unresv(npages); 10147c478bd9Sstevel@tonic-gate 10157c478bd9Sstevel@tonic-gate if (vmp != NULL) 10167c478bd9Sstevel@tonic-gate vmem_free(vmp, inaddr, size); 1017ad23a2dbSjohansen 1018ad23a2dbSjohansen } 1019ad23a2dbSjohansen 1020ad23a2dbSjohansen void 1021843e1988Sjohnlev segkmem_xfree(vmem_t *vmp, void *inaddr, size_t size, void (*func)(page_t *)) 1022843e1988Sjohnlev { 1023843e1988Sjohnlev segkmem_free_vn(vmp, inaddr, size, &kvp, func); 1024843e1988Sjohnlev } 1025843e1988Sjohnlev 1026843e1988Sjohnlev void 1027ad23a2dbSjohansen segkmem_free(vmem_t *vmp, void *inaddr, size_t size) 1028ad23a2dbSjohansen { 1029843e1988Sjohnlev segkmem_free_vn(vmp, inaddr, size, &kvp, NULL); 1030ad23a2dbSjohansen } 1031ad23a2dbSjohansen 1032ad23a2dbSjohansen void 1033ad23a2dbSjohansen segkmem_zio_free(vmem_t *vmp, void *inaddr, size_t size) 1034ad23a2dbSjohansen { 1035843e1988Sjohnlev segkmem_free_vn(vmp, inaddr, size, &zvp, NULL); 10367c478bd9Sstevel@tonic-gate } 10377c478bd9Sstevel@tonic-gate 10387c478bd9Sstevel@tonic-gate void 10397c478bd9Sstevel@tonic-gate segkmem_gc(void) 10407c478bd9Sstevel@tonic-gate { 10417c478bd9Sstevel@tonic-gate ASSERT(kvseg.s_base != NULL); 10427c478bd9Sstevel@tonic-gate while (segkmem_gc_list != NULL) { 10437c478bd9Sstevel@tonic-gate segkmem_gc_list_t *gc = segkmem_gc_list; 10447c478bd9Sstevel@tonic-gate segkmem_gc_list = gc->gc_next; 10457c478bd9Sstevel@tonic-gate segkmem_free(gc->gc_arena, gc, gc->gc_size); 10467c478bd9Sstevel@tonic-gate } 10477c478bd9Sstevel@tonic-gate } 10487c478bd9Sstevel@tonic-gate 10497c478bd9Sstevel@tonic-gate /* 10507c478bd9Sstevel@tonic-gate * Legacy entry points from here to end of file. 10517c478bd9Sstevel@tonic-gate */ 10527c478bd9Sstevel@tonic-gate void 10537c478bd9Sstevel@tonic-gate segkmem_mapin(struct seg *seg, void *addr, size_t size, uint_t vprot, 10547c478bd9Sstevel@tonic-gate pfn_t pfn, uint_t flags) 10557c478bd9Sstevel@tonic-gate { 10567c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK); 10577c478bd9Sstevel@tonic-gate hat_devload(seg->s_as->a_hat, addr, size, pfn, vprot, 10587c478bd9Sstevel@tonic-gate flags | HAT_LOAD_LOCK); 10597c478bd9Sstevel@tonic-gate } 10607c478bd9Sstevel@tonic-gate 10617c478bd9Sstevel@tonic-gate void 10627c478bd9Sstevel@tonic-gate segkmem_mapout(struct seg *seg, void *addr, size_t size) 10637c478bd9Sstevel@tonic-gate { 10647c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK); 10657c478bd9Sstevel@tonic-gate } 10667c478bd9Sstevel@tonic-gate 10677c478bd9Sstevel@tonic-gate void * 10687c478bd9Sstevel@tonic-gate kmem_getpages(pgcnt_t npages, int kmflag) 10697c478bd9Sstevel@tonic-gate { 10707c478bd9Sstevel@tonic-gate return (kmem_alloc(ptob(npages), kmflag)); 10717c478bd9Sstevel@tonic-gate } 10727c478bd9Sstevel@tonic-gate 10737c478bd9Sstevel@tonic-gate void 10747c478bd9Sstevel@tonic-gate kmem_freepages(void *addr, pgcnt_t npages) 10757c478bd9Sstevel@tonic-gate { 10767c478bd9Sstevel@tonic-gate kmem_free(addr, ptob(npages)); 10777c478bd9Sstevel@tonic-gate } 10787c478bd9Sstevel@tonic-gate 10797c478bd9Sstevel@tonic-gate /* 10807c478bd9Sstevel@tonic-gate * segkmem_page_create_large() allocates a large page to be used for the kmem 10817c478bd9Sstevel@tonic-gate * caches. If kpr is enabled we ask for a relocatable page unless requested 10827c478bd9Sstevel@tonic-gate * otherwise. If kpr is disabled we have to ask for a non-reloc page 10837c478bd9Sstevel@tonic-gate */ 10847c478bd9Sstevel@tonic-gate static page_t * 10857c478bd9Sstevel@tonic-gate segkmem_page_create_large(void *addr, size_t size, int vmflag, void *arg) 10867c478bd9Sstevel@tonic-gate { 10877c478bd9Sstevel@tonic-gate int pgflags; 10887c478bd9Sstevel@tonic-gate 10897c478bd9Sstevel@tonic-gate pgflags = PG_EXCL; 10907c478bd9Sstevel@tonic-gate 10917c478bd9Sstevel@tonic-gate if (segkmem_reloc == 0 || (vmflag & VM_NORELOC)) 10927c478bd9Sstevel@tonic-gate pgflags |= PG_NORELOC; 10937c478bd9Sstevel@tonic-gate if (!(vmflag & VM_NOSLEEP)) 10947c478bd9Sstevel@tonic-gate pgflags |= PG_WAIT; 10957c478bd9Sstevel@tonic-gate if (vmflag & VM_PUSHPAGE) 10967c478bd9Sstevel@tonic-gate pgflags |= PG_PUSHPAGE; 10977c478bd9Sstevel@tonic-gate 10987c478bd9Sstevel@tonic-gate return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size, 10997c478bd9Sstevel@tonic-gate pgflags, &kvseg, addr, arg)); 11007c478bd9Sstevel@tonic-gate } 11017c478bd9Sstevel@tonic-gate 11027c478bd9Sstevel@tonic-gate /* 11037c478bd9Sstevel@tonic-gate * Allocate a large page to back the virtual address range 11047c478bd9Sstevel@tonic-gate * [addr, addr + size). If addr is NULL, allocate the virtual address 11057c478bd9Sstevel@tonic-gate * space as well. 11067c478bd9Sstevel@tonic-gate */ 11077c478bd9Sstevel@tonic-gate static void * 11087c478bd9Sstevel@tonic-gate segkmem_xalloc_lp(vmem_t *vmp, void *inaddr, size_t size, int vmflag, 11097c478bd9Sstevel@tonic-gate uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *), 11107c478bd9Sstevel@tonic-gate void *pcarg) 11117c478bd9Sstevel@tonic-gate { 11127c478bd9Sstevel@tonic-gate caddr_t addr = inaddr, pa; 11137c478bd9Sstevel@tonic-gate size_t lpsize = segkmem_lpsize; 11147c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size); 11157c478bd9Sstevel@tonic-gate pgcnt_t nbpages = btop(lpsize); 11167c478bd9Sstevel@tonic-gate pgcnt_t nlpages = size >> segkmem_lpshift; 11177c478bd9Sstevel@tonic-gate size_t ppasize = nbpages * sizeof (page_t *); 11187c478bd9Sstevel@tonic-gate page_t *pp, *rootpp, **ppa, *pplist = NULL; 11197c478bd9Sstevel@tonic-gate int i; 11207c478bd9Sstevel@tonic-gate 1121423908e1Sdp78419 vmflag |= VM_NOSLEEP; 1122423908e1Sdp78419 11237c478bd9Sstevel@tonic-gate if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) { 11247c478bd9Sstevel@tonic-gate return (NULL); 11257c478bd9Sstevel@tonic-gate } 11267c478bd9Sstevel@tonic-gate 11277c478bd9Sstevel@tonic-gate /* 11287c478bd9Sstevel@tonic-gate * allocate an array we need for hat_memload_array. 11297c478bd9Sstevel@tonic-gate * we use a separate arena to avoid recursion. 11307c478bd9Sstevel@tonic-gate * we will not need this array when hat_memload_array learns pp++ 11317c478bd9Sstevel@tonic-gate */ 11327c478bd9Sstevel@tonic-gate if ((ppa = vmem_alloc(segkmem_ppa_arena, ppasize, vmflag)) == NULL) { 11337c478bd9Sstevel@tonic-gate goto fail_array_alloc; 11347c478bd9Sstevel@tonic-gate } 11357c478bd9Sstevel@tonic-gate 11367c478bd9Sstevel@tonic-gate if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL) 11377c478bd9Sstevel@tonic-gate goto fail_vmem_alloc; 11387c478bd9Sstevel@tonic-gate 11397c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & (lpsize - 1)) == 0); 11407c478bd9Sstevel@tonic-gate 11417c478bd9Sstevel@tonic-gate /* create all the pages */ 11427c478bd9Sstevel@tonic-gate for (pa = addr, i = 0; i < nlpages; i++, pa += lpsize) { 11437c478bd9Sstevel@tonic-gate if ((pp = page_create_func(pa, lpsize, vmflag, pcarg)) == NULL) 11447c478bd9Sstevel@tonic-gate goto fail_page_create; 11457c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &pp); 11467c478bd9Sstevel@tonic-gate } 11477c478bd9Sstevel@tonic-gate 11487c478bd9Sstevel@tonic-gate /* at this point we have all the resource to complete the request */ 11497c478bd9Sstevel@tonic-gate while ((rootpp = pplist) != NULL) { 11507c478bd9Sstevel@tonic-gate for (i = 0; i < nbpages; i++) { 11517c478bd9Sstevel@tonic-gate ASSERT(pplist != NULL); 11527c478bd9Sstevel@tonic-gate pp = pplist; 11537c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 11547c478bd9Sstevel@tonic-gate ASSERT(page_iolock_assert(pp)); 11557c478bd9Sstevel@tonic-gate page_io_unlock(pp); 11567c478bd9Sstevel@tonic-gate ppa[i] = pp; 11577c478bd9Sstevel@tonic-gate } 11587c478bd9Sstevel@tonic-gate /* 11597c478bd9Sstevel@tonic-gate * Load the locked entry. It's OK to preload the entry into the 11607c478bd9Sstevel@tonic-gate * TSB since we now support large mappings in the kernel TSB. 11617c478bd9Sstevel@tonic-gate */ 11627c478bd9Sstevel@tonic-gate hat_memload_array(kas.a_hat, 11637c478bd9Sstevel@tonic-gate (caddr_t)(uintptr_t)rootpp->p_offset, lpsize, 11647c478bd9Sstevel@tonic-gate ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr, 11657c478bd9Sstevel@tonic-gate HAT_LOAD_LOCK); 11667c478bd9Sstevel@tonic-gate 11677c478bd9Sstevel@tonic-gate for (--i; i >= 0; --i) { 11687c478bd9Sstevel@tonic-gate ppa[i]->p_lckcnt = 1; 11697c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 11707c478bd9Sstevel@tonic-gate } 11717c478bd9Sstevel@tonic-gate } 11727c478bd9Sstevel@tonic-gate 11737c478bd9Sstevel@tonic-gate vmem_free(segkmem_ppa_arena, ppa, ppasize); 11747c478bd9Sstevel@tonic-gate return (addr); 11757c478bd9Sstevel@tonic-gate 11767c478bd9Sstevel@tonic-gate fail_page_create: 11777c478bd9Sstevel@tonic-gate while ((rootpp = pplist) != NULL) { 11787c478bd9Sstevel@tonic-gate for (i = 0, pp = pplist; i < nbpages; i++, pp = pplist) { 11797c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 11807c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 11817c478bd9Sstevel@tonic-gate ASSERT(page_iolock_assert(pp)); 11827c478bd9Sstevel@tonic-gate page_io_unlock(pp); 11837c478bd9Sstevel@tonic-gate } 11847c478bd9Sstevel@tonic-gate page_destroy_pages(rootpp); 11857c478bd9Sstevel@tonic-gate } 11867c478bd9Sstevel@tonic-gate 11877c478bd9Sstevel@tonic-gate if (inaddr == NULL) 11887c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size); 11897c478bd9Sstevel@tonic-gate 11907c478bd9Sstevel@tonic-gate fail_vmem_alloc: 11917c478bd9Sstevel@tonic-gate vmem_free(segkmem_ppa_arena, ppa, ppasize); 11927c478bd9Sstevel@tonic-gate 11937c478bd9Sstevel@tonic-gate fail_array_alloc: 11947c478bd9Sstevel@tonic-gate page_unresv(npages); 11957c478bd9Sstevel@tonic-gate 11967c478bd9Sstevel@tonic-gate return (NULL); 11977c478bd9Sstevel@tonic-gate } 11987c478bd9Sstevel@tonic-gate 11997c478bd9Sstevel@tonic-gate static void 12007c478bd9Sstevel@tonic-gate segkmem_free_one_lp(caddr_t addr, size_t size) 12017c478bd9Sstevel@tonic-gate { 12027c478bd9Sstevel@tonic-gate page_t *pp, *rootpp = NULL; 12037c478bd9Sstevel@tonic-gate pgcnt_t pgs_left = btopr(size); 12047c478bd9Sstevel@tonic-gate 12057c478bd9Sstevel@tonic-gate ASSERT(size == segkmem_lpsize); 12067c478bd9Sstevel@tonic-gate 12077c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK); 12087c478bd9Sstevel@tonic-gate 12097c478bd9Sstevel@tonic-gate for (; pgs_left > 0; addr += PAGESIZE, pgs_left--) { 12107c478bd9Sstevel@tonic-gate pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL); 12117c478bd9Sstevel@tonic-gate if (pp == NULL) 12127c478bd9Sstevel@tonic-gate panic("segkmem_free_one_lp: page not found"); 12137c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 12147c478bd9Sstevel@tonic-gate pp->p_lckcnt = 0; 12157c478bd9Sstevel@tonic-gate if (rootpp == NULL) 12167c478bd9Sstevel@tonic-gate rootpp = pp; 12177c478bd9Sstevel@tonic-gate } 12187c478bd9Sstevel@tonic-gate ASSERT(rootpp != NULL); 12197c478bd9Sstevel@tonic-gate page_destroy_pages(rootpp); 12207c478bd9Sstevel@tonic-gate 12217c478bd9Sstevel@tonic-gate /* page_unresv() is done by the caller */ 12227c478bd9Sstevel@tonic-gate } 12237c478bd9Sstevel@tonic-gate 12247c478bd9Sstevel@tonic-gate /* 12257c478bd9Sstevel@tonic-gate * This function is called to import new spans into the vmem arenas like 12267c478bd9Sstevel@tonic-gate * kmem_default_arena and kmem_oversize_arena. It first tries to import 12277c478bd9Sstevel@tonic-gate * spans from large page arena - kmem_lp_arena. In order to do this it might 12287c478bd9Sstevel@tonic-gate * have to "upgrade the requested size" to kmem_lp_arena quantum. If 12297c478bd9Sstevel@tonic-gate * it was not able to satisfy the upgraded request it then calls regular 12307c478bd9Sstevel@tonic-gate * segkmem_alloc() that satisfies the request by importing from "*vmp" arena 12317c478bd9Sstevel@tonic-gate */ 1232aaa10e67Sha137994 /*ARGSUSED*/ 12337c478bd9Sstevel@tonic-gate void * 1234aaa10e67Sha137994 segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, size_t align, int vmflag) 12357c478bd9Sstevel@tonic-gate { 12367c478bd9Sstevel@tonic-gate size_t size; 12377c478bd9Sstevel@tonic-gate kthread_t *t = curthread; 12387c478bd9Sstevel@tonic-gate segkmem_lpcb_t *lpcb = &segkmem_lpcb; 12397c478bd9Sstevel@tonic-gate 12407c478bd9Sstevel@tonic-gate ASSERT(sizep != NULL); 12417c478bd9Sstevel@tonic-gate 12427c478bd9Sstevel@tonic-gate size = *sizep; 12437c478bd9Sstevel@tonic-gate 12447c478bd9Sstevel@tonic-gate if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) && 12457c478bd9Sstevel@tonic-gate !(vmflag & SEGKMEM_SHARELOCKED)) { 12467c478bd9Sstevel@tonic-gate 12477c478bd9Sstevel@tonic-gate size_t kmemlp_qnt = segkmem_kmemlp_quantum; 12487c478bd9Sstevel@tonic-gate size_t asize = P2ROUNDUP(size, kmemlp_qnt); 12497c478bd9Sstevel@tonic-gate void *addr = NULL; 12507c478bd9Sstevel@tonic-gate ulong_t *lpthrtp = &lpcb->lp_throttle; 12517c478bd9Sstevel@tonic-gate ulong_t lpthrt = *lpthrtp; 12527c478bd9Sstevel@tonic-gate int dowakeup = 0; 12537c478bd9Sstevel@tonic-gate int doalloc = 1; 12547c478bd9Sstevel@tonic-gate 12557c478bd9Sstevel@tonic-gate ASSERT(kmem_lp_arena != NULL); 12567c478bd9Sstevel@tonic-gate ASSERT(asize >= size); 12577c478bd9Sstevel@tonic-gate 12587c478bd9Sstevel@tonic-gate if (lpthrt != 0) { 12597c478bd9Sstevel@tonic-gate /* try to update the throttle value */ 12607c478bd9Sstevel@tonic-gate lpthrt = atomic_add_long_nv(lpthrtp, 1); 12617c478bd9Sstevel@tonic-gate if (lpthrt >= segkmem_lpthrottle_max) { 12627c478bd9Sstevel@tonic-gate lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 12637c478bd9Sstevel@tonic-gate segkmem_lpthrottle_max / 4); 12647c478bd9Sstevel@tonic-gate } 12657c478bd9Sstevel@tonic-gate 12667c478bd9Sstevel@tonic-gate /* 12677c478bd9Sstevel@tonic-gate * when we get above throttle start do an exponential 12687c478bd9Sstevel@tonic-gate * backoff at trying large pages and reaping 12697c478bd9Sstevel@tonic-gate */ 12707c478bd9Sstevel@tonic-gate if (lpthrt > segkmem_lpthrottle_start && 12717c478bd9Sstevel@tonic-gate (lpthrt & (lpthrt - 1))) { 1272501f3284Seg155566 lpcb->allocs_throttled++; 12737c478bd9Sstevel@tonic-gate lpthrt--; 12747c478bd9Sstevel@tonic-gate if ((lpthrt & (lpthrt - 1)) == 0) 12757c478bd9Sstevel@tonic-gate kmem_reap(); 12767c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, vmflag)); 12777c478bd9Sstevel@tonic-gate } 12787c478bd9Sstevel@tonic-gate } 12797c478bd9Sstevel@tonic-gate 12807c478bd9Sstevel@tonic-gate if (!(vmflag & VM_NOSLEEP) && 12817c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum >= (8 * kmemlp_qnt) && 12827c478bd9Sstevel@tonic-gate vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt && 12837c478bd9Sstevel@tonic-gate asize < (segkmem_heaplp_quantum - kmemlp_qnt)) { 12847c478bd9Sstevel@tonic-gate 12857c478bd9Sstevel@tonic-gate /* 12867c478bd9Sstevel@tonic-gate * we are low on free memory in kmem_lp_arena 12877c478bd9Sstevel@tonic-gate * we let only one guy to allocate heap_lp 12887c478bd9Sstevel@tonic-gate * quantum size chunk that everybody is going to 12897c478bd9Sstevel@tonic-gate * share 12907c478bd9Sstevel@tonic-gate */ 12917c478bd9Sstevel@tonic-gate mutex_enter(&lpcb->lp_lock); 12927c478bd9Sstevel@tonic-gate 12937c478bd9Sstevel@tonic-gate if (lpcb->lp_wait) { 12947c478bd9Sstevel@tonic-gate 12957c478bd9Sstevel@tonic-gate /* we are not the first one - wait */ 12967c478bd9Sstevel@tonic-gate cv_wait(&lpcb->lp_cv, &lpcb->lp_lock); 12977c478bd9Sstevel@tonic-gate if (vmem_size(kmem_lp_arena, VMEM_FREE) < 12987c478bd9Sstevel@tonic-gate kmemlp_qnt) { 12997c478bd9Sstevel@tonic-gate doalloc = 0; 13007c478bd9Sstevel@tonic-gate } 13017c478bd9Sstevel@tonic-gate } else if (vmem_size(kmem_lp_arena, VMEM_FREE) <= 13027c478bd9Sstevel@tonic-gate kmemlp_qnt) { 13037c478bd9Sstevel@tonic-gate 13047c478bd9Sstevel@tonic-gate /* 13057c478bd9Sstevel@tonic-gate * we are the first one, make sure we import 13067c478bd9Sstevel@tonic-gate * a large page 13077c478bd9Sstevel@tonic-gate */ 13087c478bd9Sstevel@tonic-gate if (asize == kmemlp_qnt) 13097c478bd9Sstevel@tonic-gate asize += kmemlp_qnt; 13107c478bd9Sstevel@tonic-gate dowakeup = 1; 13117c478bd9Sstevel@tonic-gate lpcb->lp_wait = 1; 13127c478bd9Sstevel@tonic-gate } 13137c478bd9Sstevel@tonic-gate 13147c478bd9Sstevel@tonic-gate mutex_exit(&lpcb->lp_lock); 13157c478bd9Sstevel@tonic-gate } 13167c478bd9Sstevel@tonic-gate 13177c478bd9Sstevel@tonic-gate /* 13187c478bd9Sstevel@tonic-gate * VM_ABORT flag prevents sleeps in vmem_xalloc when 13197c478bd9Sstevel@tonic-gate * large pages are not available. In that case this allocation 13207c478bd9Sstevel@tonic-gate * attempt will fail and we will retry allocation with small 13217c478bd9Sstevel@tonic-gate * pages. We also do not want to panic if this allocation fails 13227c478bd9Sstevel@tonic-gate * because we are going to retry. 13237c478bd9Sstevel@tonic-gate */ 13247c478bd9Sstevel@tonic-gate if (doalloc) { 13257c478bd9Sstevel@tonic-gate addr = vmem_alloc(kmem_lp_arena, asize, 13267c478bd9Sstevel@tonic-gate (vmflag | VM_ABORT) & ~VM_PANIC); 13277c478bd9Sstevel@tonic-gate 13287c478bd9Sstevel@tonic-gate if (dowakeup) { 13297c478bd9Sstevel@tonic-gate mutex_enter(&lpcb->lp_lock); 13307c478bd9Sstevel@tonic-gate ASSERT(lpcb->lp_wait != 0); 13317c478bd9Sstevel@tonic-gate lpcb->lp_wait = 0; 13327c478bd9Sstevel@tonic-gate cv_broadcast(&lpcb->lp_cv); 13337c478bd9Sstevel@tonic-gate mutex_exit(&lpcb->lp_lock); 13347c478bd9Sstevel@tonic-gate } 13357c478bd9Sstevel@tonic-gate } 13367c478bd9Sstevel@tonic-gate 13377c478bd9Sstevel@tonic-gate if (addr != NULL) { 13387c478bd9Sstevel@tonic-gate *sizep = asize; 13397c478bd9Sstevel@tonic-gate *lpthrtp = 0; 13407c478bd9Sstevel@tonic-gate return (addr); 13417c478bd9Sstevel@tonic-gate } 13427c478bd9Sstevel@tonic-gate 13437c478bd9Sstevel@tonic-gate if (vmflag & VM_NOSLEEP) 1344501f3284Seg155566 lpcb->nosleep_allocs_failed++; 13457c478bd9Sstevel@tonic-gate else 1346501f3284Seg155566 lpcb->sleep_allocs_failed++; 1347501f3284Seg155566 lpcb->alloc_bytes_failed += size; 13487c478bd9Sstevel@tonic-gate 13497c478bd9Sstevel@tonic-gate /* if large page throttling is not started yet do it */ 13507c478bd9Sstevel@tonic-gate if (segkmem_use_lpthrottle && lpthrt == 0) { 13517c478bd9Sstevel@tonic-gate lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1); 13527c478bd9Sstevel@tonic-gate } 13537c478bd9Sstevel@tonic-gate } 13547c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, vmflag)); 13557c478bd9Sstevel@tonic-gate } 13567c478bd9Sstevel@tonic-gate 13577c478bd9Sstevel@tonic-gate void 13587c478bd9Sstevel@tonic-gate segkmem_free_lp(vmem_t *vmp, void *inaddr, size_t size) 13597c478bd9Sstevel@tonic-gate { 13607c478bd9Sstevel@tonic-gate if (kmem_lp_arena == NULL || !IS_KMEM_VA_LARGEPAGE((caddr_t)inaddr)) { 13617c478bd9Sstevel@tonic-gate segkmem_free(vmp, inaddr, size); 13627c478bd9Sstevel@tonic-gate } else { 13637c478bd9Sstevel@tonic-gate vmem_free(kmem_lp_arena, inaddr, size); 13647c478bd9Sstevel@tonic-gate } 13657c478bd9Sstevel@tonic-gate } 13667c478bd9Sstevel@tonic-gate 13677c478bd9Sstevel@tonic-gate /* 13687c478bd9Sstevel@tonic-gate * segkmem_alloc_lpi() imports virtual memory from large page heap arena 13697c478bd9Sstevel@tonic-gate * into kmem_lp arena. In the process it maps the imported segment with 13707c478bd9Sstevel@tonic-gate * large pages 13717c478bd9Sstevel@tonic-gate */ 13727c478bd9Sstevel@tonic-gate static void * 13737c478bd9Sstevel@tonic-gate segkmem_alloc_lpi(vmem_t *vmp, size_t size, int vmflag) 13747c478bd9Sstevel@tonic-gate { 13757c478bd9Sstevel@tonic-gate segkmem_lpcb_t *lpcb = &segkmem_lpcb; 13767c478bd9Sstevel@tonic-gate void *addr; 13777c478bd9Sstevel@tonic-gate 13787c478bd9Sstevel@tonic-gate ASSERT(size != 0); 13797c478bd9Sstevel@tonic-gate ASSERT(vmp == heap_lp_arena); 13807c478bd9Sstevel@tonic-gate 13817c478bd9Sstevel@tonic-gate /* do not allow large page heap grow beyound limits */ 13827c478bd9Sstevel@tonic-gate if (vmem_size(vmp, VMEM_ALLOC) >= segkmem_kmemlp_max) { 1383501f3284Seg155566 lpcb->allocs_limited++; 13847c478bd9Sstevel@tonic-gate return (NULL); 13857c478bd9Sstevel@tonic-gate } 13867c478bd9Sstevel@tonic-gate 13877c478bd9Sstevel@tonic-gate addr = segkmem_xalloc_lp(vmp, NULL, size, vmflag, 0, 13887c478bd9Sstevel@tonic-gate segkmem_page_create_large, NULL); 13897c478bd9Sstevel@tonic-gate return (addr); 13907c478bd9Sstevel@tonic-gate } 13917c478bd9Sstevel@tonic-gate 13927c478bd9Sstevel@tonic-gate /* 13937c478bd9Sstevel@tonic-gate * segkmem_free_lpi() returns virtual memory back into large page heap arena 13947c478bd9Sstevel@tonic-gate * from kmem_lp arena. Beore doing this it unmaps the segment and frees 13957c478bd9Sstevel@tonic-gate * large pages used to map it. 13967c478bd9Sstevel@tonic-gate */ 13977c478bd9Sstevel@tonic-gate static void 13987c478bd9Sstevel@tonic-gate segkmem_free_lpi(vmem_t *vmp, void *inaddr, size_t size) 13997c478bd9Sstevel@tonic-gate { 14007c478bd9Sstevel@tonic-gate pgcnt_t nlpages = size >> segkmem_lpshift; 14017c478bd9Sstevel@tonic-gate size_t lpsize = segkmem_lpsize; 14027c478bd9Sstevel@tonic-gate caddr_t addr = inaddr; 14037c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size); 14047c478bd9Sstevel@tonic-gate int i; 14057c478bd9Sstevel@tonic-gate 14067c478bd9Sstevel@tonic-gate ASSERT(vmp == heap_lp_arena); 14077c478bd9Sstevel@tonic-gate ASSERT(IS_KMEM_VA_LARGEPAGE(addr)); 14087c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)inaddr & (lpsize - 1)) == 0); 14097c478bd9Sstevel@tonic-gate 14107c478bd9Sstevel@tonic-gate for (i = 0; i < nlpages; i++) { 14117c478bd9Sstevel@tonic-gate segkmem_free_one_lp(addr, lpsize); 14127c478bd9Sstevel@tonic-gate addr += lpsize; 14137c478bd9Sstevel@tonic-gate } 14147c478bd9Sstevel@tonic-gate 14157c478bd9Sstevel@tonic-gate page_unresv(npages); 14167c478bd9Sstevel@tonic-gate 14177c478bd9Sstevel@tonic-gate vmem_free(vmp, inaddr, size); 14187c478bd9Sstevel@tonic-gate } 14197c478bd9Sstevel@tonic-gate 14207c478bd9Sstevel@tonic-gate /* 14217c478bd9Sstevel@tonic-gate * This function is called at system boot time by kmem_init right after 14227c478bd9Sstevel@tonic-gate * /etc/system file has been read. It checks based on hardware configuration 14237c478bd9Sstevel@tonic-gate * and /etc/system settings if system is going to use large pages. The 14247c478bd9Sstevel@tonic-gate * initialiazation necessary to actually start using large pages 14257c478bd9Sstevel@tonic-gate * happens later in the process after segkmem_heap_lp_init() is called. 14267c478bd9Sstevel@tonic-gate */ 14277c478bd9Sstevel@tonic-gate int 14287c478bd9Sstevel@tonic-gate segkmem_lpsetup() 14297c478bd9Sstevel@tonic-gate { 14307c478bd9Sstevel@tonic-gate int use_large_pages = 0; 14317c478bd9Sstevel@tonic-gate 14327c478bd9Sstevel@tonic-gate #ifdef __sparc 14337c478bd9Sstevel@tonic-gate 14347c478bd9Sstevel@tonic-gate size_t memtotal = physmem * PAGESIZE; 14357c478bd9Sstevel@tonic-gate 14367c478bd9Sstevel@tonic-gate if (heap_lp_base == NULL) { 14377c478bd9Sstevel@tonic-gate segkmem_lpsize = PAGESIZE; 14387c478bd9Sstevel@tonic-gate return (0); 14397c478bd9Sstevel@tonic-gate } 14407c478bd9Sstevel@tonic-gate 14417c478bd9Sstevel@tonic-gate /* get a platform dependent value of large page size for kernel heap */ 14427c478bd9Sstevel@tonic-gate segkmem_lpsize = get_segkmem_lpsize(segkmem_lpsize); 14437c478bd9Sstevel@tonic-gate 14447c478bd9Sstevel@tonic-gate if (segkmem_lpsize <= PAGESIZE) { 14457c478bd9Sstevel@tonic-gate /* 14467c478bd9Sstevel@tonic-gate * put virtual space reserved for the large page kernel 14477c478bd9Sstevel@tonic-gate * back to the regular heap 14487c478bd9Sstevel@tonic-gate */ 14497c478bd9Sstevel@tonic-gate vmem_xfree(heap_arena, heap_lp_base, 14507c478bd9Sstevel@tonic-gate heap_lp_end - heap_lp_base); 14517c478bd9Sstevel@tonic-gate heap_lp_base = NULL; 14527c478bd9Sstevel@tonic-gate heap_lp_end = NULL; 14537c478bd9Sstevel@tonic-gate segkmem_lpsize = PAGESIZE; 14547c478bd9Sstevel@tonic-gate return (0); 14557c478bd9Sstevel@tonic-gate } 14567c478bd9Sstevel@tonic-gate 14577c478bd9Sstevel@tonic-gate /* set heap_lp quantum if necessary */ 14587c478bd9Sstevel@tonic-gate if (segkmem_heaplp_quantum == 0 || 14597c478bd9Sstevel@tonic-gate (segkmem_heaplp_quantum & (segkmem_heaplp_quantum - 1)) || 14607c478bd9Sstevel@tonic-gate P2PHASE(segkmem_heaplp_quantum, segkmem_lpsize)) { 14617c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum = segkmem_lpsize; 14627c478bd9Sstevel@tonic-gate } 14637c478bd9Sstevel@tonic-gate 14647c478bd9Sstevel@tonic-gate /* set kmem_lp quantum if necessary */ 14657c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_quantum == 0 || 14667c478bd9Sstevel@tonic-gate (segkmem_kmemlp_quantum & (segkmem_kmemlp_quantum - 1)) || 14677c478bd9Sstevel@tonic-gate segkmem_kmemlp_quantum > segkmem_heaplp_quantum) { 14687c478bd9Sstevel@tonic-gate segkmem_kmemlp_quantum = segkmem_heaplp_quantum; 14697c478bd9Sstevel@tonic-gate } 14707c478bd9Sstevel@tonic-gate 14717c478bd9Sstevel@tonic-gate /* set total amount of memory allowed for large page kernel heap */ 14727c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_max == 0) { 14737c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_pcnt == 0 || segkmem_kmemlp_pcnt > 100) 1474501f3284Seg155566 segkmem_kmemlp_pcnt = 12; 1475501f3284Seg155566 segkmem_kmemlp_max = (memtotal * segkmem_kmemlp_pcnt) / 100; 14767c478bd9Sstevel@tonic-gate } 14777c478bd9Sstevel@tonic-gate segkmem_kmemlp_max = P2ROUNDUP(segkmem_kmemlp_max, 14787c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum); 14797c478bd9Sstevel@tonic-gate 14807c478bd9Sstevel@tonic-gate /* fix lp kmem preallocation request if necesssary */ 14817c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_min) { 14827c478bd9Sstevel@tonic-gate segkmem_kmemlp_min = P2ROUNDUP(segkmem_kmemlp_min, 14837c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum); 14847c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_min > segkmem_kmemlp_max) 14857c478bd9Sstevel@tonic-gate segkmem_kmemlp_min = segkmem_kmemlp_max; 14867c478bd9Sstevel@tonic-gate } 14877c478bd9Sstevel@tonic-gate 14887c478bd9Sstevel@tonic-gate use_large_pages = 1; 1489081a94b0Saguzovsk segkmem_lpszc = page_szc(segkmem_lpsize); 1490081a94b0Saguzovsk segkmem_lpshift = page_get_shift(segkmem_lpszc); 14917c478bd9Sstevel@tonic-gate 14927c478bd9Sstevel@tonic-gate #endif 14937c478bd9Sstevel@tonic-gate return (use_large_pages); 14947c478bd9Sstevel@tonic-gate } 14957c478bd9Sstevel@tonic-gate 1496ad23a2dbSjohansen void 1497ad23a2dbSjohansen segkmem_zio_init(void *zio_mem_base, size_t zio_mem_size) 1498ad23a2dbSjohansen { 1499ad23a2dbSjohansen ASSERT(zio_mem_base != NULL); 1500ad23a2dbSjohansen ASSERT(zio_mem_size != 0); 1501ad23a2dbSjohansen 1502*0fab61baSJonathan W Adams zio_arena = vmem_create("zfs_file_data", zio_mem_base, zio_mem_size, 1503*0fab61baSJonathan W Adams PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP); 1504ad23a2dbSjohansen 1505*0fab61baSJonathan W Adams zio_alloc_arena = vmem_create("zfs_file_data_buf", NULL, 0, PAGESIZE, 1506ad23a2dbSjohansen segkmem_zio_alloc, segkmem_zio_free, zio_arena, 0, VM_SLEEP); 1507ad23a2dbSjohansen 1508ad23a2dbSjohansen ASSERT(zio_arena != NULL); 1509ad23a2dbSjohansen ASSERT(zio_alloc_arena != NULL); 1510ad23a2dbSjohansen } 1511ad23a2dbSjohansen 15127c478bd9Sstevel@tonic-gate #ifdef __sparc 15137c478bd9Sstevel@tonic-gate 15147c478bd9Sstevel@tonic-gate 15157c478bd9Sstevel@tonic-gate static void * 15167c478bd9Sstevel@tonic-gate segkmem_alloc_ppa(vmem_t *vmp, size_t size, int vmflag) 15177c478bd9Sstevel@tonic-gate { 15187c478bd9Sstevel@tonic-gate size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *); 15197c478bd9Sstevel@tonic-gate void *addr; 15207c478bd9Sstevel@tonic-gate 15217c478bd9Sstevel@tonic-gate if (ppaquantum <= PAGESIZE) 15227c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, vmflag)); 15237c478bd9Sstevel@tonic-gate 15247c478bd9Sstevel@tonic-gate ASSERT((size & (ppaquantum - 1)) == 0); 15257c478bd9Sstevel@tonic-gate 15267c478bd9Sstevel@tonic-gate addr = vmem_xalloc(vmp, size, ppaquantum, 0, 0, NULL, NULL, vmflag); 15277c478bd9Sstevel@tonic-gate if (addr != NULL && segkmem_xalloc(vmp, addr, size, vmflag, 0, 15287c478bd9Sstevel@tonic-gate segkmem_page_create, NULL) == NULL) { 15297c478bd9Sstevel@tonic-gate vmem_xfree(vmp, addr, size); 15307c478bd9Sstevel@tonic-gate addr = NULL; 15317c478bd9Sstevel@tonic-gate } 15327c478bd9Sstevel@tonic-gate 15337c478bd9Sstevel@tonic-gate return (addr); 15347c478bd9Sstevel@tonic-gate } 15357c478bd9Sstevel@tonic-gate 15367c478bd9Sstevel@tonic-gate static void 15377c478bd9Sstevel@tonic-gate segkmem_free_ppa(vmem_t *vmp, void *addr, size_t size) 15387c478bd9Sstevel@tonic-gate { 15397c478bd9Sstevel@tonic-gate size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *); 15407c478bd9Sstevel@tonic-gate 15417c478bd9Sstevel@tonic-gate ASSERT(addr != NULL); 15427c478bd9Sstevel@tonic-gate 15437c478bd9Sstevel@tonic-gate if (ppaquantum <= PAGESIZE) { 15447c478bd9Sstevel@tonic-gate segkmem_free(vmp, addr, size); 15457c478bd9Sstevel@tonic-gate } else { 15467c478bd9Sstevel@tonic-gate segkmem_free(NULL, addr, size); 15477c478bd9Sstevel@tonic-gate vmem_xfree(vmp, addr, size); 15487c478bd9Sstevel@tonic-gate } 15497c478bd9Sstevel@tonic-gate } 15507c478bd9Sstevel@tonic-gate 15517c478bd9Sstevel@tonic-gate void 15527c478bd9Sstevel@tonic-gate segkmem_heap_lp_init() 15537c478bd9Sstevel@tonic-gate { 15547c478bd9Sstevel@tonic-gate segkmem_lpcb_t *lpcb = &segkmem_lpcb; 15557c478bd9Sstevel@tonic-gate size_t heap_lp_size = heap_lp_end - heap_lp_base; 15567c478bd9Sstevel@tonic-gate size_t lpsize = segkmem_lpsize; 15577c478bd9Sstevel@tonic-gate size_t ppaquantum; 15587c478bd9Sstevel@tonic-gate void *addr; 15597c478bd9Sstevel@tonic-gate 15607c478bd9Sstevel@tonic-gate if (segkmem_lpsize <= PAGESIZE) { 15617c478bd9Sstevel@tonic-gate ASSERT(heap_lp_base == NULL); 15627c478bd9Sstevel@tonic-gate ASSERT(heap_lp_end == NULL); 15637c478bd9Sstevel@tonic-gate return; 15647c478bd9Sstevel@tonic-gate } 15657c478bd9Sstevel@tonic-gate 15667c478bd9Sstevel@tonic-gate ASSERT(segkmem_heaplp_quantum >= lpsize); 15677c478bd9Sstevel@tonic-gate ASSERT((segkmem_heaplp_quantum & (lpsize - 1)) == 0); 15687c478bd9Sstevel@tonic-gate ASSERT(lpcb->lp_uselp == 0); 15697c478bd9Sstevel@tonic-gate ASSERT(heap_lp_base != NULL); 15707c478bd9Sstevel@tonic-gate ASSERT(heap_lp_end != NULL); 15717c478bd9Sstevel@tonic-gate ASSERT(heap_lp_base < heap_lp_end); 15727c478bd9Sstevel@tonic-gate ASSERT(heap_lp_arena == NULL); 15737c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)heap_lp_base & (lpsize - 1)) == 0); 15747c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)heap_lp_end & (lpsize - 1)) == 0); 15757c478bd9Sstevel@tonic-gate 15767c478bd9Sstevel@tonic-gate /* create large page heap arena */ 15777c478bd9Sstevel@tonic-gate heap_lp_arena = vmem_create("heap_lp", heap_lp_base, heap_lp_size, 15787c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum, NULL, NULL, NULL, 0, VM_SLEEP); 15797c478bd9Sstevel@tonic-gate 15807c478bd9Sstevel@tonic-gate ASSERT(heap_lp_arena != NULL); 15817c478bd9Sstevel@tonic-gate 15827c478bd9Sstevel@tonic-gate /* This arena caches memory already mapped by large pages */ 15837c478bd9Sstevel@tonic-gate kmem_lp_arena = vmem_create("kmem_lp", NULL, 0, segkmem_kmemlp_quantum, 15847c478bd9Sstevel@tonic-gate segkmem_alloc_lpi, segkmem_free_lpi, heap_lp_arena, 0, VM_SLEEP); 15857c478bd9Sstevel@tonic-gate 15867c478bd9Sstevel@tonic-gate ASSERT(kmem_lp_arena != NULL); 15877c478bd9Sstevel@tonic-gate 15887c478bd9Sstevel@tonic-gate mutex_init(&lpcb->lp_lock, NULL, MUTEX_DEFAULT, NULL); 15897c478bd9Sstevel@tonic-gate cv_init(&lpcb->lp_cv, NULL, CV_DEFAULT, NULL); 15907c478bd9Sstevel@tonic-gate 15917c478bd9Sstevel@tonic-gate /* 15927c478bd9Sstevel@tonic-gate * this arena is used for the array of page_t pointers necessary 15937c478bd9Sstevel@tonic-gate * to call hat_mem_load_array 15947c478bd9Sstevel@tonic-gate */ 15957c478bd9Sstevel@tonic-gate ppaquantum = btopr(lpsize) * sizeof (page_t *); 15967c478bd9Sstevel@tonic-gate segkmem_ppa_arena = vmem_create("segkmem_ppa", NULL, 0, ppaquantum, 15977c478bd9Sstevel@tonic-gate segkmem_alloc_ppa, segkmem_free_ppa, heap_arena, ppaquantum, 15987c478bd9Sstevel@tonic-gate VM_SLEEP); 15997c478bd9Sstevel@tonic-gate 16007c478bd9Sstevel@tonic-gate ASSERT(segkmem_ppa_arena != NULL); 16017c478bd9Sstevel@tonic-gate 16027c478bd9Sstevel@tonic-gate /* prealloacate some memory for the lp kernel heap */ 16037c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_min) { 16047c478bd9Sstevel@tonic-gate 16057c478bd9Sstevel@tonic-gate ASSERT(P2PHASE(segkmem_kmemlp_min, 16067c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum) == 0); 16077c478bd9Sstevel@tonic-gate 16087c478bd9Sstevel@tonic-gate if ((addr = segkmem_alloc_lpi(heap_lp_arena, 16097c478bd9Sstevel@tonic-gate segkmem_kmemlp_min, VM_SLEEP)) != NULL) { 16107c478bd9Sstevel@tonic-gate 16117c478bd9Sstevel@tonic-gate addr = vmem_add(kmem_lp_arena, addr, 16127c478bd9Sstevel@tonic-gate segkmem_kmemlp_min, VM_SLEEP); 16137c478bd9Sstevel@tonic-gate ASSERT(addr != NULL); 16147c478bd9Sstevel@tonic-gate } 16157c478bd9Sstevel@tonic-gate } 16167c478bd9Sstevel@tonic-gate 16177c478bd9Sstevel@tonic-gate lpcb->lp_uselp = 1; 16187c478bd9Sstevel@tonic-gate } 16197c478bd9Sstevel@tonic-gate 16207c478bd9Sstevel@tonic-gate #endif 1621