17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5ad23a2dbSjohansen * Common Development and Distribution License (the "License"). 6ad23a2dbSjohansen * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22081a94b0Saguzovsk * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate #include <sys/types.h> 297c478bd9Sstevel@tonic-gate #include <sys/t_lock.h> 307c478bd9Sstevel@tonic-gate #include <sys/param.h> 317c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 327c478bd9Sstevel@tonic-gate #include <sys/tuneable.h> 337c478bd9Sstevel@tonic-gate #include <sys/systm.h> 347c478bd9Sstevel@tonic-gate #include <sys/vm.h> 357c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 367c478bd9Sstevel@tonic-gate #include <sys/vmem.h> 377c478bd9Sstevel@tonic-gate #include <sys/mman.h> 387c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 397c478bd9Sstevel@tonic-gate #include <sys/debug.h> 407c478bd9Sstevel@tonic-gate #include <sys/dumphdr.h> 417c478bd9Sstevel@tonic-gate #include <sys/bootconf.h> 427c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 437c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 447c478bd9Sstevel@tonic-gate #include <vm/hat.h> 457c478bd9Sstevel@tonic-gate #include <vm/page.h> 467c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h> 477c478bd9Sstevel@tonic-gate #include <vm/faultcode.h> 487c478bd9Sstevel@tonic-gate #include <sys/promif.h> 497c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h> 507c478bd9Sstevel@tonic-gate #include <sys/bitmap.h> 517c478bd9Sstevel@tonic-gate #include <sys/mem_cage.h> 527c478bd9Sstevel@tonic-gate 537c478bd9Sstevel@tonic-gate /* 547c478bd9Sstevel@tonic-gate * seg_kmem is the primary kernel memory segment driver. It 557c478bd9Sstevel@tonic-gate * maps the kernel heap [kernelheap, ekernelheap), module text, 567c478bd9Sstevel@tonic-gate * and all memory which was allocated before the VM was initialized 577c478bd9Sstevel@tonic-gate * into kas. 587c478bd9Sstevel@tonic-gate * 597c478bd9Sstevel@tonic-gate * Pages which belong to seg_kmem are hashed into &kvp vnode at 607c478bd9Sstevel@tonic-gate * an offset equal to (u_offset_t)virt_addr, and have p_lckcnt >= 1. 617c478bd9Sstevel@tonic-gate * They must never be paged out since segkmem_fault() is a no-op to 627c478bd9Sstevel@tonic-gate * prevent recursive faults. 637c478bd9Sstevel@tonic-gate * 647c478bd9Sstevel@tonic-gate * Currently, seg_kmem pages are sharelocked (p_sharelock == 1) on 657c478bd9Sstevel@tonic-gate * __x86 and are unlocked (p_sharelock == 0) on __sparc. Once __x86 667c478bd9Sstevel@tonic-gate * supports relocation the #ifdef kludges can be removed. 677c478bd9Sstevel@tonic-gate * 687c478bd9Sstevel@tonic-gate * seg_kmem pages may be subject to relocation by page_relocate(), 697c478bd9Sstevel@tonic-gate * provided that the HAT supports it; if this is so, segkmem_reloc 707c478bd9Sstevel@tonic-gate * will be set to a nonzero value. All boot time allocated memory as 717c478bd9Sstevel@tonic-gate * well as static memory is considered off limits to relocation. 727c478bd9Sstevel@tonic-gate * Pages are "relocatable" if p_state does not have P_NORELOC set, so 737c478bd9Sstevel@tonic-gate * we request P_NORELOC pages for memory that isn't safe to relocate. 747c478bd9Sstevel@tonic-gate * 757c478bd9Sstevel@tonic-gate * The kernel heap is logically divided up into four pieces: 767c478bd9Sstevel@tonic-gate * 777c478bd9Sstevel@tonic-gate * heap32_arena is for allocations that require 32-bit absolute 787c478bd9Sstevel@tonic-gate * virtual addresses (e.g. code that uses 32-bit pointers/offsets). 797c478bd9Sstevel@tonic-gate * 807c478bd9Sstevel@tonic-gate * heap_core is for allocations that require 2GB *relative* 817c478bd9Sstevel@tonic-gate * offsets; in other words all memory from heap_core is within 827c478bd9Sstevel@tonic-gate * 2GB of all other memory from the same arena. This is a requirement 837c478bd9Sstevel@tonic-gate * of the addressing modes of some processors in supervisor code. 847c478bd9Sstevel@tonic-gate * 857c478bd9Sstevel@tonic-gate * heap_arena is the general heap arena. 867c478bd9Sstevel@tonic-gate * 877c478bd9Sstevel@tonic-gate * static_arena is the static memory arena. Allocations from it 887c478bd9Sstevel@tonic-gate * are not subject to relocation so it is safe to use the memory 897c478bd9Sstevel@tonic-gate * physical address as well as the virtual address (e.g. the VA to 907c478bd9Sstevel@tonic-gate * PA translations are static). Caches may import from static_arena; 917c478bd9Sstevel@tonic-gate * all other static memory allocations should use static_alloc_arena. 927c478bd9Sstevel@tonic-gate * 937c478bd9Sstevel@tonic-gate * On some platforms which have limited virtual address space, seg_kmem 947c478bd9Sstevel@tonic-gate * may share [kernelheap, ekernelheap) with seg_kp; if this is so, 957c478bd9Sstevel@tonic-gate * segkp_bitmap is non-NULL, and each bit represents a page of virtual 967c478bd9Sstevel@tonic-gate * address space which is actually seg_kp mapped. 977c478bd9Sstevel@tonic-gate */ 987c478bd9Sstevel@tonic-gate 997c478bd9Sstevel@tonic-gate extern ulong_t *segkp_bitmap; /* Is set if segkp is from the kernel heap */ 1007c478bd9Sstevel@tonic-gate 1017c478bd9Sstevel@tonic-gate char *kernelheap; /* start of primary kernel heap */ 1027c478bd9Sstevel@tonic-gate char *ekernelheap; /* end of primary kernel heap */ 1037c478bd9Sstevel@tonic-gate struct seg kvseg; /* primary kernel heap segment */ 1047c478bd9Sstevel@tonic-gate struct seg kvseg_core; /* "core" kernel heap segment */ 105ad23a2dbSjohansen struct seg kzioseg; /* Segment for zio mappings */ 1067c478bd9Sstevel@tonic-gate vmem_t *heap_arena; /* primary kernel heap arena */ 1077c478bd9Sstevel@tonic-gate vmem_t *heap_core_arena; /* core kernel heap arena */ 1087c478bd9Sstevel@tonic-gate char *heap_core_base; /* start of core kernel heap arena */ 1097c478bd9Sstevel@tonic-gate char *heap_lp_base; /* start of kernel large page heap arena */ 1107c478bd9Sstevel@tonic-gate char *heap_lp_end; /* end of kernel large page heap arena */ 1117c478bd9Sstevel@tonic-gate vmem_t *hat_memload_arena; /* HAT translation data */ 1127c478bd9Sstevel@tonic-gate struct seg kvseg32; /* 32-bit kernel heap segment */ 1137c478bd9Sstevel@tonic-gate vmem_t *heap32_arena; /* 32-bit kernel heap arena */ 1147c478bd9Sstevel@tonic-gate vmem_t *heaptext_arena; /* heaptext arena */ 1157c478bd9Sstevel@tonic-gate struct as kas; /* kernel address space */ 1167c478bd9Sstevel@tonic-gate struct vnode kvp; /* vnode for all segkmem pages */ 117ad23a2dbSjohansen struct vnode zvp; /* vnode for zfs pages */ 1187c478bd9Sstevel@tonic-gate int segkmem_reloc; /* enable/disable relocatable segkmem pages */ 1197c478bd9Sstevel@tonic-gate vmem_t *static_arena; /* arena for caches to import static memory */ 1207c478bd9Sstevel@tonic-gate vmem_t *static_alloc_arena; /* arena for allocating static memory */ 121ad23a2dbSjohansen vmem_t *zio_arena = NULL; /* arena for allocating zio memory */ 122ad23a2dbSjohansen vmem_t *zio_alloc_arena = NULL; /* arena for allocating zio memory */ 1237c478bd9Sstevel@tonic-gate 1247c478bd9Sstevel@tonic-gate /* 1257c478bd9Sstevel@tonic-gate * seg_kmem driver can map part of the kernel heap with large pages. 1267c478bd9Sstevel@tonic-gate * Currently this functionality is implemented for sparc platforms only. 1277c478bd9Sstevel@tonic-gate * 1287c478bd9Sstevel@tonic-gate * The large page size "segkmem_lpsize" for kernel heap is selected in the 1297c478bd9Sstevel@tonic-gate * platform specific code. It can also be modified via /etc/system file. 1307c478bd9Sstevel@tonic-gate * Setting segkmem_lpsize to PAGESIZE in /etc/system disables usage of large 1317c478bd9Sstevel@tonic-gate * pages for kernel heap. "segkmem_lpshift" is adjusted appropriately to 1327c478bd9Sstevel@tonic-gate * match segkmem_lpsize. 1337c478bd9Sstevel@tonic-gate * 1347c478bd9Sstevel@tonic-gate * At boot time we carve from kernel heap arena a range of virtual addresses 1357c478bd9Sstevel@tonic-gate * that will be used for large page mappings. This range [heap_lp_base, 1367c478bd9Sstevel@tonic-gate * heap_lp_end) is set up as a separate vmem arena - "heap_lp_arena". We also 1377c478bd9Sstevel@tonic-gate * create "kmem_lp_arena" that caches memory already backed up by large 1387c478bd9Sstevel@tonic-gate * pages. kmem_lp_arena imports virtual segments from heap_lp_arena. 1397c478bd9Sstevel@tonic-gate */ 1407c478bd9Sstevel@tonic-gate 1417c478bd9Sstevel@tonic-gate size_t segkmem_lpsize; 1427c478bd9Sstevel@tonic-gate static uint_t segkmem_lpshift = PAGESHIFT; 143081a94b0Saguzovsk int segkmem_lpszc = 0; 1447c478bd9Sstevel@tonic-gate 1457c478bd9Sstevel@tonic-gate size_t segkmem_kmemlp_quantum = 0x400000; /* 4MB */ 1467c478bd9Sstevel@tonic-gate size_t segkmem_heaplp_quantum; 1477bc98a2eSeg155566 vmem_t *heap_lp_arena; 1487c478bd9Sstevel@tonic-gate static vmem_t *kmem_lp_arena; 1497c478bd9Sstevel@tonic-gate static vmem_t *segkmem_ppa_arena; 1507c478bd9Sstevel@tonic-gate static segkmem_lpcb_t segkmem_lpcb; 1517c478bd9Sstevel@tonic-gate 1527c478bd9Sstevel@tonic-gate /* 1537c478bd9Sstevel@tonic-gate * We use "segkmem_kmemlp_max" to limit the total amount of physical memory 154501f3284Seg155566 * consumed by the large page heap. By default this parameter is set to 1/8 of 1557c478bd9Sstevel@tonic-gate * physmem but can be adjusted through /etc/system either directly or 1567c478bd9Sstevel@tonic-gate * indirectly by setting "segkmem_kmemlp_pcnt" to the percent of physmem 1577c478bd9Sstevel@tonic-gate * we allow for large page heap. 1587c478bd9Sstevel@tonic-gate */ 1597c478bd9Sstevel@tonic-gate size_t segkmem_kmemlp_max; 1607c478bd9Sstevel@tonic-gate static uint_t segkmem_kmemlp_pcnt; 1617c478bd9Sstevel@tonic-gate 1627c478bd9Sstevel@tonic-gate /* 1637c478bd9Sstevel@tonic-gate * Getting large pages for kernel heap could be problematic due to 1647c478bd9Sstevel@tonic-gate * physical memory fragmentation. That's why we allow to preallocate 1657c478bd9Sstevel@tonic-gate * "segkmem_kmemlp_min" bytes at boot time. 1667c478bd9Sstevel@tonic-gate */ 1677c478bd9Sstevel@tonic-gate static size_t segkmem_kmemlp_min; 1687c478bd9Sstevel@tonic-gate 1697c478bd9Sstevel@tonic-gate /* 1707c478bd9Sstevel@tonic-gate * Throttling is used to avoid expensive tries to allocate large pages 1717c478bd9Sstevel@tonic-gate * for kernel heap when a lot of succesive attempts to do so fail. 1727c478bd9Sstevel@tonic-gate */ 1737c478bd9Sstevel@tonic-gate static ulong_t segkmem_lpthrottle_max = 0x400000; 1747c478bd9Sstevel@tonic-gate static ulong_t segkmem_lpthrottle_start = 0x40; 1757c478bd9Sstevel@tonic-gate static ulong_t segkmem_use_lpthrottle = 1; 1767c478bd9Sstevel@tonic-gate 1777c478bd9Sstevel@tonic-gate /* 1787c478bd9Sstevel@tonic-gate * Freed pages accumulate on a garbage list until segkmem is ready, 1797c478bd9Sstevel@tonic-gate * at which point we call segkmem_gc() to free it all. 1807c478bd9Sstevel@tonic-gate */ 1817c478bd9Sstevel@tonic-gate typedef struct segkmem_gc_list { 1827c478bd9Sstevel@tonic-gate struct segkmem_gc_list *gc_next; 1837c478bd9Sstevel@tonic-gate vmem_t *gc_arena; 1847c478bd9Sstevel@tonic-gate size_t gc_size; 1857c478bd9Sstevel@tonic-gate } segkmem_gc_list_t; 1867c478bd9Sstevel@tonic-gate 1877c478bd9Sstevel@tonic-gate static segkmem_gc_list_t *segkmem_gc_list; 1887c478bd9Sstevel@tonic-gate 1897c478bd9Sstevel@tonic-gate /* 1907c478bd9Sstevel@tonic-gate * Allocations from the hat_memload arena add VM_MEMLOAD to their 1917c478bd9Sstevel@tonic-gate * vmflags so that segkmem_xalloc() can inform the hat layer that it needs 1927c478bd9Sstevel@tonic-gate * to take steps to prevent infinite recursion. HAT allocations also 1937c478bd9Sstevel@tonic-gate * must be non-relocatable to prevent recursive page faults. 1947c478bd9Sstevel@tonic-gate */ 1957c478bd9Sstevel@tonic-gate static void * 1967c478bd9Sstevel@tonic-gate hat_memload_alloc(vmem_t *vmp, size_t size, int flags) 1977c478bd9Sstevel@tonic-gate { 1987c478bd9Sstevel@tonic-gate flags |= (VM_MEMLOAD | VM_NORELOC); 1997c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, flags)); 2007c478bd9Sstevel@tonic-gate } 2017c478bd9Sstevel@tonic-gate 2027c478bd9Sstevel@tonic-gate /* 2037c478bd9Sstevel@tonic-gate * Allocations from static_arena arena (or any other arena that uses 2047c478bd9Sstevel@tonic-gate * segkmem_alloc_permanent()) require non-relocatable (permanently 2057c478bd9Sstevel@tonic-gate * wired) memory pages, since these pages are referenced by physical 2067c478bd9Sstevel@tonic-gate * as well as virtual address. 2077c478bd9Sstevel@tonic-gate */ 2087c478bd9Sstevel@tonic-gate void * 2097c478bd9Sstevel@tonic-gate segkmem_alloc_permanent(vmem_t *vmp, size_t size, int flags) 2107c478bd9Sstevel@tonic-gate { 2117c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, flags | VM_NORELOC)); 2127c478bd9Sstevel@tonic-gate } 2137c478bd9Sstevel@tonic-gate 2147c478bd9Sstevel@tonic-gate /* 2157c478bd9Sstevel@tonic-gate * Initialize kernel heap boundaries. 2167c478bd9Sstevel@tonic-gate */ 2177c478bd9Sstevel@tonic-gate void 2187c478bd9Sstevel@tonic-gate kernelheap_init( 2197c478bd9Sstevel@tonic-gate void *heap_start, 2207c478bd9Sstevel@tonic-gate void *heap_end, 2217c478bd9Sstevel@tonic-gate char *first_avail, 2227c478bd9Sstevel@tonic-gate void *core_start, 2237c478bd9Sstevel@tonic-gate void *core_end) 2247c478bd9Sstevel@tonic-gate { 2257c478bd9Sstevel@tonic-gate uintptr_t textbase; 2267c478bd9Sstevel@tonic-gate size_t core_size; 2277c478bd9Sstevel@tonic-gate size_t heap_size; 2287c478bd9Sstevel@tonic-gate vmem_t *heaptext_parent; 2297c478bd9Sstevel@tonic-gate size_t heap_lp_size = 0; 230*bb121940Sdp78419 #ifdef __sparc 231*bb121940Sdp78419 size_t kmem64_sz = kmem64_aligned_end - kmem64_base; 232*bb121940Sdp78419 #endif /* __sparc */ 2337c478bd9Sstevel@tonic-gate 2347c478bd9Sstevel@tonic-gate kernelheap = heap_start; 2357c478bd9Sstevel@tonic-gate ekernelheap = heap_end; 2367c478bd9Sstevel@tonic-gate 2377c478bd9Sstevel@tonic-gate #ifdef __sparc 2387c478bd9Sstevel@tonic-gate heap_lp_size = (((uintptr_t)heap_end - (uintptr_t)heap_start) / 4); 239*bb121940Sdp78419 /* 240*bb121940Sdp78419 * Bias heap_lp start address by kmem64_sz to reduce collisions 241*bb121940Sdp78419 * in 4M kernel TSB between kmem64 area and heap_lp 242*bb121940Sdp78419 */ 243*bb121940Sdp78419 kmem64_sz = P2ROUNDUP(kmem64_sz, MMU_PAGESIZE256M); 244*bb121940Sdp78419 if (kmem64_sz <= heap_lp_size / 2) 245*bb121940Sdp78419 heap_lp_size -= kmem64_sz; 2467c478bd9Sstevel@tonic-gate heap_lp_base = ekernelheap - heap_lp_size; 2477c478bd9Sstevel@tonic-gate heap_lp_end = heap_lp_base + heap_lp_size; 2487c478bd9Sstevel@tonic-gate #endif /* __sparc */ 2497c478bd9Sstevel@tonic-gate 2507c478bd9Sstevel@tonic-gate /* 2517c478bd9Sstevel@tonic-gate * If this platform has a 'core' heap area, then the space for 2527c478bd9Sstevel@tonic-gate * overflow module text should be carved out of the end of that 2537c478bd9Sstevel@tonic-gate * heap. Otherwise, it gets carved out of the general purpose 2547c478bd9Sstevel@tonic-gate * heap. 2557c478bd9Sstevel@tonic-gate */ 2567c478bd9Sstevel@tonic-gate core_size = (uintptr_t)core_end - (uintptr_t)core_start; 2577c478bd9Sstevel@tonic-gate if (core_size > 0) { 2587c478bd9Sstevel@tonic-gate ASSERT(core_size >= HEAPTEXT_SIZE); 2597c478bd9Sstevel@tonic-gate textbase = (uintptr_t)core_end - HEAPTEXT_SIZE; 2607c478bd9Sstevel@tonic-gate core_size -= HEAPTEXT_SIZE; 2617c478bd9Sstevel@tonic-gate } 2627c478bd9Sstevel@tonic-gate #ifndef __sparc 2637c478bd9Sstevel@tonic-gate else { 2647c478bd9Sstevel@tonic-gate ekernelheap -= HEAPTEXT_SIZE; 2657c478bd9Sstevel@tonic-gate textbase = (uintptr_t)ekernelheap; 2667c478bd9Sstevel@tonic-gate } 2677c478bd9Sstevel@tonic-gate #endif 2687c478bd9Sstevel@tonic-gate 2697c478bd9Sstevel@tonic-gate heap_size = (uintptr_t)ekernelheap - (uintptr_t)kernelheap; 2707c478bd9Sstevel@tonic-gate heap_arena = vmem_init("heap", kernelheap, heap_size, PAGESIZE, 2717c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free); 2727c478bd9Sstevel@tonic-gate 2737c478bd9Sstevel@tonic-gate if (core_size > 0) { 2747c478bd9Sstevel@tonic-gate heap_core_arena = vmem_create("heap_core", core_start, 2757c478bd9Sstevel@tonic-gate core_size, PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP); 2767c478bd9Sstevel@tonic-gate heap_core_base = core_start; 2777c478bd9Sstevel@tonic-gate } else { 2787c478bd9Sstevel@tonic-gate heap_core_arena = heap_arena; 2797c478bd9Sstevel@tonic-gate heap_core_base = kernelheap; 2807c478bd9Sstevel@tonic-gate } 2817c478bd9Sstevel@tonic-gate 2827c478bd9Sstevel@tonic-gate /* 2837c478bd9Sstevel@tonic-gate * reserve space for the large page heap. If large pages for kernel 2847c478bd9Sstevel@tonic-gate * heap is enabled large page heap arean will be created later in the 2857c478bd9Sstevel@tonic-gate * boot sequence in segkmem_heap_lp_init(). Otherwise the allocated 2867c478bd9Sstevel@tonic-gate * range will be returned back to the heap_arena. 2877c478bd9Sstevel@tonic-gate */ 2887c478bd9Sstevel@tonic-gate if (heap_lp_size) { 2897c478bd9Sstevel@tonic-gate (void) vmem_xalloc(heap_arena, heap_lp_size, PAGESIZE, 0, 0, 2907c478bd9Sstevel@tonic-gate heap_lp_base, heap_lp_end, 2917c478bd9Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 2927c478bd9Sstevel@tonic-gate } 2937c478bd9Sstevel@tonic-gate 2947c478bd9Sstevel@tonic-gate /* 2957c478bd9Sstevel@tonic-gate * Remove the already-spoken-for memory range [kernelheap, first_avail). 2967c478bd9Sstevel@tonic-gate */ 2977c478bd9Sstevel@tonic-gate (void) vmem_xalloc(heap_arena, first_avail - kernelheap, PAGESIZE, 2987c478bd9Sstevel@tonic-gate 0, 0, kernelheap, first_avail, VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 2997c478bd9Sstevel@tonic-gate 3007c478bd9Sstevel@tonic-gate #ifdef __sparc 3017c478bd9Sstevel@tonic-gate heap32_arena = vmem_create("heap32", (void *)SYSBASE32, 3027c478bd9Sstevel@tonic-gate SYSLIMIT32 - SYSBASE32 - HEAPTEXT_SIZE, PAGESIZE, NULL, 3037c478bd9Sstevel@tonic-gate NULL, NULL, 0, VM_SLEEP); 3047c478bd9Sstevel@tonic-gate 3057c478bd9Sstevel@tonic-gate textbase = SYSLIMIT32 - HEAPTEXT_SIZE; 3067c478bd9Sstevel@tonic-gate heaptext_parent = NULL; 3077c478bd9Sstevel@tonic-gate #else /* __sparc */ 3087c478bd9Sstevel@tonic-gate heap32_arena = heap_core_arena; 3097c478bd9Sstevel@tonic-gate heaptext_parent = heap_core_arena; 3107c478bd9Sstevel@tonic-gate #endif /* __sparc */ 3117c478bd9Sstevel@tonic-gate 3127c478bd9Sstevel@tonic-gate heaptext_arena = vmem_create("heaptext", (void *)textbase, 3137c478bd9Sstevel@tonic-gate HEAPTEXT_SIZE, PAGESIZE, NULL, NULL, heaptext_parent, 0, VM_SLEEP); 3147c478bd9Sstevel@tonic-gate 3157c478bd9Sstevel@tonic-gate /* 3167c478bd9Sstevel@tonic-gate * Create a set of arenas for memory with static translations 3177c478bd9Sstevel@tonic-gate * (e.g. VA -> PA translations cannot change). Since using 3187c478bd9Sstevel@tonic-gate * kernel pages by physical address implies it isn't safe to 3197c478bd9Sstevel@tonic-gate * walk across page boundaries, the static_arena quantum must 3207c478bd9Sstevel@tonic-gate * be PAGESIZE. Any kmem caches that require static memory 3217c478bd9Sstevel@tonic-gate * should source from static_arena, while direct allocations 3227c478bd9Sstevel@tonic-gate * should only use static_alloc_arena. 3237c478bd9Sstevel@tonic-gate */ 3247c478bd9Sstevel@tonic-gate static_arena = vmem_create("static", NULL, 0, PAGESIZE, 3257c478bd9Sstevel@tonic-gate segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP); 3267c478bd9Sstevel@tonic-gate static_alloc_arena = vmem_create("static_alloc", NULL, 0, 3277c478bd9Sstevel@tonic-gate sizeof (uint64_t), vmem_alloc, vmem_free, static_arena, 3287c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 3297c478bd9Sstevel@tonic-gate 3307c478bd9Sstevel@tonic-gate /* 3317c478bd9Sstevel@tonic-gate * Create an arena for translation data (ptes, hmes, or hblks). 3327c478bd9Sstevel@tonic-gate * We need an arena for this because hat_memload() is essential 3337c478bd9Sstevel@tonic-gate * to vmem_populate() (see comments in common/os/vmem.c). 3347c478bd9Sstevel@tonic-gate * 3357c478bd9Sstevel@tonic-gate * Note: any kmem cache that allocates from hat_memload_arena 3367c478bd9Sstevel@tonic-gate * must be created as a KMC_NOHASH cache (i.e. no external slab 3377c478bd9Sstevel@tonic-gate * and bufctl structures to allocate) so that slab creation doesn't 3387c478bd9Sstevel@tonic-gate * require anything more than a single vmem_alloc(). 3397c478bd9Sstevel@tonic-gate */ 3407c478bd9Sstevel@tonic-gate hat_memload_arena = vmem_create("hat_memload", NULL, 0, PAGESIZE, 3417c478bd9Sstevel@tonic-gate hat_memload_alloc, segkmem_free, heap_arena, 0, 3427c478bd9Sstevel@tonic-gate VM_SLEEP | VMC_POPULATOR); 3437c478bd9Sstevel@tonic-gate } 3447c478bd9Sstevel@tonic-gate 3457c478bd9Sstevel@tonic-gate /* 3467c478bd9Sstevel@tonic-gate * Grow kernel heap downward. 3477c478bd9Sstevel@tonic-gate */ 3487c478bd9Sstevel@tonic-gate void 3497c478bd9Sstevel@tonic-gate kernelheap_extend(void *range_start, void *range_end) 3507c478bd9Sstevel@tonic-gate { 3517c478bd9Sstevel@tonic-gate size_t len = (uintptr_t)range_end - (uintptr_t)range_start; 3527c478bd9Sstevel@tonic-gate 3537c478bd9Sstevel@tonic-gate ASSERT(range_start < range_end && range_end == kernelheap); 3547c478bd9Sstevel@tonic-gate 3557c478bd9Sstevel@tonic-gate if (vmem_add(heap_arena, range_start, len, VM_NOSLEEP) == NULL) { 3567c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "Could not grow kernel heap below 0x%p", 3577c478bd9Sstevel@tonic-gate (void *)kernelheap); 3587c478bd9Sstevel@tonic-gate } else { 3597c478bd9Sstevel@tonic-gate kernelheap = range_start; 3607c478bd9Sstevel@tonic-gate } 3617c478bd9Sstevel@tonic-gate } 3627c478bd9Sstevel@tonic-gate 3637c478bd9Sstevel@tonic-gate void 3647c478bd9Sstevel@tonic-gate boot_mapin(caddr_t addr, size_t size) 3657c478bd9Sstevel@tonic-gate { 3667c478bd9Sstevel@tonic-gate caddr_t eaddr; 3677c478bd9Sstevel@tonic-gate page_t *pp; 3687c478bd9Sstevel@tonic-gate pfn_t pfnum; 3697c478bd9Sstevel@tonic-gate 3707c478bd9Sstevel@tonic-gate if (page_resv(btop(size), KM_NOSLEEP) == 0) 3717c478bd9Sstevel@tonic-gate panic("boot_mapin: page_resv failed"); 3727c478bd9Sstevel@tonic-gate 3737c478bd9Sstevel@tonic-gate for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) { 3747c478bd9Sstevel@tonic-gate pfnum = va_to_pfn(addr); 3757c478bd9Sstevel@tonic-gate if ((pp = page_numtopp_nolock(pfnum)) == NULL) 3767c478bd9Sstevel@tonic-gate panic("boot_mapin(): No pp for pfnum = %lx", pfnum); 3777c478bd9Sstevel@tonic-gate 3787c478bd9Sstevel@tonic-gate /* 3797c478bd9Sstevel@tonic-gate * must break up any large pages that may have constituent 3807c478bd9Sstevel@tonic-gate * pages being utilized for BOP_ALLOC()'s before calling 3817c478bd9Sstevel@tonic-gate * page_numtopp().The locking code (ie. page_reclaim()) 3827c478bd9Sstevel@tonic-gate * can't handle them 3837c478bd9Sstevel@tonic-gate */ 3847c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) 3857c478bd9Sstevel@tonic-gate page_boot_demote(pp); 3867c478bd9Sstevel@tonic-gate 3877c478bd9Sstevel@tonic-gate pp = page_numtopp(pfnum, SE_EXCL); 3887c478bd9Sstevel@tonic-gate if (pp == NULL || PP_ISFREE(pp)) 3897c478bd9Sstevel@tonic-gate panic("boot_alloc: pp is NULL or free"); 3907c478bd9Sstevel@tonic-gate 3917c478bd9Sstevel@tonic-gate /* 3927c478bd9Sstevel@tonic-gate * If the cage is on but doesn't yet contain this page, 3937c478bd9Sstevel@tonic-gate * mark it as non-relocatable. 3947c478bd9Sstevel@tonic-gate */ 3957c478bd9Sstevel@tonic-gate if (kcage_on && !PP_ISNORELOC(pp)) 3967c478bd9Sstevel@tonic-gate PP_SETNORELOC(pp); 3977c478bd9Sstevel@tonic-gate 3987c478bd9Sstevel@tonic-gate (void) page_hashin(pp, &kvp, (u_offset_t)(uintptr_t)addr, NULL); 3997c478bd9Sstevel@tonic-gate pp->p_lckcnt = 1; 4007c478bd9Sstevel@tonic-gate #if defined(__x86) 4017c478bd9Sstevel@tonic-gate page_downgrade(pp); 4027c478bd9Sstevel@tonic-gate #else 4037c478bd9Sstevel@tonic-gate page_unlock(pp); 4047c478bd9Sstevel@tonic-gate #endif 4057c478bd9Sstevel@tonic-gate } 4067c478bd9Sstevel@tonic-gate } 4077c478bd9Sstevel@tonic-gate 4087c478bd9Sstevel@tonic-gate /* 4097c478bd9Sstevel@tonic-gate * Get pages from boot and hash them into the kernel's vp. 4107c478bd9Sstevel@tonic-gate * Used after page structs have been allocated, but before segkmem is ready. 4117c478bd9Sstevel@tonic-gate */ 4127c478bd9Sstevel@tonic-gate void * 4137c478bd9Sstevel@tonic-gate boot_alloc(void *inaddr, size_t size, uint_t align) 4147c478bd9Sstevel@tonic-gate { 4157c478bd9Sstevel@tonic-gate caddr_t addr = inaddr; 4167c478bd9Sstevel@tonic-gate 4177c478bd9Sstevel@tonic-gate if (bootops == NULL) 4187c478bd9Sstevel@tonic-gate prom_panic("boot_alloc: attempt to allocate memory after " 4197c478bd9Sstevel@tonic-gate "BOP_GONE"); 4207c478bd9Sstevel@tonic-gate 4217c478bd9Sstevel@tonic-gate size = ptob(btopr(size)); 4227c478bd9Sstevel@tonic-gate if (BOP_ALLOC(bootops, addr, size, align) != addr) 4237c478bd9Sstevel@tonic-gate panic("boot_alloc: BOP_ALLOC failed"); 4247c478bd9Sstevel@tonic-gate boot_mapin((caddr_t)addr, size); 4257c478bd9Sstevel@tonic-gate return (addr); 4267c478bd9Sstevel@tonic-gate } 4277c478bd9Sstevel@tonic-gate 4287c478bd9Sstevel@tonic-gate static void 4297c478bd9Sstevel@tonic-gate segkmem_badop() 4307c478bd9Sstevel@tonic-gate { 4317c478bd9Sstevel@tonic-gate panic("segkmem_badop"); 4327c478bd9Sstevel@tonic-gate } 4337c478bd9Sstevel@tonic-gate 4347c478bd9Sstevel@tonic-gate #define SEGKMEM_BADOP(t) (t(*)())segkmem_badop 4357c478bd9Sstevel@tonic-gate 4367c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 4377c478bd9Sstevel@tonic-gate static faultcode_t 4387c478bd9Sstevel@tonic-gate segkmem_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t size, 4397c478bd9Sstevel@tonic-gate enum fault_type type, enum seg_rw rw) 4407c478bd9Sstevel@tonic-gate { 4414fc2445aSelowe pgcnt_t npages; 4424fc2445aSelowe spgcnt_t pg; 4434fc2445aSelowe page_t *pp; 444ad23a2dbSjohansen struct vnode *vp = seg->s_data; 4454fc2445aSelowe 4467c478bd9Sstevel@tonic-gate ASSERT(RW_READ_HELD(&seg->s_as->a_lock)); 4477c478bd9Sstevel@tonic-gate 4487c478bd9Sstevel@tonic-gate if (seg->s_as != &kas || size > seg->s_size || 4497c478bd9Sstevel@tonic-gate addr < seg->s_base || addr + size > seg->s_base + seg->s_size) 4507c478bd9Sstevel@tonic-gate panic("segkmem_fault: bad args"); 4517c478bd9Sstevel@tonic-gate 4527c478bd9Sstevel@tonic-gate if (segkp_bitmap && seg == &kvseg) { 4537c478bd9Sstevel@tonic-gate /* 4547c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call segkp_fault. 4557c478bd9Sstevel@tonic-gate */ 4567c478bd9Sstevel@tonic-gate if (BT_TEST(segkp_bitmap, 4577c478bd9Sstevel@tonic-gate btop((uintptr_t)(addr - seg->s_base)))) 4587c478bd9Sstevel@tonic-gate return (SEGOP_FAULT(hat, segkp, addr, size, type, rw)); 4597c478bd9Sstevel@tonic-gate } 4607c478bd9Sstevel@tonic-gate 4614fc2445aSelowe if (rw != S_READ && rw != S_WRITE && rw != S_OTHER) 4624fc2445aSelowe return (FC_NOSUPPORT); 4634fc2445aSelowe 4644fc2445aSelowe npages = btopr(size); 4654fc2445aSelowe 4667c478bd9Sstevel@tonic-gate switch (type) { 4677c478bd9Sstevel@tonic-gate case F_SOFTLOCK: /* lock down already-loaded translations */ 4684fc2445aSelowe for (pg = 0; pg < npages; pg++) { 469ad23a2dbSjohansen pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, 4704fc2445aSelowe SE_SHARED); 4714fc2445aSelowe if (pp == NULL) { 4724fc2445aSelowe /* 4734fc2445aSelowe * Hmm, no page. Does a kernel mapping 4744fc2445aSelowe * exist for it? 4754fc2445aSelowe */ 4764fc2445aSelowe if (!hat_probe(kas.a_hat, addr)) { 4774fc2445aSelowe addr -= PAGESIZE; 4784fc2445aSelowe while (--pg >= 0) { 479ad23a2dbSjohansen pp = page_find(vp, 4804fc2445aSelowe (u_offset_t)(uintptr_t)addr); 4814fc2445aSelowe if (pp) 4824fc2445aSelowe page_unlock(pp); 4834fc2445aSelowe addr -= PAGESIZE; 4844fc2445aSelowe } 4854fc2445aSelowe return (FC_NOMAP); 4864fc2445aSelowe } 4874fc2445aSelowe } 4884fc2445aSelowe addr += PAGESIZE; 4894fc2445aSelowe } 4904fc2445aSelowe if (rw == S_OTHER) 4917c478bd9Sstevel@tonic-gate hat_reserve(seg->s_as, addr, size); 4927c478bd9Sstevel@tonic-gate return (0); 4937c478bd9Sstevel@tonic-gate case F_SOFTUNLOCK: 4944fc2445aSelowe while (npages--) { 495ad23a2dbSjohansen pp = page_find(vp, (u_offset_t)(uintptr_t)addr); 4964fc2445aSelowe if (pp) 4974fc2445aSelowe page_unlock(pp); 4984fc2445aSelowe addr += PAGESIZE; 4997c478bd9Sstevel@tonic-gate } 5004fc2445aSelowe return (0); 5014fc2445aSelowe default: 5027c478bd9Sstevel@tonic-gate return (FC_NOSUPPORT); 5037c478bd9Sstevel@tonic-gate } 5044fc2445aSelowe /*NOTREACHED*/ 5054fc2445aSelowe } 5067c478bd9Sstevel@tonic-gate 5077c478bd9Sstevel@tonic-gate static int 5087c478bd9Sstevel@tonic-gate segkmem_setprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot) 5097c478bd9Sstevel@tonic-gate { 5107c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); 5117c478bd9Sstevel@tonic-gate 5127c478bd9Sstevel@tonic-gate if (seg->s_as != &kas || size > seg->s_size || 5137c478bd9Sstevel@tonic-gate addr < seg->s_base || addr + size > seg->s_base + seg->s_size) 5147c478bd9Sstevel@tonic-gate panic("segkmem_setprot: bad args"); 5157c478bd9Sstevel@tonic-gate 5167c478bd9Sstevel@tonic-gate if (segkp_bitmap && seg == &kvseg) { 5177c478bd9Sstevel@tonic-gate 5187c478bd9Sstevel@tonic-gate /* 5197c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call segkp. 5207c478bd9Sstevel@tonic-gate */ 5217c478bd9Sstevel@tonic-gate if (BT_TEST(segkp_bitmap, 5227c478bd9Sstevel@tonic-gate btop((uintptr_t)(addr - seg->s_base)))) 5237c478bd9Sstevel@tonic-gate return (SEGOP_SETPROT(segkp, addr, size, prot)); 5247c478bd9Sstevel@tonic-gate } 5257c478bd9Sstevel@tonic-gate 5267c478bd9Sstevel@tonic-gate if (prot == 0) 5277c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, addr, size, HAT_UNLOAD); 5287c478bd9Sstevel@tonic-gate else 5297c478bd9Sstevel@tonic-gate hat_chgprot(kas.a_hat, addr, size, prot); 5307c478bd9Sstevel@tonic-gate return (0); 5317c478bd9Sstevel@tonic-gate } 5327c478bd9Sstevel@tonic-gate 5337c478bd9Sstevel@tonic-gate /* 5347c478bd9Sstevel@tonic-gate * This is a dummy segkmem function overloaded to call segkp 5357c478bd9Sstevel@tonic-gate * when segkp is under the heap. 5367c478bd9Sstevel@tonic-gate */ 5377c478bd9Sstevel@tonic-gate /* ARGSUSED */ 5387c478bd9Sstevel@tonic-gate static int 5397c478bd9Sstevel@tonic-gate segkmem_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot) 5407c478bd9Sstevel@tonic-gate { 5417c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); 5427c478bd9Sstevel@tonic-gate 5437c478bd9Sstevel@tonic-gate if (seg->s_as != &kas) 5447c478bd9Sstevel@tonic-gate segkmem_badop(); 5457c478bd9Sstevel@tonic-gate 5467c478bd9Sstevel@tonic-gate if (segkp_bitmap && seg == &kvseg) { 5477c478bd9Sstevel@tonic-gate 5487c478bd9Sstevel@tonic-gate /* 5497c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp. 5507c478bd9Sstevel@tonic-gate */ 5517c478bd9Sstevel@tonic-gate if (BT_TEST(segkp_bitmap, 5527c478bd9Sstevel@tonic-gate btop((uintptr_t)(addr - seg->s_base)))) 5537c478bd9Sstevel@tonic-gate return (SEGOP_CHECKPROT(segkp, addr, size, prot)); 5547c478bd9Sstevel@tonic-gate } 5557c478bd9Sstevel@tonic-gate segkmem_badop(); 5567c478bd9Sstevel@tonic-gate return (0); 5577c478bd9Sstevel@tonic-gate } 5587c478bd9Sstevel@tonic-gate 5597c478bd9Sstevel@tonic-gate /* 5607c478bd9Sstevel@tonic-gate * This is a dummy segkmem function overloaded to call segkp 5617c478bd9Sstevel@tonic-gate * when segkp is under the heap. 5627c478bd9Sstevel@tonic-gate */ 5637c478bd9Sstevel@tonic-gate /* ARGSUSED */ 5647c478bd9Sstevel@tonic-gate static int 5657c478bd9Sstevel@tonic-gate segkmem_kluster(struct seg *seg, caddr_t addr, ssize_t delta) 5667c478bd9Sstevel@tonic-gate { 5677c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); 5687c478bd9Sstevel@tonic-gate 5697c478bd9Sstevel@tonic-gate if (seg->s_as != &kas) 5707c478bd9Sstevel@tonic-gate segkmem_badop(); 5717c478bd9Sstevel@tonic-gate 5727c478bd9Sstevel@tonic-gate if (segkp_bitmap && seg == &kvseg) { 5737c478bd9Sstevel@tonic-gate 5747c478bd9Sstevel@tonic-gate /* 5757c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp. 5767c478bd9Sstevel@tonic-gate */ 5777c478bd9Sstevel@tonic-gate if (BT_TEST(segkp_bitmap, 5787c478bd9Sstevel@tonic-gate btop((uintptr_t)(addr - seg->s_base)))) 5797c478bd9Sstevel@tonic-gate return (SEGOP_KLUSTER(segkp, addr, delta)); 5807c478bd9Sstevel@tonic-gate } 5817c478bd9Sstevel@tonic-gate segkmem_badop(); 5827c478bd9Sstevel@tonic-gate return (0); 5837c478bd9Sstevel@tonic-gate } 5847c478bd9Sstevel@tonic-gate 5857c478bd9Sstevel@tonic-gate static void 5867c478bd9Sstevel@tonic-gate segkmem_xdump_range(void *arg, void *start, size_t size) 5877c478bd9Sstevel@tonic-gate { 5887c478bd9Sstevel@tonic-gate struct as *as = arg; 5897c478bd9Sstevel@tonic-gate caddr_t addr = start; 5907c478bd9Sstevel@tonic-gate caddr_t addr_end = addr + size; 5917c478bd9Sstevel@tonic-gate 5927c478bd9Sstevel@tonic-gate while (addr < addr_end) { 5937c478bd9Sstevel@tonic-gate pfn_t pfn = hat_getpfnum(kas.a_hat, addr); 5947c478bd9Sstevel@tonic-gate if (pfn != PFN_INVALID && pfn <= physmax && pf_is_memory(pfn)) 5957c478bd9Sstevel@tonic-gate dump_addpage(as, addr, pfn); 5967c478bd9Sstevel@tonic-gate addr += PAGESIZE; 5977c478bd9Sstevel@tonic-gate dump_timeleft = dump_timeout; 5987c478bd9Sstevel@tonic-gate } 5997c478bd9Sstevel@tonic-gate } 6007c478bd9Sstevel@tonic-gate 6017c478bd9Sstevel@tonic-gate static void 6027c478bd9Sstevel@tonic-gate segkmem_dump_range(void *arg, void *start, size_t size) 6037c478bd9Sstevel@tonic-gate { 6047c478bd9Sstevel@tonic-gate caddr_t addr = start; 6057c478bd9Sstevel@tonic-gate caddr_t addr_end = addr + size; 6067c478bd9Sstevel@tonic-gate 6077c478bd9Sstevel@tonic-gate /* 6087c478bd9Sstevel@tonic-gate * If we are about to start dumping the range of addresses we 6097c478bd9Sstevel@tonic-gate * carved out of the kernel heap for the large page heap walk 6107c478bd9Sstevel@tonic-gate * heap_lp_arena to find what segments are actually populated 6117c478bd9Sstevel@tonic-gate */ 6127c478bd9Sstevel@tonic-gate if (SEGKMEM_USE_LARGEPAGES && 6137c478bd9Sstevel@tonic-gate addr == heap_lp_base && addr_end == heap_lp_end && 6147c478bd9Sstevel@tonic-gate vmem_size(heap_lp_arena, VMEM_ALLOC) < size) { 6157c478bd9Sstevel@tonic-gate vmem_walk(heap_lp_arena, VMEM_ALLOC | VMEM_REENTRANT, 6167c478bd9Sstevel@tonic-gate segkmem_xdump_range, arg); 6177c478bd9Sstevel@tonic-gate } else { 6187c478bd9Sstevel@tonic-gate segkmem_xdump_range(arg, start, size); 6197c478bd9Sstevel@tonic-gate } 6207c478bd9Sstevel@tonic-gate } 6217c478bd9Sstevel@tonic-gate 6227c478bd9Sstevel@tonic-gate static void 6237c478bd9Sstevel@tonic-gate segkmem_dump(struct seg *seg) 6247c478bd9Sstevel@tonic-gate { 6257c478bd9Sstevel@tonic-gate /* 6267c478bd9Sstevel@tonic-gate * The kernel's heap_arena (represented by kvseg) is a very large 6277c478bd9Sstevel@tonic-gate * VA space, most of which is typically unused. To speed up dumping 6287c478bd9Sstevel@tonic-gate * we use vmem_walk() to quickly find the pieces of heap_arena that 6297c478bd9Sstevel@tonic-gate * are actually in use. We do the same for heap32_arena and 6307c478bd9Sstevel@tonic-gate * heap_core. 6317c478bd9Sstevel@tonic-gate * 6327c478bd9Sstevel@tonic-gate * We specify VMEM_REENTRANT to vmem_walk() because dump_addpage() 6337c478bd9Sstevel@tonic-gate * may ultimately need to allocate memory. Reentrant walks are 6347c478bd9Sstevel@tonic-gate * necessarily imperfect snapshots. The kernel heap continues 6357c478bd9Sstevel@tonic-gate * to change during a live crash dump, for example. For a normal 6367c478bd9Sstevel@tonic-gate * crash dump, however, we know that there won't be any other threads 6377c478bd9Sstevel@tonic-gate * messing with the heap. Therefore, at worst, we may fail to dump 6387c478bd9Sstevel@tonic-gate * the pages that get allocated by the act of dumping; but we will 6397c478bd9Sstevel@tonic-gate * always dump every page that was allocated when the walk began. 6407c478bd9Sstevel@tonic-gate * 6417c478bd9Sstevel@tonic-gate * The other segkmem segments are dense (fully populated), so there's 6427c478bd9Sstevel@tonic-gate * no need to use this technique when dumping them. 6437c478bd9Sstevel@tonic-gate * 6447c478bd9Sstevel@tonic-gate * Note: when adding special dump handling for any new sparsely- 6457c478bd9Sstevel@tonic-gate * populated segments, be sure to add similar handling to the ::kgrep 6467c478bd9Sstevel@tonic-gate * code in mdb. 6477c478bd9Sstevel@tonic-gate */ 6487c478bd9Sstevel@tonic-gate if (seg == &kvseg) { 6497c478bd9Sstevel@tonic-gate vmem_walk(heap_arena, VMEM_ALLOC | VMEM_REENTRANT, 6507c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 6517c478bd9Sstevel@tonic-gate #ifndef __sparc 6527c478bd9Sstevel@tonic-gate vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT, 6537c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 6547c478bd9Sstevel@tonic-gate #endif 6557c478bd9Sstevel@tonic-gate } else if (seg == &kvseg_core) { 6567c478bd9Sstevel@tonic-gate vmem_walk(heap_core_arena, VMEM_ALLOC | VMEM_REENTRANT, 6577c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 6587c478bd9Sstevel@tonic-gate } else if (seg == &kvseg32) { 6597c478bd9Sstevel@tonic-gate vmem_walk(heap32_arena, VMEM_ALLOC | VMEM_REENTRANT, 6607c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 6617c478bd9Sstevel@tonic-gate vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT, 6627c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 663ad23a2dbSjohansen } else if (seg == &kzioseg) { 664ad23a2dbSjohansen /* 665ad23a2dbSjohansen * We don't want to dump pages attached to kzioseg since they 666ad23a2dbSjohansen * contain file data from ZFS. If this page's segment is 667ad23a2dbSjohansen * kzioseg return instead of writing it to the dump device. 668ad23a2dbSjohansen */ 669ad23a2dbSjohansen return; 6707c478bd9Sstevel@tonic-gate } else { 6717c478bd9Sstevel@tonic-gate segkmem_dump_range(seg->s_as, seg->s_base, seg->s_size); 6727c478bd9Sstevel@tonic-gate } 6737c478bd9Sstevel@tonic-gate } 6747c478bd9Sstevel@tonic-gate 6757c478bd9Sstevel@tonic-gate /* 6767c478bd9Sstevel@tonic-gate * lock/unlock kmem pages over a given range [addr, addr+len). 6774fc2445aSelowe * Returns a shadow list of pages in ppp. If there are holes 6784fc2445aSelowe * in the range (e.g. some of the kernel mappings do not have 6794fc2445aSelowe * underlying page_ts) returns ENOTSUP so that as_pagelock() 6804fc2445aSelowe * will handle the range via as_fault(F_SOFTLOCK). 6817c478bd9Sstevel@tonic-gate */ 6827c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 6837c478bd9Sstevel@tonic-gate static int 6847c478bd9Sstevel@tonic-gate segkmem_pagelock(struct seg *seg, caddr_t addr, size_t len, 6857c478bd9Sstevel@tonic-gate page_t ***ppp, enum lock_type type, enum seg_rw rw) 6867c478bd9Sstevel@tonic-gate { 6877c478bd9Sstevel@tonic-gate page_t **pplist, *pp; 6887c478bd9Sstevel@tonic-gate pgcnt_t npages; 6894fc2445aSelowe spgcnt_t pg; 6907c478bd9Sstevel@tonic-gate size_t nb; 691ad23a2dbSjohansen struct vnode *vp = seg->s_data; 6927c478bd9Sstevel@tonic-gate 6934fc2445aSelowe ASSERT(ppp != NULL); 6944fc2445aSelowe 6957c478bd9Sstevel@tonic-gate if (segkp_bitmap && seg == &kvseg) { 6967c478bd9Sstevel@tonic-gate /* 6977c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp. 6987c478bd9Sstevel@tonic-gate */ 6997c478bd9Sstevel@tonic-gate if (BT_TEST(segkp_bitmap, 7007c478bd9Sstevel@tonic-gate btop((uintptr_t)(addr - seg->s_base)))) 7017c478bd9Sstevel@tonic-gate return (SEGOP_PAGELOCK(segkp, addr, len, ppp, 7027c478bd9Sstevel@tonic-gate type, rw)); 7037c478bd9Sstevel@tonic-gate } 7047c478bd9Sstevel@tonic-gate 7057c478bd9Sstevel@tonic-gate if (type == L_PAGERECLAIM) 7067c478bd9Sstevel@tonic-gate return (ENOTSUP); 7077c478bd9Sstevel@tonic-gate 7087c478bd9Sstevel@tonic-gate npages = btopr(len); 7097c478bd9Sstevel@tonic-gate nb = sizeof (page_t *) * npages; 7107c478bd9Sstevel@tonic-gate 7117c478bd9Sstevel@tonic-gate if (type == L_PAGEUNLOCK) { 7124fc2445aSelowe pplist = *ppp; 7134fc2445aSelowe ASSERT(pplist != NULL); 7147c478bd9Sstevel@tonic-gate 7154fc2445aSelowe for (pg = 0; pg < npages; pg++) { 7164fc2445aSelowe pp = pplist[pg]; 7177c478bd9Sstevel@tonic-gate page_unlock(pp); 7187c478bd9Sstevel@tonic-gate } 7194fc2445aSelowe kmem_free(pplist, nb); 7207c478bd9Sstevel@tonic-gate return (0); 7217c478bd9Sstevel@tonic-gate } 7227c478bd9Sstevel@tonic-gate 7237c478bd9Sstevel@tonic-gate ASSERT(type == L_PAGELOCK); 7247c478bd9Sstevel@tonic-gate 7254fc2445aSelowe pplist = kmem_alloc(nb, KM_NOSLEEP); 7264fc2445aSelowe if (pplist == NULL) { 7274fc2445aSelowe *ppp = NULL; 7284fc2445aSelowe return (ENOTSUP); /* take the slow path */ 7294fc2445aSelowe } 7307c478bd9Sstevel@tonic-gate 7314fc2445aSelowe for (pg = 0; pg < npages; pg++) { 732ad23a2dbSjohansen pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_SHARED); 7334fc2445aSelowe if (pp == NULL) { 7344fc2445aSelowe while (--pg >= 0) 7354fc2445aSelowe page_unlock(pplist[pg]); 7364fc2445aSelowe kmem_free(pplist, nb); 7374fc2445aSelowe *ppp = NULL; 7384fc2445aSelowe return (ENOTSUP); 7394fc2445aSelowe } 7404fc2445aSelowe pplist[pg] = pp; 7417c478bd9Sstevel@tonic-gate addr += PAGESIZE; 7427c478bd9Sstevel@tonic-gate } 7434fc2445aSelowe 7444fc2445aSelowe *ppp = pplist; 7457c478bd9Sstevel@tonic-gate return (0); 7467c478bd9Sstevel@tonic-gate } 7477c478bd9Sstevel@tonic-gate 7487c478bd9Sstevel@tonic-gate /* 7497c478bd9Sstevel@tonic-gate * This is a dummy segkmem function overloaded to call segkp 7507c478bd9Sstevel@tonic-gate * when segkp is under the heap. 7517c478bd9Sstevel@tonic-gate */ 7527c478bd9Sstevel@tonic-gate /* ARGSUSED */ 7537c478bd9Sstevel@tonic-gate static int 7547c478bd9Sstevel@tonic-gate segkmem_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 7557c478bd9Sstevel@tonic-gate { 7567c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); 7577c478bd9Sstevel@tonic-gate 7587c478bd9Sstevel@tonic-gate if (seg->s_as != &kas) 7597c478bd9Sstevel@tonic-gate segkmem_badop(); 7607c478bd9Sstevel@tonic-gate 7617c478bd9Sstevel@tonic-gate if (segkp_bitmap && seg == &kvseg) { 7627c478bd9Sstevel@tonic-gate 7637c478bd9Sstevel@tonic-gate /* 7647c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp. 7657c478bd9Sstevel@tonic-gate */ 7667c478bd9Sstevel@tonic-gate if (BT_TEST(segkp_bitmap, 7677c478bd9Sstevel@tonic-gate btop((uintptr_t)(addr - seg->s_base)))) 7687c478bd9Sstevel@tonic-gate return (SEGOP_GETMEMID(segkp, addr, memidp)); 7697c478bd9Sstevel@tonic-gate } 7707c478bd9Sstevel@tonic-gate segkmem_badop(); 7717c478bd9Sstevel@tonic-gate return (0); 7727c478bd9Sstevel@tonic-gate } 7737c478bd9Sstevel@tonic-gate 7747c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 7757c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t * 7767c478bd9Sstevel@tonic-gate segkmem_getpolicy(struct seg *seg, caddr_t addr) 7777c478bd9Sstevel@tonic-gate { 7787c478bd9Sstevel@tonic-gate return (NULL); 7797c478bd9Sstevel@tonic-gate } 7807c478bd9Sstevel@tonic-gate 7811bd5c35fSelowe /*ARGSUSED*/ 7821bd5c35fSelowe static int 7831bd5c35fSelowe segkmem_capable(struct seg *seg, segcapability_t capability) 7841bd5c35fSelowe { 7851bd5c35fSelowe if (capability == S_CAPABILITY_NOMINFLT) 7861bd5c35fSelowe return (1); 7871bd5c35fSelowe return (0); 7881bd5c35fSelowe } 7897c478bd9Sstevel@tonic-gate 7907c478bd9Sstevel@tonic-gate static struct seg_ops segkmem_ops = { 7917c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* dup */ 7927c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* unmap */ 7937c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(void), /* free */ 7947c478bd9Sstevel@tonic-gate segkmem_fault, 7957c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(faultcode_t), /* faulta */ 7967c478bd9Sstevel@tonic-gate segkmem_setprot, 7977c478bd9Sstevel@tonic-gate segkmem_checkprot, 7987c478bd9Sstevel@tonic-gate segkmem_kluster, 7997c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(size_t), /* swapout */ 8007c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* sync */ 8017c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(size_t), /* incore */ 8027c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* lockop */ 8037c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* getprot */ 8047c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(u_offset_t), /* getoffset */ 8057c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* gettype */ 8067c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* getvp */ 8077c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* advise */ 8087c478bd9Sstevel@tonic-gate segkmem_dump, 8097c478bd9Sstevel@tonic-gate segkmem_pagelock, 8107c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* setpgsz */ 8117c478bd9Sstevel@tonic-gate segkmem_getmemid, 8127c478bd9Sstevel@tonic-gate segkmem_getpolicy, /* getpolicy */ 8131bd5c35fSelowe segkmem_capable, /* capable */ 8147c478bd9Sstevel@tonic-gate }; 8157c478bd9Sstevel@tonic-gate 8167c478bd9Sstevel@tonic-gate int 817ad23a2dbSjohansen segkmem_zio_create(struct seg *seg) 818ad23a2dbSjohansen { 819ad23a2dbSjohansen ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock)); 820ad23a2dbSjohansen seg->s_ops = &segkmem_ops; 821ad23a2dbSjohansen seg->s_data = &zvp; 822ad23a2dbSjohansen kas.a_size += seg->s_size; 823ad23a2dbSjohansen return (0); 824ad23a2dbSjohansen } 825ad23a2dbSjohansen 826ad23a2dbSjohansen int 8277c478bd9Sstevel@tonic-gate segkmem_create(struct seg *seg) 8287c478bd9Sstevel@tonic-gate { 8297c478bd9Sstevel@tonic-gate ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock)); 8307c478bd9Sstevel@tonic-gate seg->s_ops = &segkmem_ops; 831ad23a2dbSjohansen seg->s_data = &kvp; 8327c478bd9Sstevel@tonic-gate kas.a_size += seg->s_size; 8337c478bd9Sstevel@tonic-gate return (0); 8347c478bd9Sstevel@tonic-gate } 8357c478bd9Sstevel@tonic-gate 8367c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 8377c478bd9Sstevel@tonic-gate page_t * 8387c478bd9Sstevel@tonic-gate segkmem_page_create(void *addr, size_t size, int vmflag, void *arg) 8397c478bd9Sstevel@tonic-gate { 8407c478bd9Sstevel@tonic-gate struct seg kseg; 8417c478bd9Sstevel@tonic-gate int pgflags; 842ad23a2dbSjohansen struct vnode *vp = arg; 843ad23a2dbSjohansen 844ad23a2dbSjohansen if (vp == NULL) 845ad23a2dbSjohansen vp = &kvp; 8467c478bd9Sstevel@tonic-gate 8477c478bd9Sstevel@tonic-gate kseg.s_as = &kas; 8487c478bd9Sstevel@tonic-gate pgflags = PG_EXCL; 8497c478bd9Sstevel@tonic-gate 8507c478bd9Sstevel@tonic-gate if (segkmem_reloc == 0 || (vmflag & VM_NORELOC)) 8517c478bd9Sstevel@tonic-gate pgflags |= PG_NORELOC; 8527c478bd9Sstevel@tonic-gate if ((vmflag & VM_NOSLEEP) == 0) 8537c478bd9Sstevel@tonic-gate pgflags |= PG_WAIT; 8547c478bd9Sstevel@tonic-gate if (vmflag & VM_PANIC) 8557c478bd9Sstevel@tonic-gate pgflags |= PG_PANIC; 8567c478bd9Sstevel@tonic-gate if (vmflag & VM_PUSHPAGE) 8577c478bd9Sstevel@tonic-gate pgflags |= PG_PUSHPAGE; 8587c478bd9Sstevel@tonic-gate 859ad23a2dbSjohansen return (page_create_va(vp, (u_offset_t)(uintptr_t)addr, size, 8607c478bd9Sstevel@tonic-gate pgflags, &kseg, addr)); 8617c478bd9Sstevel@tonic-gate } 8627c478bd9Sstevel@tonic-gate 8637c478bd9Sstevel@tonic-gate /* 8647c478bd9Sstevel@tonic-gate * Allocate pages to back the virtual address range [addr, addr + size). 8657c478bd9Sstevel@tonic-gate * If addr is NULL, allocate the virtual address space as well. 8667c478bd9Sstevel@tonic-gate */ 8677c478bd9Sstevel@tonic-gate void * 8687c478bd9Sstevel@tonic-gate segkmem_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag, uint_t attr, 8697c478bd9Sstevel@tonic-gate page_t *(*page_create_func)(void *, size_t, int, void *), void *pcarg) 8707c478bd9Sstevel@tonic-gate { 8717c478bd9Sstevel@tonic-gate page_t *ppl; 8727c478bd9Sstevel@tonic-gate caddr_t addr = inaddr; 8737c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size); 8747c478bd9Sstevel@tonic-gate int allocflag; 8757c478bd9Sstevel@tonic-gate 8767c478bd9Sstevel@tonic-gate if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL) 8777c478bd9Sstevel@tonic-gate return (NULL); 8787c478bd9Sstevel@tonic-gate 8797c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0); 8807c478bd9Sstevel@tonic-gate 8817c478bd9Sstevel@tonic-gate if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) { 8827c478bd9Sstevel@tonic-gate if (inaddr == NULL) 8837c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size); 8847c478bd9Sstevel@tonic-gate return (NULL); 8857c478bd9Sstevel@tonic-gate } 8867c478bd9Sstevel@tonic-gate 8877c478bd9Sstevel@tonic-gate ppl = page_create_func(addr, size, vmflag, pcarg); 8887c478bd9Sstevel@tonic-gate if (ppl == NULL) { 8897c478bd9Sstevel@tonic-gate if (inaddr == NULL) 8907c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size); 8917c478bd9Sstevel@tonic-gate page_unresv(npages); 8927c478bd9Sstevel@tonic-gate return (NULL); 8937c478bd9Sstevel@tonic-gate } 8947c478bd9Sstevel@tonic-gate 8957c478bd9Sstevel@tonic-gate /* 8967c478bd9Sstevel@tonic-gate * Under certain conditions, we need to let the HAT layer know 8977c478bd9Sstevel@tonic-gate * that it cannot safely allocate memory. Allocations from 8987c478bd9Sstevel@tonic-gate * the hat_memload vmem arena always need this, to prevent 8997c478bd9Sstevel@tonic-gate * infinite recursion. 9007c478bd9Sstevel@tonic-gate * 9017c478bd9Sstevel@tonic-gate * In addition, the x86 hat cannot safely do memory 9027c478bd9Sstevel@tonic-gate * allocations while in vmem_populate(), because there 9037c478bd9Sstevel@tonic-gate * is no simple bound on its usage. 9047c478bd9Sstevel@tonic-gate */ 9057c478bd9Sstevel@tonic-gate if (vmflag & VM_MEMLOAD) 9067c478bd9Sstevel@tonic-gate allocflag = HAT_NO_KALLOC; 9077c478bd9Sstevel@tonic-gate #if defined(__x86) 9087c478bd9Sstevel@tonic-gate else if (vmem_is_populator()) 9097c478bd9Sstevel@tonic-gate allocflag = HAT_NO_KALLOC; 9107c478bd9Sstevel@tonic-gate #endif 9117c478bd9Sstevel@tonic-gate else 9127c478bd9Sstevel@tonic-gate allocflag = 0; 9137c478bd9Sstevel@tonic-gate 9147c478bd9Sstevel@tonic-gate while (ppl != NULL) { 9157c478bd9Sstevel@tonic-gate page_t *pp = ppl; 9167c478bd9Sstevel@tonic-gate page_sub(&ppl, pp); 9177c478bd9Sstevel@tonic-gate ASSERT(page_iolock_assert(pp)); 9187c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 9197c478bd9Sstevel@tonic-gate page_io_unlock(pp); 9207c478bd9Sstevel@tonic-gate hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset, pp, 9217c478bd9Sstevel@tonic-gate (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr, 9227c478bd9Sstevel@tonic-gate HAT_LOAD_LOCK | allocflag); 9237c478bd9Sstevel@tonic-gate pp->p_lckcnt = 1; 9247c478bd9Sstevel@tonic-gate #if defined(__x86) 9257c478bd9Sstevel@tonic-gate page_downgrade(pp); 9267c478bd9Sstevel@tonic-gate #else 9277c478bd9Sstevel@tonic-gate if (vmflag & SEGKMEM_SHARELOCKED) 9287c478bd9Sstevel@tonic-gate page_downgrade(pp); 9297c478bd9Sstevel@tonic-gate else 9307c478bd9Sstevel@tonic-gate page_unlock(pp); 9317c478bd9Sstevel@tonic-gate #endif 9327c478bd9Sstevel@tonic-gate } 9337c478bd9Sstevel@tonic-gate 9347c478bd9Sstevel@tonic-gate return (addr); 9357c478bd9Sstevel@tonic-gate } 9367c478bd9Sstevel@tonic-gate 937ad23a2dbSjohansen static void * 938ad23a2dbSjohansen segkmem_alloc_vn(vmem_t *vmp, size_t size, int vmflag, struct vnode *vp) 9397c478bd9Sstevel@tonic-gate { 9407c478bd9Sstevel@tonic-gate void *addr; 9417c478bd9Sstevel@tonic-gate segkmem_gc_list_t *gcp, **prev_gcpp; 9427c478bd9Sstevel@tonic-gate 943ad23a2dbSjohansen ASSERT(vp != NULL); 944ad23a2dbSjohansen 9457c478bd9Sstevel@tonic-gate if (kvseg.s_base == NULL) { 9467c478bd9Sstevel@tonic-gate #ifndef __sparc 9477c478bd9Sstevel@tonic-gate if (bootops->bsys_alloc == NULL) 9487c478bd9Sstevel@tonic-gate halt("Memory allocation between bop_alloc() and " 9497c478bd9Sstevel@tonic-gate "kmem_alloc().\n"); 9507c478bd9Sstevel@tonic-gate #endif 9517c478bd9Sstevel@tonic-gate 9527c478bd9Sstevel@tonic-gate /* 9537c478bd9Sstevel@tonic-gate * There's not a lot of memory to go around during boot, 9547c478bd9Sstevel@tonic-gate * so recycle it if we can. 9557c478bd9Sstevel@tonic-gate */ 9567c478bd9Sstevel@tonic-gate for (prev_gcpp = &segkmem_gc_list; (gcp = *prev_gcpp) != NULL; 9577c478bd9Sstevel@tonic-gate prev_gcpp = &gcp->gc_next) { 9587c478bd9Sstevel@tonic-gate if (gcp->gc_arena == vmp && gcp->gc_size == size) { 9597c478bd9Sstevel@tonic-gate *prev_gcpp = gcp->gc_next; 9607c478bd9Sstevel@tonic-gate return (gcp); 9617c478bd9Sstevel@tonic-gate } 9627c478bd9Sstevel@tonic-gate } 9637c478bd9Sstevel@tonic-gate 9647c478bd9Sstevel@tonic-gate addr = vmem_alloc(vmp, size, vmflag | VM_PANIC); 9657c478bd9Sstevel@tonic-gate if (boot_alloc(addr, size, BO_NO_ALIGN) != addr) 9667c478bd9Sstevel@tonic-gate panic("segkmem_alloc: boot_alloc failed"); 9677c478bd9Sstevel@tonic-gate return (addr); 9687c478bd9Sstevel@tonic-gate } 9697c478bd9Sstevel@tonic-gate return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, 970ad23a2dbSjohansen segkmem_page_create, vp)); 971ad23a2dbSjohansen } 972ad23a2dbSjohansen 973ad23a2dbSjohansen void * 974ad23a2dbSjohansen segkmem_alloc(vmem_t *vmp, size_t size, int vmflag) 975ad23a2dbSjohansen { 976ad23a2dbSjohansen return (segkmem_alloc_vn(vmp, size, vmflag, &kvp)); 977ad23a2dbSjohansen } 978ad23a2dbSjohansen 979ad23a2dbSjohansen void * 980ad23a2dbSjohansen segkmem_zio_alloc(vmem_t *vmp, size_t size, int vmflag) 981ad23a2dbSjohansen { 982ad23a2dbSjohansen return (segkmem_alloc_vn(vmp, size, vmflag, &zvp)); 9837c478bd9Sstevel@tonic-gate } 9847c478bd9Sstevel@tonic-gate 9857c478bd9Sstevel@tonic-gate /* 9867c478bd9Sstevel@tonic-gate * Any changes to this routine must also be carried over to 9877c478bd9Sstevel@tonic-gate * devmap_free_pages() in the seg_dev driver. This is because 9887c478bd9Sstevel@tonic-gate * we currently don't have a special kernel segment for non-paged 9897c478bd9Sstevel@tonic-gate * kernel memory that is exported by drivers to user space. 9907c478bd9Sstevel@tonic-gate */ 991ad23a2dbSjohansen static void 992ad23a2dbSjohansen segkmem_free_vn(vmem_t *vmp, void *inaddr, size_t size, struct vnode *vp) 9937c478bd9Sstevel@tonic-gate { 9947c478bd9Sstevel@tonic-gate page_t *pp; 9957c478bd9Sstevel@tonic-gate caddr_t addr = inaddr; 9967c478bd9Sstevel@tonic-gate caddr_t eaddr; 9977c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size); 9987c478bd9Sstevel@tonic-gate 9997c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0); 1000ad23a2dbSjohansen ASSERT(vp != NULL); 10017c478bd9Sstevel@tonic-gate 10027c478bd9Sstevel@tonic-gate if (kvseg.s_base == NULL) { 10037c478bd9Sstevel@tonic-gate segkmem_gc_list_t *gc = inaddr; 10047c478bd9Sstevel@tonic-gate gc->gc_arena = vmp; 10057c478bd9Sstevel@tonic-gate gc->gc_size = size; 10067c478bd9Sstevel@tonic-gate gc->gc_next = segkmem_gc_list; 10077c478bd9Sstevel@tonic-gate segkmem_gc_list = gc; 10087c478bd9Sstevel@tonic-gate return; 10097c478bd9Sstevel@tonic-gate } 10107c478bd9Sstevel@tonic-gate 10117c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK); 10127c478bd9Sstevel@tonic-gate 10137c478bd9Sstevel@tonic-gate for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) { 10147c478bd9Sstevel@tonic-gate #if defined(__x86) 1015ad23a2dbSjohansen pp = page_find(vp, (u_offset_t)(uintptr_t)addr); 10167c478bd9Sstevel@tonic-gate if (pp == NULL) 10177c478bd9Sstevel@tonic-gate panic("segkmem_free: page not found"); 10187c478bd9Sstevel@tonic-gate if (!page_tryupgrade(pp)) { 10197c478bd9Sstevel@tonic-gate /* 10207c478bd9Sstevel@tonic-gate * Some other thread has a sharelock. Wait for 10217c478bd9Sstevel@tonic-gate * it to drop the lock so we can free this page. 10227c478bd9Sstevel@tonic-gate */ 10237c478bd9Sstevel@tonic-gate page_unlock(pp); 1024ad23a2dbSjohansen pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, 10257c478bd9Sstevel@tonic-gate SE_EXCL); 10267c478bd9Sstevel@tonic-gate } 10277c478bd9Sstevel@tonic-gate #else 1028ad23a2dbSjohansen pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_EXCL); 10297c478bd9Sstevel@tonic-gate #endif 10307c478bd9Sstevel@tonic-gate if (pp == NULL) 10317c478bd9Sstevel@tonic-gate panic("segkmem_free: page not found"); 10327c478bd9Sstevel@tonic-gate /* Clear p_lckcnt so page_destroy() doesn't update availrmem */ 10337c478bd9Sstevel@tonic-gate pp->p_lckcnt = 0; 10347c478bd9Sstevel@tonic-gate page_destroy(pp, 0); 10357c478bd9Sstevel@tonic-gate } 10367c478bd9Sstevel@tonic-gate page_unresv(npages); 10377c478bd9Sstevel@tonic-gate 10387c478bd9Sstevel@tonic-gate if (vmp != NULL) 10397c478bd9Sstevel@tonic-gate vmem_free(vmp, inaddr, size); 1040ad23a2dbSjohansen 1041ad23a2dbSjohansen } 1042ad23a2dbSjohansen 1043ad23a2dbSjohansen void 1044ad23a2dbSjohansen segkmem_free(vmem_t *vmp, void *inaddr, size_t size) 1045ad23a2dbSjohansen { 1046ad23a2dbSjohansen segkmem_free_vn(vmp, inaddr, size, &kvp); 1047ad23a2dbSjohansen } 1048ad23a2dbSjohansen 1049ad23a2dbSjohansen void 1050ad23a2dbSjohansen segkmem_zio_free(vmem_t *vmp, void *inaddr, size_t size) 1051ad23a2dbSjohansen { 1052ad23a2dbSjohansen segkmem_free_vn(vmp, inaddr, size, &zvp); 10537c478bd9Sstevel@tonic-gate } 10547c478bd9Sstevel@tonic-gate 10557c478bd9Sstevel@tonic-gate void 10567c478bd9Sstevel@tonic-gate segkmem_gc(void) 10577c478bd9Sstevel@tonic-gate { 10587c478bd9Sstevel@tonic-gate ASSERT(kvseg.s_base != NULL); 10597c478bd9Sstevel@tonic-gate while (segkmem_gc_list != NULL) { 10607c478bd9Sstevel@tonic-gate segkmem_gc_list_t *gc = segkmem_gc_list; 10617c478bd9Sstevel@tonic-gate segkmem_gc_list = gc->gc_next; 10627c478bd9Sstevel@tonic-gate segkmem_free(gc->gc_arena, gc, gc->gc_size); 10637c478bd9Sstevel@tonic-gate } 10647c478bd9Sstevel@tonic-gate } 10657c478bd9Sstevel@tonic-gate 10667c478bd9Sstevel@tonic-gate /* 10677c478bd9Sstevel@tonic-gate * Legacy entry points from here to end of file. 10687c478bd9Sstevel@tonic-gate */ 10697c478bd9Sstevel@tonic-gate void 10707c478bd9Sstevel@tonic-gate segkmem_mapin(struct seg *seg, void *addr, size_t size, uint_t vprot, 10717c478bd9Sstevel@tonic-gate pfn_t pfn, uint_t flags) 10727c478bd9Sstevel@tonic-gate { 10737c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK); 10747c478bd9Sstevel@tonic-gate hat_devload(seg->s_as->a_hat, addr, size, pfn, vprot, 10757c478bd9Sstevel@tonic-gate flags | HAT_LOAD_LOCK); 10767c478bd9Sstevel@tonic-gate } 10777c478bd9Sstevel@tonic-gate 10787c478bd9Sstevel@tonic-gate void 10797c478bd9Sstevel@tonic-gate segkmem_mapout(struct seg *seg, void *addr, size_t size) 10807c478bd9Sstevel@tonic-gate { 10817c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK); 10827c478bd9Sstevel@tonic-gate } 10837c478bd9Sstevel@tonic-gate 10847c478bd9Sstevel@tonic-gate void * 10857c478bd9Sstevel@tonic-gate kmem_getpages(pgcnt_t npages, int kmflag) 10867c478bd9Sstevel@tonic-gate { 10877c478bd9Sstevel@tonic-gate return (kmem_alloc(ptob(npages), kmflag)); 10887c478bd9Sstevel@tonic-gate } 10897c478bd9Sstevel@tonic-gate 10907c478bd9Sstevel@tonic-gate void 10917c478bd9Sstevel@tonic-gate kmem_freepages(void *addr, pgcnt_t npages) 10927c478bd9Sstevel@tonic-gate { 10937c478bd9Sstevel@tonic-gate kmem_free(addr, ptob(npages)); 10947c478bd9Sstevel@tonic-gate } 10957c478bd9Sstevel@tonic-gate 10967c478bd9Sstevel@tonic-gate /* 10977c478bd9Sstevel@tonic-gate * segkmem_page_create_large() allocates a large page to be used for the kmem 10987c478bd9Sstevel@tonic-gate * caches. If kpr is enabled we ask for a relocatable page unless requested 10997c478bd9Sstevel@tonic-gate * otherwise. If kpr is disabled we have to ask for a non-reloc page 11007c478bd9Sstevel@tonic-gate */ 11017c478bd9Sstevel@tonic-gate static page_t * 11027c478bd9Sstevel@tonic-gate segkmem_page_create_large(void *addr, size_t size, int vmflag, void *arg) 11037c478bd9Sstevel@tonic-gate { 11047c478bd9Sstevel@tonic-gate int pgflags; 11057c478bd9Sstevel@tonic-gate 11067c478bd9Sstevel@tonic-gate pgflags = PG_EXCL; 11077c478bd9Sstevel@tonic-gate 11087c478bd9Sstevel@tonic-gate if (segkmem_reloc == 0 || (vmflag & VM_NORELOC)) 11097c478bd9Sstevel@tonic-gate pgflags |= PG_NORELOC; 11107c478bd9Sstevel@tonic-gate if (!(vmflag & VM_NOSLEEP)) 11117c478bd9Sstevel@tonic-gate pgflags |= PG_WAIT; 11127c478bd9Sstevel@tonic-gate if (vmflag & VM_PUSHPAGE) 11137c478bd9Sstevel@tonic-gate pgflags |= PG_PUSHPAGE; 11147c478bd9Sstevel@tonic-gate 11157c478bd9Sstevel@tonic-gate return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size, 11167c478bd9Sstevel@tonic-gate pgflags, &kvseg, addr, arg)); 11177c478bd9Sstevel@tonic-gate } 11187c478bd9Sstevel@tonic-gate 11197c478bd9Sstevel@tonic-gate /* 11207c478bd9Sstevel@tonic-gate * Allocate a large page to back the virtual address range 11217c478bd9Sstevel@tonic-gate * [addr, addr + size). If addr is NULL, allocate the virtual address 11227c478bd9Sstevel@tonic-gate * space as well. 11237c478bd9Sstevel@tonic-gate */ 11247c478bd9Sstevel@tonic-gate static void * 11257c478bd9Sstevel@tonic-gate segkmem_xalloc_lp(vmem_t *vmp, void *inaddr, size_t size, int vmflag, 11267c478bd9Sstevel@tonic-gate uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *), 11277c478bd9Sstevel@tonic-gate void *pcarg) 11287c478bd9Sstevel@tonic-gate { 11297c478bd9Sstevel@tonic-gate caddr_t addr = inaddr, pa; 11307c478bd9Sstevel@tonic-gate size_t lpsize = segkmem_lpsize; 11317c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size); 11327c478bd9Sstevel@tonic-gate pgcnt_t nbpages = btop(lpsize); 11337c478bd9Sstevel@tonic-gate pgcnt_t nlpages = size >> segkmem_lpshift; 11347c478bd9Sstevel@tonic-gate size_t ppasize = nbpages * sizeof (page_t *); 11357c478bd9Sstevel@tonic-gate page_t *pp, *rootpp, **ppa, *pplist = NULL; 11367c478bd9Sstevel@tonic-gate int i; 11377c478bd9Sstevel@tonic-gate 11387c478bd9Sstevel@tonic-gate if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) { 11397c478bd9Sstevel@tonic-gate return (NULL); 11407c478bd9Sstevel@tonic-gate } 11417c478bd9Sstevel@tonic-gate 11427c478bd9Sstevel@tonic-gate /* 11437c478bd9Sstevel@tonic-gate * allocate an array we need for hat_memload_array. 11447c478bd9Sstevel@tonic-gate * we use a separate arena to avoid recursion. 11457c478bd9Sstevel@tonic-gate * we will not need this array when hat_memload_array learns pp++ 11467c478bd9Sstevel@tonic-gate */ 11477c478bd9Sstevel@tonic-gate if ((ppa = vmem_alloc(segkmem_ppa_arena, ppasize, vmflag)) == NULL) { 11487c478bd9Sstevel@tonic-gate goto fail_array_alloc; 11497c478bd9Sstevel@tonic-gate } 11507c478bd9Sstevel@tonic-gate 11517c478bd9Sstevel@tonic-gate if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL) 11527c478bd9Sstevel@tonic-gate goto fail_vmem_alloc; 11537c478bd9Sstevel@tonic-gate 11547c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & (lpsize - 1)) == 0); 11557c478bd9Sstevel@tonic-gate 11567c478bd9Sstevel@tonic-gate /* create all the pages */ 11577c478bd9Sstevel@tonic-gate for (pa = addr, i = 0; i < nlpages; i++, pa += lpsize) { 11587c478bd9Sstevel@tonic-gate if ((pp = page_create_func(pa, lpsize, vmflag, pcarg)) == NULL) 11597c478bd9Sstevel@tonic-gate goto fail_page_create; 11607c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &pp); 11617c478bd9Sstevel@tonic-gate } 11627c478bd9Sstevel@tonic-gate 11637c478bd9Sstevel@tonic-gate /* at this point we have all the resource to complete the request */ 11647c478bd9Sstevel@tonic-gate while ((rootpp = pplist) != NULL) { 11657c478bd9Sstevel@tonic-gate for (i = 0; i < nbpages; i++) { 11667c478bd9Sstevel@tonic-gate ASSERT(pplist != NULL); 11677c478bd9Sstevel@tonic-gate pp = pplist; 11687c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 11697c478bd9Sstevel@tonic-gate ASSERT(page_iolock_assert(pp)); 11707c478bd9Sstevel@tonic-gate page_io_unlock(pp); 11717c478bd9Sstevel@tonic-gate ppa[i] = pp; 11727c478bd9Sstevel@tonic-gate } 11737c478bd9Sstevel@tonic-gate /* 11747c478bd9Sstevel@tonic-gate * Load the locked entry. It's OK to preload the entry into the 11757c478bd9Sstevel@tonic-gate * TSB since we now support large mappings in the kernel TSB. 11767c478bd9Sstevel@tonic-gate */ 11777c478bd9Sstevel@tonic-gate hat_memload_array(kas.a_hat, 11787c478bd9Sstevel@tonic-gate (caddr_t)(uintptr_t)rootpp->p_offset, lpsize, 11797c478bd9Sstevel@tonic-gate ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr, 11807c478bd9Sstevel@tonic-gate HAT_LOAD_LOCK); 11817c478bd9Sstevel@tonic-gate 11827c478bd9Sstevel@tonic-gate for (--i; i >= 0; --i) { 11837c478bd9Sstevel@tonic-gate ppa[i]->p_lckcnt = 1; 11847c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 11857c478bd9Sstevel@tonic-gate } 11867c478bd9Sstevel@tonic-gate } 11877c478bd9Sstevel@tonic-gate 11887c478bd9Sstevel@tonic-gate vmem_free(segkmem_ppa_arena, ppa, ppasize); 11897c478bd9Sstevel@tonic-gate return (addr); 11907c478bd9Sstevel@tonic-gate 11917c478bd9Sstevel@tonic-gate fail_page_create: 11927c478bd9Sstevel@tonic-gate while ((rootpp = pplist) != NULL) { 11937c478bd9Sstevel@tonic-gate for (i = 0, pp = pplist; i < nbpages; i++, pp = pplist) { 11947c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 11957c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 11967c478bd9Sstevel@tonic-gate ASSERT(page_iolock_assert(pp)); 11977c478bd9Sstevel@tonic-gate page_io_unlock(pp); 11987c478bd9Sstevel@tonic-gate } 11997c478bd9Sstevel@tonic-gate page_destroy_pages(rootpp); 12007c478bd9Sstevel@tonic-gate } 12017c478bd9Sstevel@tonic-gate 12027c478bd9Sstevel@tonic-gate if (inaddr == NULL) 12037c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size); 12047c478bd9Sstevel@tonic-gate 12057c478bd9Sstevel@tonic-gate fail_vmem_alloc: 12067c478bd9Sstevel@tonic-gate vmem_free(segkmem_ppa_arena, ppa, ppasize); 12077c478bd9Sstevel@tonic-gate 12087c478bd9Sstevel@tonic-gate fail_array_alloc: 12097c478bd9Sstevel@tonic-gate page_unresv(npages); 12107c478bd9Sstevel@tonic-gate 12117c478bd9Sstevel@tonic-gate return (NULL); 12127c478bd9Sstevel@tonic-gate } 12137c478bd9Sstevel@tonic-gate 12147c478bd9Sstevel@tonic-gate static void 12157c478bd9Sstevel@tonic-gate segkmem_free_one_lp(caddr_t addr, size_t size) 12167c478bd9Sstevel@tonic-gate { 12177c478bd9Sstevel@tonic-gate page_t *pp, *rootpp = NULL; 12187c478bd9Sstevel@tonic-gate pgcnt_t pgs_left = btopr(size); 12197c478bd9Sstevel@tonic-gate 12207c478bd9Sstevel@tonic-gate ASSERT(size == segkmem_lpsize); 12217c478bd9Sstevel@tonic-gate 12227c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK); 12237c478bd9Sstevel@tonic-gate 12247c478bd9Sstevel@tonic-gate for (; pgs_left > 0; addr += PAGESIZE, pgs_left--) { 12257c478bd9Sstevel@tonic-gate pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL); 12267c478bd9Sstevel@tonic-gate if (pp == NULL) 12277c478bd9Sstevel@tonic-gate panic("segkmem_free_one_lp: page not found"); 12287c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 12297c478bd9Sstevel@tonic-gate pp->p_lckcnt = 0; 12307c478bd9Sstevel@tonic-gate if (rootpp == NULL) 12317c478bd9Sstevel@tonic-gate rootpp = pp; 12327c478bd9Sstevel@tonic-gate } 12337c478bd9Sstevel@tonic-gate ASSERT(rootpp != NULL); 12347c478bd9Sstevel@tonic-gate page_destroy_pages(rootpp); 12357c478bd9Sstevel@tonic-gate 12367c478bd9Sstevel@tonic-gate /* page_unresv() is done by the caller */ 12377c478bd9Sstevel@tonic-gate } 12387c478bd9Sstevel@tonic-gate 12397c478bd9Sstevel@tonic-gate /* 12407c478bd9Sstevel@tonic-gate * This function is called to import new spans into the vmem arenas like 12417c478bd9Sstevel@tonic-gate * kmem_default_arena and kmem_oversize_arena. It first tries to import 12427c478bd9Sstevel@tonic-gate * spans from large page arena - kmem_lp_arena. In order to do this it might 12437c478bd9Sstevel@tonic-gate * have to "upgrade the requested size" to kmem_lp_arena quantum. If 12447c478bd9Sstevel@tonic-gate * it was not able to satisfy the upgraded request it then calls regular 12457c478bd9Sstevel@tonic-gate * segkmem_alloc() that satisfies the request by importing from "*vmp" arena 12467c478bd9Sstevel@tonic-gate */ 12477c478bd9Sstevel@tonic-gate void * 12487c478bd9Sstevel@tonic-gate segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, int vmflag) 12497c478bd9Sstevel@tonic-gate { 12507c478bd9Sstevel@tonic-gate size_t size; 12517c478bd9Sstevel@tonic-gate kthread_t *t = curthread; 12527c478bd9Sstevel@tonic-gate segkmem_lpcb_t *lpcb = &segkmem_lpcb; 12537c478bd9Sstevel@tonic-gate 12547c478bd9Sstevel@tonic-gate ASSERT(sizep != NULL); 12557c478bd9Sstevel@tonic-gate 12567c478bd9Sstevel@tonic-gate size = *sizep; 12577c478bd9Sstevel@tonic-gate 12587c478bd9Sstevel@tonic-gate if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) && 12597c478bd9Sstevel@tonic-gate !(vmflag & SEGKMEM_SHARELOCKED)) { 12607c478bd9Sstevel@tonic-gate 12617c478bd9Sstevel@tonic-gate size_t kmemlp_qnt = segkmem_kmemlp_quantum; 12627c478bd9Sstevel@tonic-gate size_t asize = P2ROUNDUP(size, kmemlp_qnt); 12637c478bd9Sstevel@tonic-gate void *addr = NULL; 12647c478bd9Sstevel@tonic-gate ulong_t *lpthrtp = &lpcb->lp_throttle; 12657c478bd9Sstevel@tonic-gate ulong_t lpthrt = *lpthrtp; 12667c478bd9Sstevel@tonic-gate int dowakeup = 0; 12677c478bd9Sstevel@tonic-gate int doalloc = 1; 12687c478bd9Sstevel@tonic-gate 12697c478bd9Sstevel@tonic-gate ASSERT(kmem_lp_arena != NULL); 12707c478bd9Sstevel@tonic-gate ASSERT(asize >= size); 12717c478bd9Sstevel@tonic-gate 12727c478bd9Sstevel@tonic-gate if (lpthrt != 0) { 12737c478bd9Sstevel@tonic-gate /* try to update the throttle value */ 12747c478bd9Sstevel@tonic-gate lpthrt = atomic_add_long_nv(lpthrtp, 1); 12757c478bd9Sstevel@tonic-gate if (lpthrt >= segkmem_lpthrottle_max) { 12767c478bd9Sstevel@tonic-gate lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 12777c478bd9Sstevel@tonic-gate segkmem_lpthrottle_max / 4); 12787c478bd9Sstevel@tonic-gate } 12797c478bd9Sstevel@tonic-gate 12807c478bd9Sstevel@tonic-gate /* 12817c478bd9Sstevel@tonic-gate * when we get above throttle start do an exponential 12827c478bd9Sstevel@tonic-gate * backoff at trying large pages and reaping 12837c478bd9Sstevel@tonic-gate */ 12847c478bd9Sstevel@tonic-gate if (lpthrt > segkmem_lpthrottle_start && 12857c478bd9Sstevel@tonic-gate (lpthrt & (lpthrt - 1))) { 1286501f3284Seg155566 lpcb->allocs_throttled++; 12877c478bd9Sstevel@tonic-gate lpthrt--; 12887c478bd9Sstevel@tonic-gate if ((lpthrt & (lpthrt - 1)) == 0) 12897c478bd9Sstevel@tonic-gate kmem_reap(); 12907c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, vmflag)); 12917c478bd9Sstevel@tonic-gate } 12927c478bd9Sstevel@tonic-gate } 12937c478bd9Sstevel@tonic-gate 12947c478bd9Sstevel@tonic-gate if (!(vmflag & VM_NOSLEEP) && 12957c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum >= (8 * kmemlp_qnt) && 12967c478bd9Sstevel@tonic-gate vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt && 12977c478bd9Sstevel@tonic-gate asize < (segkmem_heaplp_quantum - kmemlp_qnt)) { 12987c478bd9Sstevel@tonic-gate 12997c478bd9Sstevel@tonic-gate /* 13007c478bd9Sstevel@tonic-gate * we are low on free memory in kmem_lp_arena 13017c478bd9Sstevel@tonic-gate * we let only one guy to allocate heap_lp 13027c478bd9Sstevel@tonic-gate * quantum size chunk that everybody is going to 13037c478bd9Sstevel@tonic-gate * share 13047c478bd9Sstevel@tonic-gate */ 13057c478bd9Sstevel@tonic-gate mutex_enter(&lpcb->lp_lock); 13067c478bd9Sstevel@tonic-gate 13077c478bd9Sstevel@tonic-gate if (lpcb->lp_wait) { 13087c478bd9Sstevel@tonic-gate 13097c478bd9Sstevel@tonic-gate /* we are not the first one - wait */ 13107c478bd9Sstevel@tonic-gate cv_wait(&lpcb->lp_cv, &lpcb->lp_lock); 13117c478bd9Sstevel@tonic-gate if (vmem_size(kmem_lp_arena, VMEM_FREE) < 13127c478bd9Sstevel@tonic-gate kmemlp_qnt) { 13137c478bd9Sstevel@tonic-gate doalloc = 0; 13147c478bd9Sstevel@tonic-gate } 13157c478bd9Sstevel@tonic-gate } else if (vmem_size(kmem_lp_arena, VMEM_FREE) <= 13167c478bd9Sstevel@tonic-gate kmemlp_qnt) { 13177c478bd9Sstevel@tonic-gate 13187c478bd9Sstevel@tonic-gate /* 13197c478bd9Sstevel@tonic-gate * we are the first one, make sure we import 13207c478bd9Sstevel@tonic-gate * a large page 13217c478bd9Sstevel@tonic-gate */ 13227c478bd9Sstevel@tonic-gate if (asize == kmemlp_qnt) 13237c478bd9Sstevel@tonic-gate asize += kmemlp_qnt; 13247c478bd9Sstevel@tonic-gate dowakeup = 1; 13257c478bd9Sstevel@tonic-gate lpcb->lp_wait = 1; 13267c478bd9Sstevel@tonic-gate } 13277c478bd9Sstevel@tonic-gate 13287c478bd9Sstevel@tonic-gate mutex_exit(&lpcb->lp_lock); 13297c478bd9Sstevel@tonic-gate } 13307c478bd9Sstevel@tonic-gate 13317c478bd9Sstevel@tonic-gate /* 13327c478bd9Sstevel@tonic-gate * VM_ABORT flag prevents sleeps in vmem_xalloc when 13337c478bd9Sstevel@tonic-gate * large pages are not available. In that case this allocation 13347c478bd9Sstevel@tonic-gate * attempt will fail and we will retry allocation with small 13357c478bd9Sstevel@tonic-gate * pages. We also do not want to panic if this allocation fails 13367c478bd9Sstevel@tonic-gate * because we are going to retry. 13377c478bd9Sstevel@tonic-gate */ 13387c478bd9Sstevel@tonic-gate if (doalloc) { 13397c478bd9Sstevel@tonic-gate addr = vmem_alloc(kmem_lp_arena, asize, 13407c478bd9Sstevel@tonic-gate (vmflag | VM_ABORT) & ~VM_PANIC); 13417c478bd9Sstevel@tonic-gate 13427c478bd9Sstevel@tonic-gate if (dowakeup) { 13437c478bd9Sstevel@tonic-gate mutex_enter(&lpcb->lp_lock); 13447c478bd9Sstevel@tonic-gate ASSERT(lpcb->lp_wait != 0); 13457c478bd9Sstevel@tonic-gate lpcb->lp_wait = 0; 13467c478bd9Sstevel@tonic-gate cv_broadcast(&lpcb->lp_cv); 13477c478bd9Sstevel@tonic-gate mutex_exit(&lpcb->lp_lock); 13487c478bd9Sstevel@tonic-gate } 13497c478bd9Sstevel@tonic-gate } 13507c478bd9Sstevel@tonic-gate 13517c478bd9Sstevel@tonic-gate if (addr != NULL) { 13527c478bd9Sstevel@tonic-gate *sizep = asize; 13537c478bd9Sstevel@tonic-gate *lpthrtp = 0; 13547c478bd9Sstevel@tonic-gate return (addr); 13557c478bd9Sstevel@tonic-gate } 13567c478bd9Sstevel@tonic-gate 13577c478bd9Sstevel@tonic-gate if (vmflag & VM_NOSLEEP) 1358501f3284Seg155566 lpcb->nosleep_allocs_failed++; 13597c478bd9Sstevel@tonic-gate else 1360501f3284Seg155566 lpcb->sleep_allocs_failed++; 1361501f3284Seg155566 lpcb->alloc_bytes_failed += size; 13627c478bd9Sstevel@tonic-gate 13637c478bd9Sstevel@tonic-gate /* if large page throttling is not started yet do it */ 13647c478bd9Sstevel@tonic-gate if (segkmem_use_lpthrottle && lpthrt == 0) { 13657c478bd9Sstevel@tonic-gate lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1); 13667c478bd9Sstevel@tonic-gate } 13677c478bd9Sstevel@tonic-gate } 13687c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, vmflag)); 13697c478bd9Sstevel@tonic-gate } 13707c478bd9Sstevel@tonic-gate 13717c478bd9Sstevel@tonic-gate void 13727c478bd9Sstevel@tonic-gate segkmem_free_lp(vmem_t *vmp, void *inaddr, size_t size) 13737c478bd9Sstevel@tonic-gate { 13747c478bd9Sstevel@tonic-gate if (kmem_lp_arena == NULL || !IS_KMEM_VA_LARGEPAGE((caddr_t)inaddr)) { 13757c478bd9Sstevel@tonic-gate segkmem_free(vmp, inaddr, size); 13767c478bd9Sstevel@tonic-gate } else { 13777c478bd9Sstevel@tonic-gate vmem_free(kmem_lp_arena, inaddr, size); 13787c478bd9Sstevel@tonic-gate } 13797c478bd9Sstevel@tonic-gate } 13807c478bd9Sstevel@tonic-gate 13817c478bd9Sstevel@tonic-gate /* 13827c478bd9Sstevel@tonic-gate * segkmem_alloc_lpi() imports virtual memory from large page heap arena 13837c478bd9Sstevel@tonic-gate * into kmem_lp arena. In the process it maps the imported segment with 13847c478bd9Sstevel@tonic-gate * large pages 13857c478bd9Sstevel@tonic-gate */ 13867c478bd9Sstevel@tonic-gate static void * 13877c478bd9Sstevel@tonic-gate segkmem_alloc_lpi(vmem_t *vmp, size_t size, int vmflag) 13887c478bd9Sstevel@tonic-gate { 13897c478bd9Sstevel@tonic-gate segkmem_lpcb_t *lpcb = &segkmem_lpcb; 13907c478bd9Sstevel@tonic-gate void *addr; 13917c478bd9Sstevel@tonic-gate 13927c478bd9Sstevel@tonic-gate ASSERT(size != 0); 13937c478bd9Sstevel@tonic-gate ASSERT(vmp == heap_lp_arena); 13947c478bd9Sstevel@tonic-gate 13957c478bd9Sstevel@tonic-gate /* do not allow large page heap grow beyound limits */ 13967c478bd9Sstevel@tonic-gate if (vmem_size(vmp, VMEM_ALLOC) >= segkmem_kmemlp_max) { 1397501f3284Seg155566 lpcb->allocs_limited++; 13987c478bd9Sstevel@tonic-gate return (NULL); 13997c478bd9Sstevel@tonic-gate } 14007c478bd9Sstevel@tonic-gate 14017c478bd9Sstevel@tonic-gate addr = segkmem_xalloc_lp(vmp, NULL, size, vmflag, 0, 14027c478bd9Sstevel@tonic-gate segkmem_page_create_large, NULL); 14037c478bd9Sstevel@tonic-gate return (addr); 14047c478bd9Sstevel@tonic-gate } 14057c478bd9Sstevel@tonic-gate 14067c478bd9Sstevel@tonic-gate /* 14077c478bd9Sstevel@tonic-gate * segkmem_free_lpi() returns virtual memory back into large page heap arena 14087c478bd9Sstevel@tonic-gate * from kmem_lp arena. Beore doing this it unmaps the segment and frees 14097c478bd9Sstevel@tonic-gate * large pages used to map it. 14107c478bd9Sstevel@tonic-gate */ 14117c478bd9Sstevel@tonic-gate static void 14127c478bd9Sstevel@tonic-gate segkmem_free_lpi(vmem_t *vmp, void *inaddr, size_t size) 14137c478bd9Sstevel@tonic-gate { 14147c478bd9Sstevel@tonic-gate pgcnt_t nlpages = size >> segkmem_lpshift; 14157c478bd9Sstevel@tonic-gate size_t lpsize = segkmem_lpsize; 14167c478bd9Sstevel@tonic-gate caddr_t addr = inaddr; 14177c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size); 14187c478bd9Sstevel@tonic-gate int i; 14197c478bd9Sstevel@tonic-gate 14207c478bd9Sstevel@tonic-gate ASSERT(vmp == heap_lp_arena); 14217c478bd9Sstevel@tonic-gate ASSERT(IS_KMEM_VA_LARGEPAGE(addr)); 14227c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)inaddr & (lpsize - 1)) == 0); 14237c478bd9Sstevel@tonic-gate 14247c478bd9Sstevel@tonic-gate for (i = 0; i < nlpages; i++) { 14257c478bd9Sstevel@tonic-gate segkmem_free_one_lp(addr, lpsize); 14267c478bd9Sstevel@tonic-gate addr += lpsize; 14277c478bd9Sstevel@tonic-gate } 14287c478bd9Sstevel@tonic-gate 14297c478bd9Sstevel@tonic-gate page_unresv(npages); 14307c478bd9Sstevel@tonic-gate 14317c478bd9Sstevel@tonic-gate vmem_free(vmp, inaddr, size); 14327c478bd9Sstevel@tonic-gate } 14337c478bd9Sstevel@tonic-gate 14347c478bd9Sstevel@tonic-gate /* 14357c478bd9Sstevel@tonic-gate * This function is called at system boot time by kmem_init right after 14367c478bd9Sstevel@tonic-gate * /etc/system file has been read. It checks based on hardware configuration 14377c478bd9Sstevel@tonic-gate * and /etc/system settings if system is going to use large pages. The 14387c478bd9Sstevel@tonic-gate * initialiazation necessary to actually start using large pages 14397c478bd9Sstevel@tonic-gate * happens later in the process after segkmem_heap_lp_init() is called. 14407c478bd9Sstevel@tonic-gate */ 14417c478bd9Sstevel@tonic-gate int 14427c478bd9Sstevel@tonic-gate segkmem_lpsetup() 14437c478bd9Sstevel@tonic-gate { 14447c478bd9Sstevel@tonic-gate int use_large_pages = 0; 14457c478bd9Sstevel@tonic-gate 14467c478bd9Sstevel@tonic-gate #ifdef __sparc 14477c478bd9Sstevel@tonic-gate 14487c478bd9Sstevel@tonic-gate size_t memtotal = physmem * PAGESIZE; 14497c478bd9Sstevel@tonic-gate 14507c478bd9Sstevel@tonic-gate if (heap_lp_base == NULL) { 14517c478bd9Sstevel@tonic-gate segkmem_lpsize = PAGESIZE; 14527c478bd9Sstevel@tonic-gate return (0); 14537c478bd9Sstevel@tonic-gate } 14547c478bd9Sstevel@tonic-gate 14557c478bd9Sstevel@tonic-gate /* get a platform dependent value of large page size for kernel heap */ 14567c478bd9Sstevel@tonic-gate segkmem_lpsize = get_segkmem_lpsize(segkmem_lpsize); 14577c478bd9Sstevel@tonic-gate 14587c478bd9Sstevel@tonic-gate if (segkmem_lpsize <= PAGESIZE) { 14597c478bd9Sstevel@tonic-gate /* 14607c478bd9Sstevel@tonic-gate * put virtual space reserved for the large page kernel 14617c478bd9Sstevel@tonic-gate * back to the regular heap 14627c478bd9Sstevel@tonic-gate */ 14637c478bd9Sstevel@tonic-gate vmem_xfree(heap_arena, heap_lp_base, 14647c478bd9Sstevel@tonic-gate heap_lp_end - heap_lp_base); 14657c478bd9Sstevel@tonic-gate heap_lp_base = NULL; 14667c478bd9Sstevel@tonic-gate heap_lp_end = NULL; 14677c478bd9Sstevel@tonic-gate segkmem_lpsize = PAGESIZE; 14687c478bd9Sstevel@tonic-gate return (0); 14697c478bd9Sstevel@tonic-gate } 14707c478bd9Sstevel@tonic-gate 14717c478bd9Sstevel@tonic-gate /* set heap_lp quantum if necessary */ 14727c478bd9Sstevel@tonic-gate if (segkmem_heaplp_quantum == 0 || 14737c478bd9Sstevel@tonic-gate (segkmem_heaplp_quantum & (segkmem_heaplp_quantum - 1)) || 14747c478bd9Sstevel@tonic-gate P2PHASE(segkmem_heaplp_quantum, segkmem_lpsize)) { 14757c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum = segkmem_lpsize; 14767c478bd9Sstevel@tonic-gate } 14777c478bd9Sstevel@tonic-gate 14787c478bd9Sstevel@tonic-gate /* set kmem_lp quantum if necessary */ 14797c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_quantum == 0 || 14807c478bd9Sstevel@tonic-gate (segkmem_kmemlp_quantum & (segkmem_kmemlp_quantum - 1)) || 14817c478bd9Sstevel@tonic-gate segkmem_kmemlp_quantum > segkmem_heaplp_quantum) { 14827c478bd9Sstevel@tonic-gate segkmem_kmemlp_quantum = segkmem_heaplp_quantum; 14837c478bd9Sstevel@tonic-gate } 14847c478bd9Sstevel@tonic-gate 14857c478bd9Sstevel@tonic-gate /* set total amount of memory allowed for large page kernel heap */ 14867c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_max == 0) { 14877c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_pcnt == 0 || segkmem_kmemlp_pcnt > 100) 1488501f3284Seg155566 segkmem_kmemlp_pcnt = 12; 1489501f3284Seg155566 segkmem_kmemlp_max = (memtotal * segkmem_kmemlp_pcnt) / 100; 14907c478bd9Sstevel@tonic-gate } 14917c478bd9Sstevel@tonic-gate segkmem_kmemlp_max = P2ROUNDUP(segkmem_kmemlp_max, 14927c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum); 14937c478bd9Sstevel@tonic-gate 14947c478bd9Sstevel@tonic-gate /* fix lp kmem preallocation request if necesssary */ 14957c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_min) { 14967c478bd9Sstevel@tonic-gate segkmem_kmemlp_min = P2ROUNDUP(segkmem_kmemlp_min, 14977c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum); 14987c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_min > segkmem_kmemlp_max) 14997c478bd9Sstevel@tonic-gate segkmem_kmemlp_min = segkmem_kmemlp_max; 15007c478bd9Sstevel@tonic-gate } 15017c478bd9Sstevel@tonic-gate 15027c478bd9Sstevel@tonic-gate use_large_pages = 1; 1503081a94b0Saguzovsk segkmem_lpszc = page_szc(segkmem_lpsize); 1504081a94b0Saguzovsk segkmem_lpshift = page_get_shift(segkmem_lpszc); 15057c478bd9Sstevel@tonic-gate 15067c478bd9Sstevel@tonic-gate #endif 15077c478bd9Sstevel@tonic-gate return (use_large_pages); 15087c478bd9Sstevel@tonic-gate } 15097c478bd9Sstevel@tonic-gate 1510ad23a2dbSjohansen void 1511ad23a2dbSjohansen segkmem_zio_init(void *zio_mem_base, size_t zio_mem_size) 1512ad23a2dbSjohansen { 1513ad23a2dbSjohansen ASSERT(zio_mem_base != NULL); 1514ad23a2dbSjohansen ASSERT(zio_mem_size != 0); 1515ad23a2dbSjohansen 1516ad23a2dbSjohansen zio_arena = vmem_create("zio", zio_mem_base, zio_mem_size, PAGESIZE, 1517ad23a2dbSjohansen NULL, NULL, NULL, 0, VM_SLEEP); 1518ad23a2dbSjohansen 1519ad23a2dbSjohansen zio_alloc_arena = vmem_create("zio_buf", NULL, 0, PAGESIZE, 1520ad23a2dbSjohansen segkmem_zio_alloc, segkmem_zio_free, zio_arena, 0, VM_SLEEP); 1521ad23a2dbSjohansen 1522ad23a2dbSjohansen ASSERT(zio_arena != NULL); 1523ad23a2dbSjohansen ASSERT(zio_alloc_arena != NULL); 1524ad23a2dbSjohansen } 1525ad23a2dbSjohansen 15267c478bd9Sstevel@tonic-gate #ifdef __sparc 15277c478bd9Sstevel@tonic-gate 15287c478bd9Sstevel@tonic-gate 15297c478bd9Sstevel@tonic-gate static void * 15307c478bd9Sstevel@tonic-gate segkmem_alloc_ppa(vmem_t *vmp, size_t size, int vmflag) 15317c478bd9Sstevel@tonic-gate { 15327c478bd9Sstevel@tonic-gate size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *); 15337c478bd9Sstevel@tonic-gate void *addr; 15347c478bd9Sstevel@tonic-gate 15357c478bd9Sstevel@tonic-gate if (ppaquantum <= PAGESIZE) 15367c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, vmflag)); 15377c478bd9Sstevel@tonic-gate 15387c478bd9Sstevel@tonic-gate ASSERT((size & (ppaquantum - 1)) == 0); 15397c478bd9Sstevel@tonic-gate 15407c478bd9Sstevel@tonic-gate addr = vmem_xalloc(vmp, size, ppaquantum, 0, 0, NULL, NULL, vmflag); 15417c478bd9Sstevel@tonic-gate if (addr != NULL && segkmem_xalloc(vmp, addr, size, vmflag, 0, 15427c478bd9Sstevel@tonic-gate segkmem_page_create, NULL) == NULL) { 15437c478bd9Sstevel@tonic-gate vmem_xfree(vmp, addr, size); 15447c478bd9Sstevel@tonic-gate addr = NULL; 15457c478bd9Sstevel@tonic-gate } 15467c478bd9Sstevel@tonic-gate 15477c478bd9Sstevel@tonic-gate return (addr); 15487c478bd9Sstevel@tonic-gate } 15497c478bd9Sstevel@tonic-gate 15507c478bd9Sstevel@tonic-gate static void 15517c478bd9Sstevel@tonic-gate segkmem_free_ppa(vmem_t *vmp, void *addr, size_t size) 15527c478bd9Sstevel@tonic-gate { 15537c478bd9Sstevel@tonic-gate size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *); 15547c478bd9Sstevel@tonic-gate 15557c478bd9Sstevel@tonic-gate ASSERT(addr != NULL); 15567c478bd9Sstevel@tonic-gate 15577c478bd9Sstevel@tonic-gate if (ppaquantum <= PAGESIZE) { 15587c478bd9Sstevel@tonic-gate segkmem_free(vmp, addr, size); 15597c478bd9Sstevel@tonic-gate } else { 15607c478bd9Sstevel@tonic-gate segkmem_free(NULL, addr, size); 15617c478bd9Sstevel@tonic-gate vmem_xfree(vmp, addr, size); 15627c478bd9Sstevel@tonic-gate } 15637c478bd9Sstevel@tonic-gate } 15647c478bd9Sstevel@tonic-gate 15657c478bd9Sstevel@tonic-gate void 15667c478bd9Sstevel@tonic-gate segkmem_heap_lp_init() 15677c478bd9Sstevel@tonic-gate { 15687c478bd9Sstevel@tonic-gate segkmem_lpcb_t *lpcb = &segkmem_lpcb; 15697c478bd9Sstevel@tonic-gate size_t heap_lp_size = heap_lp_end - heap_lp_base; 15707c478bd9Sstevel@tonic-gate size_t lpsize = segkmem_lpsize; 15717c478bd9Sstevel@tonic-gate size_t ppaquantum; 15727c478bd9Sstevel@tonic-gate void *addr; 15737c478bd9Sstevel@tonic-gate 15747c478bd9Sstevel@tonic-gate if (segkmem_lpsize <= PAGESIZE) { 15757c478bd9Sstevel@tonic-gate ASSERT(heap_lp_base == NULL); 15767c478bd9Sstevel@tonic-gate ASSERT(heap_lp_end == NULL); 15777c478bd9Sstevel@tonic-gate return; 15787c478bd9Sstevel@tonic-gate } 15797c478bd9Sstevel@tonic-gate 15807c478bd9Sstevel@tonic-gate ASSERT(segkmem_heaplp_quantum >= lpsize); 15817c478bd9Sstevel@tonic-gate ASSERT((segkmem_heaplp_quantum & (lpsize - 1)) == 0); 15827c478bd9Sstevel@tonic-gate ASSERT(lpcb->lp_uselp == 0); 15837c478bd9Sstevel@tonic-gate ASSERT(heap_lp_base != NULL); 15847c478bd9Sstevel@tonic-gate ASSERT(heap_lp_end != NULL); 15857c478bd9Sstevel@tonic-gate ASSERT(heap_lp_base < heap_lp_end); 15867c478bd9Sstevel@tonic-gate ASSERT(heap_lp_arena == NULL); 15877c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)heap_lp_base & (lpsize - 1)) == 0); 15887c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)heap_lp_end & (lpsize - 1)) == 0); 15897c478bd9Sstevel@tonic-gate 15907c478bd9Sstevel@tonic-gate /* create large page heap arena */ 15917c478bd9Sstevel@tonic-gate heap_lp_arena = vmem_create("heap_lp", heap_lp_base, heap_lp_size, 15927c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum, NULL, NULL, NULL, 0, VM_SLEEP); 15937c478bd9Sstevel@tonic-gate 15947c478bd9Sstevel@tonic-gate ASSERT(heap_lp_arena != NULL); 15957c478bd9Sstevel@tonic-gate 15967c478bd9Sstevel@tonic-gate /* This arena caches memory already mapped by large pages */ 15977c478bd9Sstevel@tonic-gate kmem_lp_arena = vmem_create("kmem_lp", NULL, 0, segkmem_kmemlp_quantum, 15987c478bd9Sstevel@tonic-gate segkmem_alloc_lpi, segkmem_free_lpi, heap_lp_arena, 0, VM_SLEEP); 15997c478bd9Sstevel@tonic-gate 16007c478bd9Sstevel@tonic-gate ASSERT(kmem_lp_arena != NULL); 16017c478bd9Sstevel@tonic-gate 16027c478bd9Sstevel@tonic-gate mutex_init(&lpcb->lp_lock, NULL, MUTEX_DEFAULT, NULL); 16037c478bd9Sstevel@tonic-gate cv_init(&lpcb->lp_cv, NULL, CV_DEFAULT, NULL); 16047c478bd9Sstevel@tonic-gate 16057c478bd9Sstevel@tonic-gate /* 16067c478bd9Sstevel@tonic-gate * this arena is used for the array of page_t pointers necessary 16077c478bd9Sstevel@tonic-gate * to call hat_mem_load_array 16087c478bd9Sstevel@tonic-gate */ 16097c478bd9Sstevel@tonic-gate ppaquantum = btopr(lpsize) * sizeof (page_t *); 16107c478bd9Sstevel@tonic-gate segkmem_ppa_arena = vmem_create("segkmem_ppa", NULL, 0, ppaquantum, 16117c478bd9Sstevel@tonic-gate segkmem_alloc_ppa, segkmem_free_ppa, heap_arena, ppaquantum, 16127c478bd9Sstevel@tonic-gate VM_SLEEP); 16137c478bd9Sstevel@tonic-gate 16147c478bd9Sstevel@tonic-gate ASSERT(segkmem_ppa_arena != NULL); 16157c478bd9Sstevel@tonic-gate 16167c478bd9Sstevel@tonic-gate /* prealloacate some memory for the lp kernel heap */ 16177c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_min) { 16187c478bd9Sstevel@tonic-gate 16197c478bd9Sstevel@tonic-gate ASSERT(P2PHASE(segkmem_kmemlp_min, 16207c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum) == 0); 16217c478bd9Sstevel@tonic-gate 16227c478bd9Sstevel@tonic-gate if ((addr = segkmem_alloc_lpi(heap_lp_arena, 16237c478bd9Sstevel@tonic-gate segkmem_kmemlp_min, VM_SLEEP)) != NULL) { 16247c478bd9Sstevel@tonic-gate 16257c478bd9Sstevel@tonic-gate addr = vmem_add(kmem_lp_arena, addr, 16267c478bd9Sstevel@tonic-gate segkmem_kmemlp_min, VM_SLEEP); 16277c478bd9Sstevel@tonic-gate ASSERT(addr != NULL); 16287c478bd9Sstevel@tonic-gate } 16297c478bd9Sstevel@tonic-gate } 16307c478bd9Sstevel@tonic-gate 16317c478bd9Sstevel@tonic-gate lpcb->lp_uselp = 1; 16327c478bd9Sstevel@tonic-gate } 16337c478bd9Sstevel@tonic-gate 16347c478bd9Sstevel@tonic-gate #endif 1635