17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 67c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 77c478bd9Sstevel@tonic-gate * with the License. 87c478bd9Sstevel@tonic-gate * 97c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 107c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 117c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 127c478bd9Sstevel@tonic-gate * and limitations under the License. 137c478bd9Sstevel@tonic-gate * 147c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 157c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 167c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 177c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 187c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 197c478bd9Sstevel@tonic-gate * 207c478bd9Sstevel@tonic-gate * CDDL HEADER END 217c478bd9Sstevel@tonic-gate */ 227c478bd9Sstevel@tonic-gate /* 237c478bd9Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate #include <sys/types.h> 307c478bd9Sstevel@tonic-gate #include <sys/t_lock.h> 317c478bd9Sstevel@tonic-gate #include <sys/param.h> 327c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 337c478bd9Sstevel@tonic-gate #include <sys/tuneable.h> 347c478bd9Sstevel@tonic-gate #include <sys/systm.h> 357c478bd9Sstevel@tonic-gate #include <sys/vm.h> 367c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 377c478bd9Sstevel@tonic-gate #include <sys/vmem.h> 387c478bd9Sstevel@tonic-gate #include <sys/mman.h> 397c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 407c478bd9Sstevel@tonic-gate #include <sys/debug.h> 417c478bd9Sstevel@tonic-gate #include <sys/dumphdr.h> 427c478bd9Sstevel@tonic-gate #include <sys/bootconf.h> 437c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 447c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 457c478bd9Sstevel@tonic-gate #include <vm/hat.h> 467c478bd9Sstevel@tonic-gate #include <vm/page.h> 477c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h> 487c478bd9Sstevel@tonic-gate #include <vm/faultcode.h> 497c478bd9Sstevel@tonic-gate #include <sys/promif.h> 507c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h> 517c478bd9Sstevel@tonic-gate #include <sys/bitmap.h> 527c478bd9Sstevel@tonic-gate #include <sys/mem_cage.h> 537c478bd9Sstevel@tonic-gate 547c478bd9Sstevel@tonic-gate /* 557c478bd9Sstevel@tonic-gate * seg_kmem is the primary kernel memory segment driver. It 567c478bd9Sstevel@tonic-gate * maps the kernel heap [kernelheap, ekernelheap), module text, 577c478bd9Sstevel@tonic-gate * and all memory which was allocated before the VM was initialized 587c478bd9Sstevel@tonic-gate * into kas. 597c478bd9Sstevel@tonic-gate * 607c478bd9Sstevel@tonic-gate * Pages which belong to seg_kmem are hashed into &kvp vnode at 617c478bd9Sstevel@tonic-gate * an offset equal to (u_offset_t)virt_addr, and have p_lckcnt >= 1. 627c478bd9Sstevel@tonic-gate * They must never be paged out since segkmem_fault() is a no-op to 637c478bd9Sstevel@tonic-gate * prevent recursive faults. 647c478bd9Sstevel@tonic-gate * 657c478bd9Sstevel@tonic-gate * Currently, seg_kmem pages are sharelocked (p_sharelock == 1) on 667c478bd9Sstevel@tonic-gate * __x86 and are unlocked (p_sharelock == 0) on __sparc. Once __x86 677c478bd9Sstevel@tonic-gate * supports relocation the #ifdef kludges can be removed. 687c478bd9Sstevel@tonic-gate * 697c478bd9Sstevel@tonic-gate * seg_kmem pages may be subject to relocation by page_relocate(), 707c478bd9Sstevel@tonic-gate * provided that the HAT supports it; if this is so, segkmem_reloc 717c478bd9Sstevel@tonic-gate * will be set to a nonzero value. All boot time allocated memory as 727c478bd9Sstevel@tonic-gate * well as static memory is considered off limits to relocation. 737c478bd9Sstevel@tonic-gate * Pages are "relocatable" if p_state does not have P_NORELOC set, so 747c478bd9Sstevel@tonic-gate * we request P_NORELOC pages for memory that isn't safe to relocate. 757c478bd9Sstevel@tonic-gate * 767c478bd9Sstevel@tonic-gate * The kernel heap is logically divided up into four pieces: 777c478bd9Sstevel@tonic-gate * 787c478bd9Sstevel@tonic-gate * heap32_arena is for allocations that require 32-bit absolute 797c478bd9Sstevel@tonic-gate * virtual addresses (e.g. code that uses 32-bit pointers/offsets). 807c478bd9Sstevel@tonic-gate * 817c478bd9Sstevel@tonic-gate * heap_core is for allocations that require 2GB *relative* 827c478bd9Sstevel@tonic-gate * offsets; in other words all memory from heap_core is within 837c478bd9Sstevel@tonic-gate * 2GB of all other memory from the same arena. This is a requirement 847c478bd9Sstevel@tonic-gate * of the addressing modes of some processors in supervisor code. 857c478bd9Sstevel@tonic-gate * 867c478bd9Sstevel@tonic-gate * heap_arena is the general heap arena. 877c478bd9Sstevel@tonic-gate * 887c478bd9Sstevel@tonic-gate * static_arena is the static memory arena. Allocations from it 897c478bd9Sstevel@tonic-gate * are not subject to relocation so it is safe to use the memory 907c478bd9Sstevel@tonic-gate * physical address as well as the virtual address (e.g. the VA to 917c478bd9Sstevel@tonic-gate * PA translations are static). Caches may import from static_arena; 927c478bd9Sstevel@tonic-gate * all other static memory allocations should use static_alloc_arena. 937c478bd9Sstevel@tonic-gate * 947c478bd9Sstevel@tonic-gate * On some platforms which have limited virtual address space, seg_kmem 957c478bd9Sstevel@tonic-gate * may share [kernelheap, ekernelheap) with seg_kp; if this is so, 967c478bd9Sstevel@tonic-gate * segkp_bitmap is non-NULL, and each bit represents a page of virtual 977c478bd9Sstevel@tonic-gate * address space which is actually seg_kp mapped. 987c478bd9Sstevel@tonic-gate */ 997c478bd9Sstevel@tonic-gate 1007c478bd9Sstevel@tonic-gate extern ulong_t *segkp_bitmap; /* Is set if segkp is from the kernel heap */ 1017c478bd9Sstevel@tonic-gate 1027c478bd9Sstevel@tonic-gate char *kernelheap; /* start of primary kernel heap */ 1037c478bd9Sstevel@tonic-gate char *ekernelheap; /* end of primary kernel heap */ 1047c478bd9Sstevel@tonic-gate struct seg kvseg; /* primary kernel heap segment */ 1057c478bd9Sstevel@tonic-gate struct seg kvseg_core; /* "core" kernel heap segment */ 1067c478bd9Sstevel@tonic-gate vmem_t *heap_arena; /* primary kernel heap arena */ 1077c478bd9Sstevel@tonic-gate vmem_t *heap_core_arena; /* core kernel heap arena */ 1087c478bd9Sstevel@tonic-gate char *heap_core_base; /* start of core kernel heap arena */ 1097c478bd9Sstevel@tonic-gate char *heap_lp_base; /* start of kernel large page heap arena */ 1107c478bd9Sstevel@tonic-gate char *heap_lp_end; /* end of kernel large page heap arena */ 1117c478bd9Sstevel@tonic-gate vmem_t *hat_memload_arena; /* HAT translation data */ 1127c478bd9Sstevel@tonic-gate struct seg kvseg32; /* 32-bit kernel heap segment */ 1137c478bd9Sstevel@tonic-gate vmem_t *heap32_arena; /* 32-bit kernel heap arena */ 1147c478bd9Sstevel@tonic-gate vmem_t *heaptext_arena; /* heaptext arena */ 1157c478bd9Sstevel@tonic-gate struct as kas; /* kernel address space */ 1167c478bd9Sstevel@tonic-gate struct vnode kvp; /* vnode for all segkmem pages */ 1177c478bd9Sstevel@tonic-gate int segkmem_reloc; /* enable/disable relocatable segkmem pages */ 1187c478bd9Sstevel@tonic-gate vmem_t *static_arena; /* arena for caches to import static memory */ 1197c478bd9Sstevel@tonic-gate vmem_t *static_alloc_arena; /* arena for allocating static memory */ 1207c478bd9Sstevel@tonic-gate 1217c478bd9Sstevel@tonic-gate /* 1227c478bd9Sstevel@tonic-gate * seg_kmem driver can map part of the kernel heap with large pages. 1237c478bd9Sstevel@tonic-gate * Currently this functionality is implemented for sparc platforms only. 1247c478bd9Sstevel@tonic-gate * 1257c478bd9Sstevel@tonic-gate * The large page size "segkmem_lpsize" for kernel heap is selected in the 1267c478bd9Sstevel@tonic-gate * platform specific code. It can also be modified via /etc/system file. 1277c478bd9Sstevel@tonic-gate * Setting segkmem_lpsize to PAGESIZE in /etc/system disables usage of large 1287c478bd9Sstevel@tonic-gate * pages for kernel heap. "segkmem_lpshift" is adjusted appropriately to 1297c478bd9Sstevel@tonic-gate * match segkmem_lpsize. 1307c478bd9Sstevel@tonic-gate * 1317c478bd9Sstevel@tonic-gate * At boot time we carve from kernel heap arena a range of virtual addresses 1327c478bd9Sstevel@tonic-gate * that will be used for large page mappings. This range [heap_lp_base, 1337c478bd9Sstevel@tonic-gate * heap_lp_end) is set up as a separate vmem arena - "heap_lp_arena". We also 1347c478bd9Sstevel@tonic-gate * create "kmem_lp_arena" that caches memory already backed up by large 1357c478bd9Sstevel@tonic-gate * pages. kmem_lp_arena imports virtual segments from heap_lp_arena. 1367c478bd9Sstevel@tonic-gate */ 1377c478bd9Sstevel@tonic-gate 1387c478bd9Sstevel@tonic-gate size_t segkmem_lpsize; 1397c478bd9Sstevel@tonic-gate static uint_t segkmem_lpshift = PAGESHIFT; 1407c478bd9Sstevel@tonic-gate 1417c478bd9Sstevel@tonic-gate size_t segkmem_kmemlp_quantum = 0x400000; /* 4MB */ 1427c478bd9Sstevel@tonic-gate size_t segkmem_heaplp_quantum; 143*7bc98a2eSeg155566 vmem_t *heap_lp_arena; 1447c478bd9Sstevel@tonic-gate static vmem_t *kmem_lp_arena; 1457c478bd9Sstevel@tonic-gate static vmem_t *segkmem_ppa_arena; 1467c478bd9Sstevel@tonic-gate static segkmem_lpcb_t segkmem_lpcb; 1477c478bd9Sstevel@tonic-gate 1487c478bd9Sstevel@tonic-gate /* 1497c478bd9Sstevel@tonic-gate * We use "segkmem_kmemlp_max" to limit the total amount of physical memory 1507c478bd9Sstevel@tonic-gate * consumed by the large page heap. By default this parameter is set to 1/4 of 1517c478bd9Sstevel@tonic-gate * physmem but can be adjusted through /etc/system either directly or 1527c478bd9Sstevel@tonic-gate * indirectly by setting "segkmem_kmemlp_pcnt" to the percent of physmem 1537c478bd9Sstevel@tonic-gate * we allow for large page heap. 1547c478bd9Sstevel@tonic-gate */ 1557c478bd9Sstevel@tonic-gate size_t segkmem_kmemlp_max; 1567c478bd9Sstevel@tonic-gate static uint_t segkmem_kmemlp_pcnt; 1577c478bd9Sstevel@tonic-gate 1587c478bd9Sstevel@tonic-gate /* 1597c478bd9Sstevel@tonic-gate * Getting large pages for kernel heap could be problematic due to 1607c478bd9Sstevel@tonic-gate * physical memory fragmentation. That's why we allow to preallocate 1617c478bd9Sstevel@tonic-gate * "segkmem_kmemlp_min" bytes at boot time. 1627c478bd9Sstevel@tonic-gate */ 1637c478bd9Sstevel@tonic-gate static size_t segkmem_kmemlp_min; 1647c478bd9Sstevel@tonic-gate 1657c478bd9Sstevel@tonic-gate /* 1667c478bd9Sstevel@tonic-gate * Throttling is used to avoid expensive tries to allocate large pages 1677c478bd9Sstevel@tonic-gate * for kernel heap when a lot of succesive attempts to do so fail. 1687c478bd9Sstevel@tonic-gate */ 1697c478bd9Sstevel@tonic-gate static ulong_t segkmem_lpthrottle_max = 0x400000; 1707c478bd9Sstevel@tonic-gate static ulong_t segkmem_lpthrottle_start = 0x40; 1717c478bd9Sstevel@tonic-gate static ulong_t segkmem_use_lpthrottle = 1; 1727c478bd9Sstevel@tonic-gate 1737c478bd9Sstevel@tonic-gate /* 1747c478bd9Sstevel@tonic-gate * Freed pages accumulate on a garbage list until segkmem is ready, 1757c478bd9Sstevel@tonic-gate * at which point we call segkmem_gc() to free it all. 1767c478bd9Sstevel@tonic-gate */ 1777c478bd9Sstevel@tonic-gate typedef struct segkmem_gc_list { 1787c478bd9Sstevel@tonic-gate struct segkmem_gc_list *gc_next; 1797c478bd9Sstevel@tonic-gate vmem_t *gc_arena; 1807c478bd9Sstevel@tonic-gate size_t gc_size; 1817c478bd9Sstevel@tonic-gate } segkmem_gc_list_t; 1827c478bd9Sstevel@tonic-gate 1837c478bd9Sstevel@tonic-gate static segkmem_gc_list_t *segkmem_gc_list; 1847c478bd9Sstevel@tonic-gate 1857c478bd9Sstevel@tonic-gate /* 1867c478bd9Sstevel@tonic-gate * Allocations from the hat_memload arena add VM_MEMLOAD to their 1877c478bd9Sstevel@tonic-gate * vmflags so that segkmem_xalloc() can inform the hat layer that it needs 1887c478bd9Sstevel@tonic-gate * to take steps to prevent infinite recursion. HAT allocations also 1897c478bd9Sstevel@tonic-gate * must be non-relocatable to prevent recursive page faults. 1907c478bd9Sstevel@tonic-gate */ 1917c478bd9Sstevel@tonic-gate static void * 1927c478bd9Sstevel@tonic-gate hat_memload_alloc(vmem_t *vmp, size_t size, int flags) 1937c478bd9Sstevel@tonic-gate { 1947c478bd9Sstevel@tonic-gate flags |= (VM_MEMLOAD | VM_NORELOC); 1957c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, flags)); 1967c478bd9Sstevel@tonic-gate } 1977c478bd9Sstevel@tonic-gate 1987c478bd9Sstevel@tonic-gate /* 1997c478bd9Sstevel@tonic-gate * Allocations from static_arena arena (or any other arena that uses 2007c478bd9Sstevel@tonic-gate * segkmem_alloc_permanent()) require non-relocatable (permanently 2017c478bd9Sstevel@tonic-gate * wired) memory pages, since these pages are referenced by physical 2027c478bd9Sstevel@tonic-gate * as well as virtual address. 2037c478bd9Sstevel@tonic-gate */ 2047c478bd9Sstevel@tonic-gate void * 2057c478bd9Sstevel@tonic-gate segkmem_alloc_permanent(vmem_t *vmp, size_t size, int flags) 2067c478bd9Sstevel@tonic-gate { 2077c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, flags | VM_NORELOC)); 2087c478bd9Sstevel@tonic-gate } 2097c478bd9Sstevel@tonic-gate 2107c478bd9Sstevel@tonic-gate /* 2117c478bd9Sstevel@tonic-gate * Initialize kernel heap boundaries. 2127c478bd9Sstevel@tonic-gate */ 2137c478bd9Sstevel@tonic-gate void 2147c478bd9Sstevel@tonic-gate kernelheap_init( 2157c478bd9Sstevel@tonic-gate void *heap_start, 2167c478bd9Sstevel@tonic-gate void *heap_end, 2177c478bd9Sstevel@tonic-gate char *first_avail, 2187c478bd9Sstevel@tonic-gate void *core_start, 2197c478bd9Sstevel@tonic-gate void *core_end) 2207c478bd9Sstevel@tonic-gate { 2217c478bd9Sstevel@tonic-gate uintptr_t textbase; 2227c478bd9Sstevel@tonic-gate size_t core_size; 2237c478bd9Sstevel@tonic-gate size_t heap_size; 2247c478bd9Sstevel@tonic-gate vmem_t *heaptext_parent; 2257c478bd9Sstevel@tonic-gate size_t heap_lp_size = 0; 2267c478bd9Sstevel@tonic-gate 2277c478bd9Sstevel@tonic-gate kernelheap = heap_start; 2287c478bd9Sstevel@tonic-gate ekernelheap = heap_end; 2297c478bd9Sstevel@tonic-gate 2307c478bd9Sstevel@tonic-gate #ifdef __sparc 2317c478bd9Sstevel@tonic-gate heap_lp_size = (((uintptr_t)heap_end - (uintptr_t)heap_start) / 4); 2327c478bd9Sstevel@tonic-gate heap_lp_base = ekernelheap - heap_lp_size; 2337c478bd9Sstevel@tonic-gate heap_lp_end = heap_lp_base + heap_lp_size; 2347c478bd9Sstevel@tonic-gate #endif /* __sparc */ 2357c478bd9Sstevel@tonic-gate 2367c478bd9Sstevel@tonic-gate /* 2377c478bd9Sstevel@tonic-gate * If this platform has a 'core' heap area, then the space for 2387c478bd9Sstevel@tonic-gate * overflow module text should be carved out of the end of that 2397c478bd9Sstevel@tonic-gate * heap. Otherwise, it gets carved out of the general purpose 2407c478bd9Sstevel@tonic-gate * heap. 2417c478bd9Sstevel@tonic-gate */ 2427c478bd9Sstevel@tonic-gate core_size = (uintptr_t)core_end - (uintptr_t)core_start; 2437c478bd9Sstevel@tonic-gate if (core_size > 0) { 2447c478bd9Sstevel@tonic-gate ASSERT(core_size >= HEAPTEXT_SIZE); 2457c478bd9Sstevel@tonic-gate textbase = (uintptr_t)core_end - HEAPTEXT_SIZE; 2467c478bd9Sstevel@tonic-gate core_size -= HEAPTEXT_SIZE; 2477c478bd9Sstevel@tonic-gate } 2487c478bd9Sstevel@tonic-gate #ifndef __sparc 2497c478bd9Sstevel@tonic-gate else { 2507c478bd9Sstevel@tonic-gate ekernelheap -= HEAPTEXT_SIZE; 2517c478bd9Sstevel@tonic-gate textbase = (uintptr_t)ekernelheap; 2527c478bd9Sstevel@tonic-gate } 2537c478bd9Sstevel@tonic-gate #endif 2547c478bd9Sstevel@tonic-gate 2557c478bd9Sstevel@tonic-gate heap_size = (uintptr_t)ekernelheap - (uintptr_t)kernelheap; 2567c478bd9Sstevel@tonic-gate heap_arena = vmem_init("heap", kernelheap, heap_size, PAGESIZE, 2577c478bd9Sstevel@tonic-gate segkmem_alloc, segkmem_free); 2587c478bd9Sstevel@tonic-gate 2597c478bd9Sstevel@tonic-gate if (core_size > 0) { 2607c478bd9Sstevel@tonic-gate heap_core_arena = vmem_create("heap_core", core_start, 2617c478bd9Sstevel@tonic-gate core_size, PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP); 2627c478bd9Sstevel@tonic-gate heap_core_base = core_start; 2637c478bd9Sstevel@tonic-gate } else { 2647c478bd9Sstevel@tonic-gate heap_core_arena = heap_arena; 2657c478bd9Sstevel@tonic-gate heap_core_base = kernelheap; 2667c478bd9Sstevel@tonic-gate } 2677c478bd9Sstevel@tonic-gate 2687c478bd9Sstevel@tonic-gate /* 2697c478bd9Sstevel@tonic-gate * reserve space for the large page heap. If large pages for kernel 2707c478bd9Sstevel@tonic-gate * heap is enabled large page heap arean will be created later in the 2717c478bd9Sstevel@tonic-gate * boot sequence in segkmem_heap_lp_init(). Otherwise the allocated 2727c478bd9Sstevel@tonic-gate * range will be returned back to the heap_arena. 2737c478bd9Sstevel@tonic-gate */ 2747c478bd9Sstevel@tonic-gate if (heap_lp_size) { 2757c478bd9Sstevel@tonic-gate (void) vmem_xalloc(heap_arena, heap_lp_size, PAGESIZE, 0, 0, 2767c478bd9Sstevel@tonic-gate heap_lp_base, heap_lp_end, 2777c478bd9Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 2787c478bd9Sstevel@tonic-gate } 2797c478bd9Sstevel@tonic-gate 2807c478bd9Sstevel@tonic-gate /* 2817c478bd9Sstevel@tonic-gate * Remove the already-spoken-for memory range [kernelheap, first_avail). 2827c478bd9Sstevel@tonic-gate */ 2837c478bd9Sstevel@tonic-gate (void) vmem_xalloc(heap_arena, first_avail - kernelheap, PAGESIZE, 2847c478bd9Sstevel@tonic-gate 0, 0, kernelheap, first_avail, VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 2857c478bd9Sstevel@tonic-gate 2867c478bd9Sstevel@tonic-gate #ifdef __sparc 2877c478bd9Sstevel@tonic-gate heap32_arena = vmem_create("heap32", (void *)SYSBASE32, 2887c478bd9Sstevel@tonic-gate SYSLIMIT32 - SYSBASE32 - HEAPTEXT_SIZE, PAGESIZE, NULL, 2897c478bd9Sstevel@tonic-gate NULL, NULL, 0, VM_SLEEP); 2907c478bd9Sstevel@tonic-gate 2917c478bd9Sstevel@tonic-gate textbase = SYSLIMIT32 - HEAPTEXT_SIZE; 2927c478bd9Sstevel@tonic-gate heaptext_parent = NULL; 2937c478bd9Sstevel@tonic-gate #else /* __sparc */ 2947c478bd9Sstevel@tonic-gate heap32_arena = heap_core_arena; 2957c478bd9Sstevel@tonic-gate heaptext_parent = heap_core_arena; 2967c478bd9Sstevel@tonic-gate #endif /* __sparc */ 2977c478bd9Sstevel@tonic-gate 2987c478bd9Sstevel@tonic-gate heaptext_arena = vmem_create("heaptext", (void *)textbase, 2997c478bd9Sstevel@tonic-gate HEAPTEXT_SIZE, PAGESIZE, NULL, NULL, heaptext_parent, 0, VM_SLEEP); 3007c478bd9Sstevel@tonic-gate 3017c478bd9Sstevel@tonic-gate /* 3027c478bd9Sstevel@tonic-gate * Create a set of arenas for memory with static translations 3037c478bd9Sstevel@tonic-gate * (e.g. VA -> PA translations cannot change). Since using 3047c478bd9Sstevel@tonic-gate * kernel pages by physical address implies it isn't safe to 3057c478bd9Sstevel@tonic-gate * walk across page boundaries, the static_arena quantum must 3067c478bd9Sstevel@tonic-gate * be PAGESIZE. Any kmem caches that require static memory 3077c478bd9Sstevel@tonic-gate * should source from static_arena, while direct allocations 3087c478bd9Sstevel@tonic-gate * should only use static_alloc_arena. 3097c478bd9Sstevel@tonic-gate */ 3107c478bd9Sstevel@tonic-gate static_arena = vmem_create("static", NULL, 0, PAGESIZE, 3117c478bd9Sstevel@tonic-gate segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP); 3127c478bd9Sstevel@tonic-gate static_alloc_arena = vmem_create("static_alloc", NULL, 0, 3137c478bd9Sstevel@tonic-gate sizeof (uint64_t), vmem_alloc, vmem_free, static_arena, 3147c478bd9Sstevel@tonic-gate 0, VM_SLEEP); 3157c478bd9Sstevel@tonic-gate 3167c478bd9Sstevel@tonic-gate /* 3177c478bd9Sstevel@tonic-gate * Create an arena for translation data (ptes, hmes, or hblks). 3187c478bd9Sstevel@tonic-gate * We need an arena for this because hat_memload() is essential 3197c478bd9Sstevel@tonic-gate * to vmem_populate() (see comments in common/os/vmem.c). 3207c478bd9Sstevel@tonic-gate * 3217c478bd9Sstevel@tonic-gate * Note: any kmem cache that allocates from hat_memload_arena 3227c478bd9Sstevel@tonic-gate * must be created as a KMC_NOHASH cache (i.e. no external slab 3237c478bd9Sstevel@tonic-gate * and bufctl structures to allocate) so that slab creation doesn't 3247c478bd9Sstevel@tonic-gate * require anything more than a single vmem_alloc(). 3257c478bd9Sstevel@tonic-gate */ 3267c478bd9Sstevel@tonic-gate hat_memload_arena = vmem_create("hat_memload", NULL, 0, PAGESIZE, 3277c478bd9Sstevel@tonic-gate hat_memload_alloc, segkmem_free, heap_arena, 0, 3287c478bd9Sstevel@tonic-gate VM_SLEEP | VMC_POPULATOR); 3297c478bd9Sstevel@tonic-gate } 3307c478bd9Sstevel@tonic-gate 3317c478bd9Sstevel@tonic-gate /* 3327c478bd9Sstevel@tonic-gate * Grow kernel heap downward. 3337c478bd9Sstevel@tonic-gate */ 3347c478bd9Sstevel@tonic-gate void 3357c478bd9Sstevel@tonic-gate kernelheap_extend(void *range_start, void *range_end) 3367c478bd9Sstevel@tonic-gate { 3377c478bd9Sstevel@tonic-gate size_t len = (uintptr_t)range_end - (uintptr_t)range_start; 3387c478bd9Sstevel@tonic-gate 3397c478bd9Sstevel@tonic-gate ASSERT(range_start < range_end && range_end == kernelheap); 3407c478bd9Sstevel@tonic-gate 3417c478bd9Sstevel@tonic-gate if (vmem_add(heap_arena, range_start, len, VM_NOSLEEP) == NULL) { 3427c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "Could not grow kernel heap below 0x%p", 3437c478bd9Sstevel@tonic-gate (void *)kernelheap); 3447c478bd9Sstevel@tonic-gate } else { 3457c478bd9Sstevel@tonic-gate kernelheap = range_start; 3467c478bd9Sstevel@tonic-gate } 3477c478bd9Sstevel@tonic-gate } 3487c478bd9Sstevel@tonic-gate 3497c478bd9Sstevel@tonic-gate void 3507c478bd9Sstevel@tonic-gate boot_mapin(caddr_t addr, size_t size) 3517c478bd9Sstevel@tonic-gate { 3527c478bd9Sstevel@tonic-gate caddr_t eaddr; 3537c478bd9Sstevel@tonic-gate page_t *pp; 3547c478bd9Sstevel@tonic-gate pfn_t pfnum; 3557c478bd9Sstevel@tonic-gate 3567c478bd9Sstevel@tonic-gate if (page_resv(btop(size), KM_NOSLEEP) == 0) 3577c478bd9Sstevel@tonic-gate panic("boot_mapin: page_resv failed"); 3587c478bd9Sstevel@tonic-gate 3597c478bd9Sstevel@tonic-gate for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) { 3607c478bd9Sstevel@tonic-gate pfnum = va_to_pfn(addr); 3617c478bd9Sstevel@tonic-gate if ((pp = page_numtopp_nolock(pfnum)) == NULL) 3627c478bd9Sstevel@tonic-gate panic("boot_mapin(): No pp for pfnum = %lx", pfnum); 3637c478bd9Sstevel@tonic-gate 3647c478bd9Sstevel@tonic-gate /* 3657c478bd9Sstevel@tonic-gate * must break up any large pages that may have constituent 3667c478bd9Sstevel@tonic-gate * pages being utilized for BOP_ALLOC()'s before calling 3677c478bd9Sstevel@tonic-gate * page_numtopp().The locking code (ie. page_reclaim()) 3687c478bd9Sstevel@tonic-gate * can't handle them 3697c478bd9Sstevel@tonic-gate */ 3707c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) 3717c478bd9Sstevel@tonic-gate page_boot_demote(pp); 3727c478bd9Sstevel@tonic-gate 3737c478bd9Sstevel@tonic-gate pp = page_numtopp(pfnum, SE_EXCL); 3747c478bd9Sstevel@tonic-gate if (pp == NULL || PP_ISFREE(pp)) 3757c478bd9Sstevel@tonic-gate panic("boot_alloc: pp is NULL or free"); 3767c478bd9Sstevel@tonic-gate 3777c478bd9Sstevel@tonic-gate /* 3787c478bd9Sstevel@tonic-gate * If the cage is on but doesn't yet contain this page, 3797c478bd9Sstevel@tonic-gate * mark it as non-relocatable. 3807c478bd9Sstevel@tonic-gate */ 3817c478bd9Sstevel@tonic-gate if (kcage_on && !PP_ISNORELOC(pp)) 3827c478bd9Sstevel@tonic-gate PP_SETNORELOC(pp); 3837c478bd9Sstevel@tonic-gate 3847c478bd9Sstevel@tonic-gate (void) page_hashin(pp, &kvp, (u_offset_t)(uintptr_t)addr, NULL); 3857c478bd9Sstevel@tonic-gate pp->p_lckcnt = 1; 3867c478bd9Sstevel@tonic-gate #if defined(__x86) 3877c478bd9Sstevel@tonic-gate page_downgrade(pp); 3887c478bd9Sstevel@tonic-gate #else 3897c478bd9Sstevel@tonic-gate page_unlock(pp); 3907c478bd9Sstevel@tonic-gate #endif 3917c478bd9Sstevel@tonic-gate } 3927c478bd9Sstevel@tonic-gate } 3937c478bd9Sstevel@tonic-gate 3947c478bd9Sstevel@tonic-gate /* 3957c478bd9Sstevel@tonic-gate * Get pages from boot and hash them into the kernel's vp. 3967c478bd9Sstevel@tonic-gate * Used after page structs have been allocated, but before segkmem is ready. 3977c478bd9Sstevel@tonic-gate */ 3987c478bd9Sstevel@tonic-gate void * 3997c478bd9Sstevel@tonic-gate boot_alloc(void *inaddr, size_t size, uint_t align) 4007c478bd9Sstevel@tonic-gate { 4017c478bd9Sstevel@tonic-gate caddr_t addr = inaddr; 4027c478bd9Sstevel@tonic-gate 4037c478bd9Sstevel@tonic-gate if (bootops == NULL) 4047c478bd9Sstevel@tonic-gate prom_panic("boot_alloc: attempt to allocate memory after " 4057c478bd9Sstevel@tonic-gate "BOP_GONE"); 4067c478bd9Sstevel@tonic-gate 4077c478bd9Sstevel@tonic-gate size = ptob(btopr(size)); 4087c478bd9Sstevel@tonic-gate if (BOP_ALLOC(bootops, addr, size, align) != addr) 4097c478bd9Sstevel@tonic-gate panic("boot_alloc: BOP_ALLOC failed"); 4107c478bd9Sstevel@tonic-gate boot_mapin((caddr_t)addr, size); 4117c478bd9Sstevel@tonic-gate return (addr); 4127c478bd9Sstevel@tonic-gate } 4137c478bd9Sstevel@tonic-gate 4147c478bd9Sstevel@tonic-gate static void 4157c478bd9Sstevel@tonic-gate segkmem_badop() 4167c478bd9Sstevel@tonic-gate { 4177c478bd9Sstevel@tonic-gate panic("segkmem_badop"); 4187c478bd9Sstevel@tonic-gate } 4197c478bd9Sstevel@tonic-gate 4207c478bd9Sstevel@tonic-gate #define SEGKMEM_BADOP(t) (t(*)())segkmem_badop 4217c478bd9Sstevel@tonic-gate 4227c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 4237c478bd9Sstevel@tonic-gate static faultcode_t 4247c478bd9Sstevel@tonic-gate segkmem_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t size, 4257c478bd9Sstevel@tonic-gate enum fault_type type, enum seg_rw rw) 4267c478bd9Sstevel@tonic-gate { 4277c478bd9Sstevel@tonic-gate ASSERT(RW_READ_HELD(&seg->s_as->a_lock)); 4287c478bd9Sstevel@tonic-gate 4297c478bd9Sstevel@tonic-gate if (seg->s_as != &kas || size > seg->s_size || 4307c478bd9Sstevel@tonic-gate addr < seg->s_base || addr + size > seg->s_base + seg->s_size) 4317c478bd9Sstevel@tonic-gate panic("segkmem_fault: bad args"); 4327c478bd9Sstevel@tonic-gate 4337c478bd9Sstevel@tonic-gate if (segkp_bitmap && seg == &kvseg) { 4347c478bd9Sstevel@tonic-gate 4357c478bd9Sstevel@tonic-gate /* 4367c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call segkp_fault. 4377c478bd9Sstevel@tonic-gate */ 4387c478bd9Sstevel@tonic-gate if (BT_TEST(segkp_bitmap, 4397c478bd9Sstevel@tonic-gate btop((uintptr_t)(addr - seg->s_base)))) 4407c478bd9Sstevel@tonic-gate return (SEGOP_FAULT(hat, segkp, addr, size, type, rw)); 4417c478bd9Sstevel@tonic-gate } 4427c478bd9Sstevel@tonic-gate 4437c478bd9Sstevel@tonic-gate switch (type) { 4447c478bd9Sstevel@tonic-gate case F_SOFTLOCK: /* lock down already-loaded translations */ 4457c478bd9Sstevel@tonic-gate if (rw == S_OTHER) { 4467c478bd9Sstevel@tonic-gate hat_reserve(seg->s_as, addr, size); 4477c478bd9Sstevel@tonic-gate return (0); 4487c478bd9Sstevel@tonic-gate } 4497c478bd9Sstevel@tonic-gate /*FALLTHROUGH*/ 4507c478bd9Sstevel@tonic-gate case F_SOFTUNLOCK: 4517c478bd9Sstevel@tonic-gate if (rw == S_READ || rw == S_WRITE) 4527c478bd9Sstevel@tonic-gate return (0); 4537c478bd9Sstevel@tonic-gate /*FALLTHROUGH*/ 4547c478bd9Sstevel@tonic-gate default: 4557c478bd9Sstevel@tonic-gate break; 4567c478bd9Sstevel@tonic-gate } 4577c478bd9Sstevel@tonic-gate return (FC_NOSUPPORT); 4587c478bd9Sstevel@tonic-gate } 4597c478bd9Sstevel@tonic-gate 4607c478bd9Sstevel@tonic-gate static int 4617c478bd9Sstevel@tonic-gate segkmem_setprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot) 4627c478bd9Sstevel@tonic-gate { 4637c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); 4647c478bd9Sstevel@tonic-gate 4657c478bd9Sstevel@tonic-gate if (seg->s_as != &kas || size > seg->s_size || 4667c478bd9Sstevel@tonic-gate addr < seg->s_base || addr + size > seg->s_base + seg->s_size) 4677c478bd9Sstevel@tonic-gate panic("segkmem_setprot: bad args"); 4687c478bd9Sstevel@tonic-gate 4697c478bd9Sstevel@tonic-gate if (segkp_bitmap && seg == &kvseg) { 4707c478bd9Sstevel@tonic-gate 4717c478bd9Sstevel@tonic-gate /* 4727c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call segkp. 4737c478bd9Sstevel@tonic-gate */ 4747c478bd9Sstevel@tonic-gate if (BT_TEST(segkp_bitmap, 4757c478bd9Sstevel@tonic-gate btop((uintptr_t)(addr - seg->s_base)))) 4767c478bd9Sstevel@tonic-gate return (SEGOP_SETPROT(segkp, addr, size, prot)); 4777c478bd9Sstevel@tonic-gate } 4787c478bd9Sstevel@tonic-gate 4797c478bd9Sstevel@tonic-gate if (prot == 0) 4807c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, addr, size, HAT_UNLOAD); 4817c478bd9Sstevel@tonic-gate else 4827c478bd9Sstevel@tonic-gate hat_chgprot(kas.a_hat, addr, size, prot); 4837c478bd9Sstevel@tonic-gate return (0); 4847c478bd9Sstevel@tonic-gate } 4857c478bd9Sstevel@tonic-gate 4867c478bd9Sstevel@tonic-gate /* 4877c478bd9Sstevel@tonic-gate * This is a dummy segkmem function overloaded to call segkp 4887c478bd9Sstevel@tonic-gate * when segkp is under the heap. 4897c478bd9Sstevel@tonic-gate */ 4907c478bd9Sstevel@tonic-gate /* ARGSUSED */ 4917c478bd9Sstevel@tonic-gate static int 4927c478bd9Sstevel@tonic-gate segkmem_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot) 4937c478bd9Sstevel@tonic-gate { 4947c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); 4957c478bd9Sstevel@tonic-gate 4967c478bd9Sstevel@tonic-gate if (seg->s_as != &kas) 4977c478bd9Sstevel@tonic-gate segkmem_badop(); 4987c478bd9Sstevel@tonic-gate 4997c478bd9Sstevel@tonic-gate if (segkp_bitmap && seg == &kvseg) { 5007c478bd9Sstevel@tonic-gate 5017c478bd9Sstevel@tonic-gate /* 5027c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp. 5037c478bd9Sstevel@tonic-gate */ 5047c478bd9Sstevel@tonic-gate if (BT_TEST(segkp_bitmap, 5057c478bd9Sstevel@tonic-gate btop((uintptr_t)(addr - seg->s_base)))) 5067c478bd9Sstevel@tonic-gate return (SEGOP_CHECKPROT(segkp, addr, size, prot)); 5077c478bd9Sstevel@tonic-gate } 5087c478bd9Sstevel@tonic-gate segkmem_badop(); 5097c478bd9Sstevel@tonic-gate return (0); 5107c478bd9Sstevel@tonic-gate } 5117c478bd9Sstevel@tonic-gate 5127c478bd9Sstevel@tonic-gate /* 5137c478bd9Sstevel@tonic-gate * This is a dummy segkmem function overloaded to call segkp 5147c478bd9Sstevel@tonic-gate * when segkp is under the heap. 5157c478bd9Sstevel@tonic-gate */ 5167c478bd9Sstevel@tonic-gate /* ARGSUSED */ 5177c478bd9Sstevel@tonic-gate static int 5187c478bd9Sstevel@tonic-gate segkmem_kluster(struct seg *seg, caddr_t addr, ssize_t delta) 5197c478bd9Sstevel@tonic-gate { 5207c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); 5217c478bd9Sstevel@tonic-gate 5227c478bd9Sstevel@tonic-gate if (seg->s_as != &kas) 5237c478bd9Sstevel@tonic-gate segkmem_badop(); 5247c478bd9Sstevel@tonic-gate 5257c478bd9Sstevel@tonic-gate if (segkp_bitmap && seg == &kvseg) { 5267c478bd9Sstevel@tonic-gate 5277c478bd9Sstevel@tonic-gate /* 5287c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp. 5297c478bd9Sstevel@tonic-gate */ 5307c478bd9Sstevel@tonic-gate if (BT_TEST(segkp_bitmap, 5317c478bd9Sstevel@tonic-gate btop((uintptr_t)(addr - seg->s_base)))) 5327c478bd9Sstevel@tonic-gate return (SEGOP_KLUSTER(segkp, addr, delta)); 5337c478bd9Sstevel@tonic-gate } 5347c478bd9Sstevel@tonic-gate segkmem_badop(); 5357c478bd9Sstevel@tonic-gate return (0); 5367c478bd9Sstevel@tonic-gate } 5377c478bd9Sstevel@tonic-gate 5387c478bd9Sstevel@tonic-gate static void 5397c478bd9Sstevel@tonic-gate segkmem_xdump_range(void *arg, void *start, size_t size) 5407c478bd9Sstevel@tonic-gate { 5417c478bd9Sstevel@tonic-gate struct as *as = arg; 5427c478bd9Sstevel@tonic-gate caddr_t addr = start; 5437c478bd9Sstevel@tonic-gate caddr_t addr_end = addr + size; 5447c478bd9Sstevel@tonic-gate 5457c478bd9Sstevel@tonic-gate while (addr < addr_end) { 5467c478bd9Sstevel@tonic-gate pfn_t pfn = hat_getpfnum(kas.a_hat, addr); 5477c478bd9Sstevel@tonic-gate if (pfn != PFN_INVALID && pfn <= physmax && pf_is_memory(pfn)) 5487c478bd9Sstevel@tonic-gate dump_addpage(as, addr, pfn); 5497c478bd9Sstevel@tonic-gate addr += PAGESIZE; 5507c478bd9Sstevel@tonic-gate dump_timeleft = dump_timeout; 5517c478bd9Sstevel@tonic-gate } 5527c478bd9Sstevel@tonic-gate } 5537c478bd9Sstevel@tonic-gate 5547c478bd9Sstevel@tonic-gate static void 5557c478bd9Sstevel@tonic-gate segkmem_dump_range(void *arg, void *start, size_t size) 5567c478bd9Sstevel@tonic-gate { 5577c478bd9Sstevel@tonic-gate caddr_t addr = start; 5587c478bd9Sstevel@tonic-gate caddr_t addr_end = addr + size; 5597c478bd9Sstevel@tonic-gate 5607c478bd9Sstevel@tonic-gate /* 5617c478bd9Sstevel@tonic-gate * If we are about to start dumping the range of addresses we 5627c478bd9Sstevel@tonic-gate * carved out of the kernel heap for the large page heap walk 5637c478bd9Sstevel@tonic-gate * heap_lp_arena to find what segments are actually populated 5647c478bd9Sstevel@tonic-gate */ 5657c478bd9Sstevel@tonic-gate if (SEGKMEM_USE_LARGEPAGES && 5667c478bd9Sstevel@tonic-gate addr == heap_lp_base && addr_end == heap_lp_end && 5677c478bd9Sstevel@tonic-gate vmem_size(heap_lp_arena, VMEM_ALLOC) < size) { 5687c478bd9Sstevel@tonic-gate vmem_walk(heap_lp_arena, VMEM_ALLOC | VMEM_REENTRANT, 5697c478bd9Sstevel@tonic-gate segkmem_xdump_range, arg); 5707c478bd9Sstevel@tonic-gate } else { 5717c478bd9Sstevel@tonic-gate segkmem_xdump_range(arg, start, size); 5727c478bd9Sstevel@tonic-gate } 5737c478bd9Sstevel@tonic-gate } 5747c478bd9Sstevel@tonic-gate 5757c478bd9Sstevel@tonic-gate static void 5767c478bd9Sstevel@tonic-gate segkmem_dump(struct seg *seg) 5777c478bd9Sstevel@tonic-gate { 5787c478bd9Sstevel@tonic-gate /* 5797c478bd9Sstevel@tonic-gate * The kernel's heap_arena (represented by kvseg) is a very large 5807c478bd9Sstevel@tonic-gate * VA space, most of which is typically unused. To speed up dumping 5817c478bd9Sstevel@tonic-gate * we use vmem_walk() to quickly find the pieces of heap_arena that 5827c478bd9Sstevel@tonic-gate * are actually in use. We do the same for heap32_arena and 5837c478bd9Sstevel@tonic-gate * heap_core. 5847c478bd9Sstevel@tonic-gate * 5857c478bd9Sstevel@tonic-gate * We specify VMEM_REENTRANT to vmem_walk() because dump_addpage() 5867c478bd9Sstevel@tonic-gate * may ultimately need to allocate memory. Reentrant walks are 5877c478bd9Sstevel@tonic-gate * necessarily imperfect snapshots. The kernel heap continues 5887c478bd9Sstevel@tonic-gate * to change during a live crash dump, for example. For a normal 5897c478bd9Sstevel@tonic-gate * crash dump, however, we know that there won't be any other threads 5907c478bd9Sstevel@tonic-gate * messing with the heap. Therefore, at worst, we may fail to dump 5917c478bd9Sstevel@tonic-gate * the pages that get allocated by the act of dumping; but we will 5927c478bd9Sstevel@tonic-gate * always dump every page that was allocated when the walk began. 5937c478bd9Sstevel@tonic-gate * 5947c478bd9Sstevel@tonic-gate * The other segkmem segments are dense (fully populated), so there's 5957c478bd9Sstevel@tonic-gate * no need to use this technique when dumping them. 5967c478bd9Sstevel@tonic-gate * 5977c478bd9Sstevel@tonic-gate * Note: when adding special dump handling for any new sparsely- 5987c478bd9Sstevel@tonic-gate * populated segments, be sure to add similar handling to the ::kgrep 5997c478bd9Sstevel@tonic-gate * code in mdb. 6007c478bd9Sstevel@tonic-gate */ 6017c478bd9Sstevel@tonic-gate if (seg == &kvseg) { 6027c478bd9Sstevel@tonic-gate vmem_walk(heap_arena, VMEM_ALLOC | VMEM_REENTRANT, 6037c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 6047c478bd9Sstevel@tonic-gate #ifndef __sparc 6057c478bd9Sstevel@tonic-gate vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT, 6067c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 6077c478bd9Sstevel@tonic-gate #endif 6087c478bd9Sstevel@tonic-gate } else if (seg == &kvseg_core) { 6097c478bd9Sstevel@tonic-gate vmem_walk(heap_core_arena, VMEM_ALLOC | VMEM_REENTRANT, 6107c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 6117c478bd9Sstevel@tonic-gate } else if (seg == &kvseg32) { 6127c478bd9Sstevel@tonic-gate vmem_walk(heap32_arena, VMEM_ALLOC | VMEM_REENTRANT, 6137c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 6147c478bd9Sstevel@tonic-gate vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT, 6157c478bd9Sstevel@tonic-gate segkmem_dump_range, seg->s_as); 6167c478bd9Sstevel@tonic-gate } else { 6177c478bd9Sstevel@tonic-gate segkmem_dump_range(seg->s_as, seg->s_base, seg->s_size); 6187c478bd9Sstevel@tonic-gate } 6197c478bd9Sstevel@tonic-gate } 6207c478bd9Sstevel@tonic-gate 6217c478bd9Sstevel@tonic-gate /* 6227c478bd9Sstevel@tonic-gate * lock/unlock kmem pages over a given range [addr, addr+len). 6237c478bd9Sstevel@tonic-gate * Returns a shadow list of pages in ppp if *ppp is not NULL 6247c478bd9Sstevel@tonic-gate * and memory can be allocated to hold the shadow list. 6257c478bd9Sstevel@tonic-gate */ 6267c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 6277c478bd9Sstevel@tonic-gate static int 6287c478bd9Sstevel@tonic-gate segkmem_pagelock(struct seg *seg, caddr_t addr, size_t len, 6297c478bd9Sstevel@tonic-gate page_t ***ppp, enum lock_type type, enum seg_rw rw) 6307c478bd9Sstevel@tonic-gate { 6317c478bd9Sstevel@tonic-gate page_t **pplist, *pp; 6327c478bd9Sstevel@tonic-gate pgcnt_t npages; 6337c478bd9Sstevel@tonic-gate size_t nb; 6347c478bd9Sstevel@tonic-gate 6357c478bd9Sstevel@tonic-gate if (segkp_bitmap && seg == &kvseg) { 6367c478bd9Sstevel@tonic-gate /* 6377c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp. 6387c478bd9Sstevel@tonic-gate */ 6397c478bd9Sstevel@tonic-gate if (BT_TEST(segkp_bitmap, 6407c478bd9Sstevel@tonic-gate btop((uintptr_t)(addr - seg->s_base)))) 6417c478bd9Sstevel@tonic-gate return (SEGOP_PAGELOCK(segkp, addr, len, ppp, 6427c478bd9Sstevel@tonic-gate type, rw)); 6437c478bd9Sstevel@tonic-gate } 6447c478bd9Sstevel@tonic-gate 6457c478bd9Sstevel@tonic-gate if (type == L_PAGERECLAIM) 6467c478bd9Sstevel@tonic-gate return (ENOTSUP); 6477c478bd9Sstevel@tonic-gate 6487c478bd9Sstevel@tonic-gate npages = btopr(len); 6497c478bd9Sstevel@tonic-gate nb = sizeof (page_t *) * npages; 6507c478bd9Sstevel@tonic-gate 6517c478bd9Sstevel@tonic-gate if (type == L_PAGEUNLOCK) { 6527c478bd9Sstevel@tonic-gate if ((pplist = *ppp) == NULL) { 6537c478bd9Sstevel@tonic-gate /* 6547c478bd9Sstevel@tonic-gate * No shadow list. Iterate over the range 6557c478bd9Sstevel@tonic-gate * using page_find() and unlock the pages 6567c478bd9Sstevel@tonic-gate * that we encounter. 6577c478bd9Sstevel@tonic-gate */ 6587c478bd9Sstevel@tonic-gate while (npages--) { 6597c478bd9Sstevel@tonic-gate pp = page_find(&kvp, 6607c478bd9Sstevel@tonic-gate (u_offset_t)(uintptr_t)addr); 6617c478bd9Sstevel@tonic-gate if (pp) 6627c478bd9Sstevel@tonic-gate page_unlock(pp); 6637c478bd9Sstevel@tonic-gate addr += PAGESIZE; 6647c478bd9Sstevel@tonic-gate } 6657c478bd9Sstevel@tonic-gate return (0); 6667c478bd9Sstevel@tonic-gate } 6677c478bd9Sstevel@tonic-gate 6687c478bd9Sstevel@tonic-gate while (npages--) { 6697c478bd9Sstevel@tonic-gate pp = *pplist++; 6707c478bd9Sstevel@tonic-gate if (pp) 6717c478bd9Sstevel@tonic-gate page_unlock(pp); 6727c478bd9Sstevel@tonic-gate } 6737c478bd9Sstevel@tonic-gate kmem_free(*ppp, nb); 6747c478bd9Sstevel@tonic-gate return (0); 6757c478bd9Sstevel@tonic-gate } 6767c478bd9Sstevel@tonic-gate 6777c478bd9Sstevel@tonic-gate ASSERT(type == L_PAGELOCK); 6787c478bd9Sstevel@tonic-gate 6797c478bd9Sstevel@tonic-gate pplist = NULL; 6807c478bd9Sstevel@tonic-gate if (ppp != NULL) 6817c478bd9Sstevel@tonic-gate *ppp = pplist = kmem_alloc(nb, KM_NOSLEEP); 6827c478bd9Sstevel@tonic-gate 6837c478bd9Sstevel@tonic-gate while (npages--) { 6847c478bd9Sstevel@tonic-gate pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_SHARED); 6857c478bd9Sstevel@tonic-gate /* 6867c478bd9Sstevel@tonic-gate * We'd like to ASSERT(pp != NULL) here, but we can't 6877c478bd9Sstevel@tonic-gate * because there are legitimate cases where the address 6887c478bd9Sstevel@tonic-gate * isn't really mapped -- for instance, attaching a 6897c478bd9Sstevel@tonic-gate * kernel debugger and poking at a non-existent address. 6907c478bd9Sstevel@tonic-gate */ 6917c478bd9Sstevel@tonic-gate if (pplist) 6927c478bd9Sstevel@tonic-gate *pplist++ = pp; 6937c478bd9Sstevel@tonic-gate addr += PAGESIZE; 6947c478bd9Sstevel@tonic-gate } 6957c478bd9Sstevel@tonic-gate return (0); 6967c478bd9Sstevel@tonic-gate } 6977c478bd9Sstevel@tonic-gate 6987c478bd9Sstevel@tonic-gate /* 6997c478bd9Sstevel@tonic-gate * This is a dummy segkmem function overloaded to call segkp 7007c478bd9Sstevel@tonic-gate * when segkp is under the heap. 7017c478bd9Sstevel@tonic-gate */ 7027c478bd9Sstevel@tonic-gate /* ARGSUSED */ 7037c478bd9Sstevel@tonic-gate static int 7047c478bd9Sstevel@tonic-gate segkmem_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 7057c478bd9Sstevel@tonic-gate { 7067c478bd9Sstevel@tonic-gate ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); 7077c478bd9Sstevel@tonic-gate 7087c478bd9Sstevel@tonic-gate if (seg->s_as != &kas) 7097c478bd9Sstevel@tonic-gate segkmem_badop(); 7107c478bd9Sstevel@tonic-gate 7117c478bd9Sstevel@tonic-gate if (segkp_bitmap && seg == &kvseg) { 7127c478bd9Sstevel@tonic-gate 7137c478bd9Sstevel@tonic-gate /* 7147c478bd9Sstevel@tonic-gate * If it is one of segkp pages, call into segkp. 7157c478bd9Sstevel@tonic-gate */ 7167c478bd9Sstevel@tonic-gate if (BT_TEST(segkp_bitmap, 7177c478bd9Sstevel@tonic-gate btop((uintptr_t)(addr - seg->s_base)))) 7187c478bd9Sstevel@tonic-gate return (SEGOP_GETMEMID(segkp, addr, memidp)); 7197c478bd9Sstevel@tonic-gate } 7207c478bd9Sstevel@tonic-gate segkmem_badop(); 7217c478bd9Sstevel@tonic-gate return (0); 7227c478bd9Sstevel@tonic-gate } 7237c478bd9Sstevel@tonic-gate 7247c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 7257c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t * 7267c478bd9Sstevel@tonic-gate segkmem_getpolicy(struct seg *seg, caddr_t addr) 7277c478bd9Sstevel@tonic-gate { 7287c478bd9Sstevel@tonic-gate return (NULL); 7297c478bd9Sstevel@tonic-gate } 7307c478bd9Sstevel@tonic-gate 7317c478bd9Sstevel@tonic-gate 7327c478bd9Sstevel@tonic-gate static struct seg_ops segkmem_ops = { 7337c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* dup */ 7347c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* unmap */ 7357c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(void), /* free */ 7367c478bd9Sstevel@tonic-gate segkmem_fault, 7377c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(faultcode_t), /* faulta */ 7387c478bd9Sstevel@tonic-gate segkmem_setprot, 7397c478bd9Sstevel@tonic-gate segkmem_checkprot, 7407c478bd9Sstevel@tonic-gate segkmem_kluster, 7417c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(size_t), /* swapout */ 7427c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* sync */ 7437c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(size_t), /* incore */ 7447c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* lockop */ 7457c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* getprot */ 7467c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(u_offset_t), /* getoffset */ 7477c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* gettype */ 7487c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* getvp */ 7497c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* advise */ 7507c478bd9Sstevel@tonic-gate segkmem_dump, 7517c478bd9Sstevel@tonic-gate segkmem_pagelock, 7527c478bd9Sstevel@tonic-gate SEGKMEM_BADOP(int), /* setpgsz */ 7537c478bd9Sstevel@tonic-gate segkmem_getmemid, 7547c478bd9Sstevel@tonic-gate segkmem_getpolicy, /* getpolicy */ 7557c478bd9Sstevel@tonic-gate }; 7567c478bd9Sstevel@tonic-gate 7577c478bd9Sstevel@tonic-gate int 7587c478bd9Sstevel@tonic-gate segkmem_create(struct seg *seg) 7597c478bd9Sstevel@tonic-gate { 7607c478bd9Sstevel@tonic-gate ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock)); 7617c478bd9Sstevel@tonic-gate seg->s_ops = &segkmem_ops; 7627c478bd9Sstevel@tonic-gate seg->s_data = NULL; 7637c478bd9Sstevel@tonic-gate kas.a_size += seg->s_size; 7647c478bd9Sstevel@tonic-gate return (0); 7657c478bd9Sstevel@tonic-gate } 7667c478bd9Sstevel@tonic-gate 7677c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 7687c478bd9Sstevel@tonic-gate page_t * 7697c478bd9Sstevel@tonic-gate segkmem_page_create(void *addr, size_t size, int vmflag, void *arg) 7707c478bd9Sstevel@tonic-gate { 7717c478bd9Sstevel@tonic-gate struct seg kseg; 7727c478bd9Sstevel@tonic-gate int pgflags; 7737c478bd9Sstevel@tonic-gate 7747c478bd9Sstevel@tonic-gate kseg.s_as = &kas; 7757c478bd9Sstevel@tonic-gate pgflags = PG_EXCL; 7767c478bd9Sstevel@tonic-gate 7777c478bd9Sstevel@tonic-gate if (segkmem_reloc == 0 || (vmflag & VM_NORELOC)) 7787c478bd9Sstevel@tonic-gate pgflags |= PG_NORELOC; 7797c478bd9Sstevel@tonic-gate if ((vmflag & VM_NOSLEEP) == 0) 7807c478bd9Sstevel@tonic-gate pgflags |= PG_WAIT; 7817c478bd9Sstevel@tonic-gate if (vmflag & VM_PANIC) 7827c478bd9Sstevel@tonic-gate pgflags |= PG_PANIC; 7837c478bd9Sstevel@tonic-gate if (vmflag & VM_PUSHPAGE) 7847c478bd9Sstevel@tonic-gate pgflags |= PG_PUSHPAGE; 7857c478bd9Sstevel@tonic-gate 7867c478bd9Sstevel@tonic-gate return (page_create_va(&kvp, (u_offset_t)(uintptr_t)addr, size, 7877c478bd9Sstevel@tonic-gate pgflags, &kseg, addr)); 7887c478bd9Sstevel@tonic-gate } 7897c478bd9Sstevel@tonic-gate 7907c478bd9Sstevel@tonic-gate /* 7917c478bd9Sstevel@tonic-gate * Allocate pages to back the virtual address range [addr, addr + size). 7927c478bd9Sstevel@tonic-gate * If addr is NULL, allocate the virtual address space as well. 7937c478bd9Sstevel@tonic-gate */ 7947c478bd9Sstevel@tonic-gate void * 7957c478bd9Sstevel@tonic-gate segkmem_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag, uint_t attr, 7967c478bd9Sstevel@tonic-gate page_t *(*page_create_func)(void *, size_t, int, void *), void *pcarg) 7977c478bd9Sstevel@tonic-gate { 7987c478bd9Sstevel@tonic-gate page_t *ppl; 7997c478bd9Sstevel@tonic-gate caddr_t addr = inaddr; 8007c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size); 8017c478bd9Sstevel@tonic-gate int allocflag; 8027c478bd9Sstevel@tonic-gate 8037c478bd9Sstevel@tonic-gate if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL) 8047c478bd9Sstevel@tonic-gate return (NULL); 8057c478bd9Sstevel@tonic-gate 8067c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0); 8077c478bd9Sstevel@tonic-gate 8087c478bd9Sstevel@tonic-gate if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) { 8097c478bd9Sstevel@tonic-gate if (inaddr == NULL) 8107c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size); 8117c478bd9Sstevel@tonic-gate return (NULL); 8127c478bd9Sstevel@tonic-gate } 8137c478bd9Sstevel@tonic-gate 8147c478bd9Sstevel@tonic-gate ppl = page_create_func(addr, size, vmflag, pcarg); 8157c478bd9Sstevel@tonic-gate if (ppl == NULL) { 8167c478bd9Sstevel@tonic-gate if (inaddr == NULL) 8177c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size); 8187c478bd9Sstevel@tonic-gate page_unresv(npages); 8197c478bd9Sstevel@tonic-gate return (NULL); 8207c478bd9Sstevel@tonic-gate } 8217c478bd9Sstevel@tonic-gate 8227c478bd9Sstevel@tonic-gate /* 8237c478bd9Sstevel@tonic-gate * Under certain conditions, we need to let the HAT layer know 8247c478bd9Sstevel@tonic-gate * that it cannot safely allocate memory. Allocations from 8257c478bd9Sstevel@tonic-gate * the hat_memload vmem arena always need this, to prevent 8267c478bd9Sstevel@tonic-gate * infinite recursion. 8277c478bd9Sstevel@tonic-gate * 8287c478bd9Sstevel@tonic-gate * In addition, the x86 hat cannot safely do memory 8297c478bd9Sstevel@tonic-gate * allocations while in vmem_populate(), because there 8307c478bd9Sstevel@tonic-gate * is no simple bound on its usage. 8317c478bd9Sstevel@tonic-gate */ 8327c478bd9Sstevel@tonic-gate if (vmflag & VM_MEMLOAD) 8337c478bd9Sstevel@tonic-gate allocflag = HAT_NO_KALLOC; 8347c478bd9Sstevel@tonic-gate #if defined(__x86) 8357c478bd9Sstevel@tonic-gate else if (vmem_is_populator()) 8367c478bd9Sstevel@tonic-gate allocflag = HAT_NO_KALLOC; 8377c478bd9Sstevel@tonic-gate #endif 8387c478bd9Sstevel@tonic-gate else 8397c478bd9Sstevel@tonic-gate allocflag = 0; 8407c478bd9Sstevel@tonic-gate 8417c478bd9Sstevel@tonic-gate while (ppl != NULL) { 8427c478bd9Sstevel@tonic-gate page_t *pp = ppl; 8437c478bd9Sstevel@tonic-gate page_sub(&ppl, pp); 8447c478bd9Sstevel@tonic-gate ASSERT(page_iolock_assert(pp)); 8457c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 8467c478bd9Sstevel@tonic-gate page_io_unlock(pp); 8477c478bd9Sstevel@tonic-gate hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset, pp, 8487c478bd9Sstevel@tonic-gate (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr, 8497c478bd9Sstevel@tonic-gate HAT_LOAD_LOCK | allocflag); 8507c478bd9Sstevel@tonic-gate pp->p_lckcnt = 1; 8517c478bd9Sstevel@tonic-gate #if defined(__x86) 8527c478bd9Sstevel@tonic-gate page_downgrade(pp); 8537c478bd9Sstevel@tonic-gate #else 8547c478bd9Sstevel@tonic-gate if (vmflag & SEGKMEM_SHARELOCKED) 8557c478bd9Sstevel@tonic-gate page_downgrade(pp); 8567c478bd9Sstevel@tonic-gate else 8577c478bd9Sstevel@tonic-gate page_unlock(pp); 8587c478bd9Sstevel@tonic-gate #endif 8597c478bd9Sstevel@tonic-gate } 8607c478bd9Sstevel@tonic-gate 8617c478bd9Sstevel@tonic-gate return (addr); 8627c478bd9Sstevel@tonic-gate } 8637c478bd9Sstevel@tonic-gate 8647c478bd9Sstevel@tonic-gate void * 8657c478bd9Sstevel@tonic-gate segkmem_alloc(vmem_t *vmp, size_t size, int vmflag) 8667c478bd9Sstevel@tonic-gate { 8677c478bd9Sstevel@tonic-gate void *addr; 8687c478bd9Sstevel@tonic-gate segkmem_gc_list_t *gcp, **prev_gcpp; 8697c478bd9Sstevel@tonic-gate 8707c478bd9Sstevel@tonic-gate if (kvseg.s_base == NULL) { 8717c478bd9Sstevel@tonic-gate #ifndef __sparc 8727c478bd9Sstevel@tonic-gate if (bootops->bsys_alloc == NULL) 8737c478bd9Sstevel@tonic-gate halt("Memory allocation between bop_alloc() and " 8747c478bd9Sstevel@tonic-gate "kmem_alloc().\n"); 8757c478bd9Sstevel@tonic-gate #endif 8767c478bd9Sstevel@tonic-gate 8777c478bd9Sstevel@tonic-gate /* 8787c478bd9Sstevel@tonic-gate * There's not a lot of memory to go around during boot, 8797c478bd9Sstevel@tonic-gate * so recycle it if we can. 8807c478bd9Sstevel@tonic-gate */ 8817c478bd9Sstevel@tonic-gate for (prev_gcpp = &segkmem_gc_list; (gcp = *prev_gcpp) != NULL; 8827c478bd9Sstevel@tonic-gate prev_gcpp = &gcp->gc_next) { 8837c478bd9Sstevel@tonic-gate if (gcp->gc_arena == vmp && gcp->gc_size == size) { 8847c478bd9Sstevel@tonic-gate *prev_gcpp = gcp->gc_next; 8857c478bd9Sstevel@tonic-gate return (gcp); 8867c478bd9Sstevel@tonic-gate } 8877c478bd9Sstevel@tonic-gate } 8887c478bd9Sstevel@tonic-gate 8897c478bd9Sstevel@tonic-gate addr = vmem_alloc(vmp, size, vmflag | VM_PANIC); 8907c478bd9Sstevel@tonic-gate if (boot_alloc(addr, size, BO_NO_ALIGN) != addr) 8917c478bd9Sstevel@tonic-gate panic("segkmem_alloc: boot_alloc failed"); 8927c478bd9Sstevel@tonic-gate return (addr); 8937c478bd9Sstevel@tonic-gate } 8947c478bd9Sstevel@tonic-gate return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, 8957c478bd9Sstevel@tonic-gate segkmem_page_create, NULL)); 8967c478bd9Sstevel@tonic-gate } 8977c478bd9Sstevel@tonic-gate 8987c478bd9Sstevel@tonic-gate /* 8997c478bd9Sstevel@tonic-gate * Any changes to this routine must also be carried over to 9007c478bd9Sstevel@tonic-gate * devmap_free_pages() in the seg_dev driver. This is because 9017c478bd9Sstevel@tonic-gate * we currently don't have a special kernel segment for non-paged 9027c478bd9Sstevel@tonic-gate * kernel memory that is exported by drivers to user space. 9037c478bd9Sstevel@tonic-gate */ 9047c478bd9Sstevel@tonic-gate void 9057c478bd9Sstevel@tonic-gate segkmem_free(vmem_t *vmp, void *inaddr, size_t size) 9067c478bd9Sstevel@tonic-gate { 9077c478bd9Sstevel@tonic-gate page_t *pp; 9087c478bd9Sstevel@tonic-gate caddr_t addr = inaddr; 9097c478bd9Sstevel@tonic-gate caddr_t eaddr; 9107c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size); 9117c478bd9Sstevel@tonic-gate 9127c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0); 9137c478bd9Sstevel@tonic-gate 9147c478bd9Sstevel@tonic-gate if (kvseg.s_base == NULL) { 9157c478bd9Sstevel@tonic-gate segkmem_gc_list_t *gc = inaddr; 9167c478bd9Sstevel@tonic-gate gc->gc_arena = vmp; 9177c478bd9Sstevel@tonic-gate gc->gc_size = size; 9187c478bd9Sstevel@tonic-gate gc->gc_next = segkmem_gc_list; 9197c478bd9Sstevel@tonic-gate segkmem_gc_list = gc; 9207c478bd9Sstevel@tonic-gate return; 9217c478bd9Sstevel@tonic-gate } 9227c478bd9Sstevel@tonic-gate 9237c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK); 9247c478bd9Sstevel@tonic-gate 9257c478bd9Sstevel@tonic-gate for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) { 9267c478bd9Sstevel@tonic-gate #if defined(__x86) 9277c478bd9Sstevel@tonic-gate pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr); 9287c478bd9Sstevel@tonic-gate if (pp == NULL) 9297c478bd9Sstevel@tonic-gate panic("segkmem_free: page not found"); 9307c478bd9Sstevel@tonic-gate if (!page_tryupgrade(pp)) { 9317c478bd9Sstevel@tonic-gate /* 9327c478bd9Sstevel@tonic-gate * Some other thread has a sharelock. Wait for 9337c478bd9Sstevel@tonic-gate * it to drop the lock so we can free this page. 9347c478bd9Sstevel@tonic-gate */ 9357c478bd9Sstevel@tonic-gate page_unlock(pp); 9367c478bd9Sstevel@tonic-gate pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, 9377c478bd9Sstevel@tonic-gate SE_EXCL); 9387c478bd9Sstevel@tonic-gate } 9397c478bd9Sstevel@tonic-gate #else 9407c478bd9Sstevel@tonic-gate pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL); 9417c478bd9Sstevel@tonic-gate #endif 9427c478bd9Sstevel@tonic-gate if (pp == NULL) 9437c478bd9Sstevel@tonic-gate panic("segkmem_free: page not found"); 9447c478bd9Sstevel@tonic-gate /* Clear p_lckcnt so page_destroy() doesn't update availrmem */ 9457c478bd9Sstevel@tonic-gate pp->p_lckcnt = 0; 9467c478bd9Sstevel@tonic-gate page_destroy(pp, 0); 9477c478bd9Sstevel@tonic-gate } 9487c478bd9Sstevel@tonic-gate page_unresv(npages); 9497c478bd9Sstevel@tonic-gate 9507c478bd9Sstevel@tonic-gate if (vmp != NULL) 9517c478bd9Sstevel@tonic-gate vmem_free(vmp, inaddr, size); 9527c478bd9Sstevel@tonic-gate } 9537c478bd9Sstevel@tonic-gate 9547c478bd9Sstevel@tonic-gate void 9557c478bd9Sstevel@tonic-gate segkmem_gc(void) 9567c478bd9Sstevel@tonic-gate { 9577c478bd9Sstevel@tonic-gate ASSERT(kvseg.s_base != NULL); 9587c478bd9Sstevel@tonic-gate while (segkmem_gc_list != NULL) { 9597c478bd9Sstevel@tonic-gate segkmem_gc_list_t *gc = segkmem_gc_list; 9607c478bd9Sstevel@tonic-gate segkmem_gc_list = gc->gc_next; 9617c478bd9Sstevel@tonic-gate segkmem_free(gc->gc_arena, gc, gc->gc_size); 9627c478bd9Sstevel@tonic-gate } 9637c478bd9Sstevel@tonic-gate } 9647c478bd9Sstevel@tonic-gate 9657c478bd9Sstevel@tonic-gate /* 9667c478bd9Sstevel@tonic-gate * Legacy entry points from here to end of file. 9677c478bd9Sstevel@tonic-gate */ 9687c478bd9Sstevel@tonic-gate void 9697c478bd9Sstevel@tonic-gate segkmem_mapin(struct seg *seg, void *addr, size_t size, uint_t vprot, 9707c478bd9Sstevel@tonic-gate pfn_t pfn, uint_t flags) 9717c478bd9Sstevel@tonic-gate { 9727c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK); 9737c478bd9Sstevel@tonic-gate hat_devload(seg->s_as->a_hat, addr, size, pfn, vprot, 9747c478bd9Sstevel@tonic-gate flags | HAT_LOAD_LOCK); 9757c478bd9Sstevel@tonic-gate } 9767c478bd9Sstevel@tonic-gate 9777c478bd9Sstevel@tonic-gate void 9787c478bd9Sstevel@tonic-gate segkmem_mapout(struct seg *seg, void *addr, size_t size) 9797c478bd9Sstevel@tonic-gate { 9807c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK); 9817c478bd9Sstevel@tonic-gate } 9827c478bd9Sstevel@tonic-gate 9837c478bd9Sstevel@tonic-gate void * 9847c478bd9Sstevel@tonic-gate kmem_getpages(pgcnt_t npages, int kmflag) 9857c478bd9Sstevel@tonic-gate { 9867c478bd9Sstevel@tonic-gate return (kmem_alloc(ptob(npages), kmflag)); 9877c478bd9Sstevel@tonic-gate } 9887c478bd9Sstevel@tonic-gate 9897c478bd9Sstevel@tonic-gate void 9907c478bd9Sstevel@tonic-gate kmem_freepages(void *addr, pgcnt_t npages) 9917c478bd9Sstevel@tonic-gate { 9927c478bd9Sstevel@tonic-gate kmem_free(addr, ptob(npages)); 9937c478bd9Sstevel@tonic-gate } 9947c478bd9Sstevel@tonic-gate 9957c478bd9Sstevel@tonic-gate /* 9967c478bd9Sstevel@tonic-gate * segkmem_page_create_large() allocates a large page to be used for the kmem 9977c478bd9Sstevel@tonic-gate * caches. If kpr is enabled we ask for a relocatable page unless requested 9987c478bd9Sstevel@tonic-gate * otherwise. If kpr is disabled we have to ask for a non-reloc page 9997c478bd9Sstevel@tonic-gate */ 10007c478bd9Sstevel@tonic-gate static page_t * 10017c478bd9Sstevel@tonic-gate segkmem_page_create_large(void *addr, size_t size, int vmflag, void *arg) 10027c478bd9Sstevel@tonic-gate { 10037c478bd9Sstevel@tonic-gate int pgflags; 10047c478bd9Sstevel@tonic-gate 10057c478bd9Sstevel@tonic-gate pgflags = PG_EXCL; 10067c478bd9Sstevel@tonic-gate 10077c478bd9Sstevel@tonic-gate if (segkmem_reloc == 0 || (vmflag & VM_NORELOC)) 10087c478bd9Sstevel@tonic-gate pgflags |= PG_NORELOC; 10097c478bd9Sstevel@tonic-gate if (!(vmflag & VM_NOSLEEP)) 10107c478bd9Sstevel@tonic-gate pgflags |= PG_WAIT; 10117c478bd9Sstevel@tonic-gate if (vmflag & VM_PUSHPAGE) 10127c478bd9Sstevel@tonic-gate pgflags |= PG_PUSHPAGE; 10137c478bd9Sstevel@tonic-gate 10147c478bd9Sstevel@tonic-gate return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size, 10157c478bd9Sstevel@tonic-gate pgflags, &kvseg, addr, arg)); 10167c478bd9Sstevel@tonic-gate } 10177c478bd9Sstevel@tonic-gate 10187c478bd9Sstevel@tonic-gate /* 10197c478bd9Sstevel@tonic-gate * Allocate a large page to back the virtual address range 10207c478bd9Sstevel@tonic-gate * [addr, addr + size). If addr is NULL, allocate the virtual address 10217c478bd9Sstevel@tonic-gate * space as well. 10227c478bd9Sstevel@tonic-gate */ 10237c478bd9Sstevel@tonic-gate static void * 10247c478bd9Sstevel@tonic-gate segkmem_xalloc_lp(vmem_t *vmp, void *inaddr, size_t size, int vmflag, 10257c478bd9Sstevel@tonic-gate uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *), 10267c478bd9Sstevel@tonic-gate void *pcarg) 10277c478bd9Sstevel@tonic-gate { 10287c478bd9Sstevel@tonic-gate caddr_t addr = inaddr, pa; 10297c478bd9Sstevel@tonic-gate size_t lpsize = segkmem_lpsize; 10307c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size); 10317c478bd9Sstevel@tonic-gate pgcnt_t nbpages = btop(lpsize); 10327c478bd9Sstevel@tonic-gate pgcnt_t nlpages = size >> segkmem_lpshift; 10337c478bd9Sstevel@tonic-gate size_t ppasize = nbpages * sizeof (page_t *); 10347c478bd9Sstevel@tonic-gate page_t *pp, *rootpp, **ppa, *pplist = NULL; 10357c478bd9Sstevel@tonic-gate int i; 10367c478bd9Sstevel@tonic-gate 10377c478bd9Sstevel@tonic-gate if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) { 10387c478bd9Sstevel@tonic-gate return (NULL); 10397c478bd9Sstevel@tonic-gate } 10407c478bd9Sstevel@tonic-gate 10417c478bd9Sstevel@tonic-gate /* 10427c478bd9Sstevel@tonic-gate * allocate an array we need for hat_memload_array. 10437c478bd9Sstevel@tonic-gate * we use a separate arena to avoid recursion. 10447c478bd9Sstevel@tonic-gate * we will not need this array when hat_memload_array learns pp++ 10457c478bd9Sstevel@tonic-gate */ 10467c478bd9Sstevel@tonic-gate if ((ppa = vmem_alloc(segkmem_ppa_arena, ppasize, vmflag)) == NULL) { 10477c478bd9Sstevel@tonic-gate goto fail_array_alloc; 10487c478bd9Sstevel@tonic-gate } 10497c478bd9Sstevel@tonic-gate 10507c478bd9Sstevel@tonic-gate if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL) 10517c478bd9Sstevel@tonic-gate goto fail_vmem_alloc; 10527c478bd9Sstevel@tonic-gate 10537c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & (lpsize - 1)) == 0); 10547c478bd9Sstevel@tonic-gate 10557c478bd9Sstevel@tonic-gate /* create all the pages */ 10567c478bd9Sstevel@tonic-gate for (pa = addr, i = 0; i < nlpages; i++, pa += lpsize) { 10577c478bd9Sstevel@tonic-gate if ((pp = page_create_func(pa, lpsize, vmflag, pcarg)) == NULL) 10587c478bd9Sstevel@tonic-gate goto fail_page_create; 10597c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &pp); 10607c478bd9Sstevel@tonic-gate } 10617c478bd9Sstevel@tonic-gate 10627c478bd9Sstevel@tonic-gate /* at this point we have all the resource to complete the request */ 10637c478bd9Sstevel@tonic-gate while ((rootpp = pplist) != NULL) { 10647c478bd9Sstevel@tonic-gate for (i = 0; i < nbpages; i++) { 10657c478bd9Sstevel@tonic-gate ASSERT(pplist != NULL); 10667c478bd9Sstevel@tonic-gate pp = pplist; 10677c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 10687c478bd9Sstevel@tonic-gate ASSERT(page_iolock_assert(pp)); 10697c478bd9Sstevel@tonic-gate page_io_unlock(pp); 10707c478bd9Sstevel@tonic-gate ppa[i] = pp; 10717c478bd9Sstevel@tonic-gate } 10727c478bd9Sstevel@tonic-gate /* 10737c478bd9Sstevel@tonic-gate * Load the locked entry. It's OK to preload the entry into the 10747c478bd9Sstevel@tonic-gate * TSB since we now support large mappings in the kernel TSB. 10757c478bd9Sstevel@tonic-gate */ 10767c478bd9Sstevel@tonic-gate hat_memload_array(kas.a_hat, 10777c478bd9Sstevel@tonic-gate (caddr_t)(uintptr_t)rootpp->p_offset, lpsize, 10787c478bd9Sstevel@tonic-gate ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr, 10797c478bd9Sstevel@tonic-gate HAT_LOAD_LOCK); 10807c478bd9Sstevel@tonic-gate 10817c478bd9Sstevel@tonic-gate for (--i; i >= 0; --i) { 10827c478bd9Sstevel@tonic-gate ppa[i]->p_lckcnt = 1; 10837c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 10847c478bd9Sstevel@tonic-gate } 10857c478bd9Sstevel@tonic-gate } 10867c478bd9Sstevel@tonic-gate 10877c478bd9Sstevel@tonic-gate vmem_free(segkmem_ppa_arena, ppa, ppasize); 10887c478bd9Sstevel@tonic-gate return (addr); 10897c478bd9Sstevel@tonic-gate 10907c478bd9Sstevel@tonic-gate fail_page_create: 10917c478bd9Sstevel@tonic-gate while ((rootpp = pplist) != NULL) { 10927c478bd9Sstevel@tonic-gate for (i = 0, pp = pplist; i < nbpages; i++, pp = pplist) { 10937c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 10947c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 10957c478bd9Sstevel@tonic-gate ASSERT(page_iolock_assert(pp)); 10967c478bd9Sstevel@tonic-gate page_io_unlock(pp); 10977c478bd9Sstevel@tonic-gate } 10987c478bd9Sstevel@tonic-gate page_destroy_pages(rootpp); 10997c478bd9Sstevel@tonic-gate } 11007c478bd9Sstevel@tonic-gate 11017c478bd9Sstevel@tonic-gate if (inaddr == NULL) 11027c478bd9Sstevel@tonic-gate vmem_free(vmp, addr, size); 11037c478bd9Sstevel@tonic-gate 11047c478bd9Sstevel@tonic-gate fail_vmem_alloc: 11057c478bd9Sstevel@tonic-gate vmem_free(segkmem_ppa_arena, ppa, ppasize); 11067c478bd9Sstevel@tonic-gate 11077c478bd9Sstevel@tonic-gate fail_array_alloc: 11087c478bd9Sstevel@tonic-gate page_unresv(npages); 11097c478bd9Sstevel@tonic-gate 11107c478bd9Sstevel@tonic-gate return (NULL); 11117c478bd9Sstevel@tonic-gate } 11127c478bd9Sstevel@tonic-gate 11137c478bd9Sstevel@tonic-gate static void 11147c478bd9Sstevel@tonic-gate segkmem_free_one_lp(caddr_t addr, size_t size) 11157c478bd9Sstevel@tonic-gate { 11167c478bd9Sstevel@tonic-gate page_t *pp, *rootpp = NULL; 11177c478bd9Sstevel@tonic-gate pgcnt_t pgs_left = btopr(size); 11187c478bd9Sstevel@tonic-gate 11197c478bd9Sstevel@tonic-gate ASSERT(size == segkmem_lpsize); 11207c478bd9Sstevel@tonic-gate 11217c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK); 11227c478bd9Sstevel@tonic-gate 11237c478bd9Sstevel@tonic-gate for (; pgs_left > 0; addr += PAGESIZE, pgs_left--) { 11247c478bd9Sstevel@tonic-gate pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL); 11257c478bd9Sstevel@tonic-gate if (pp == NULL) 11267c478bd9Sstevel@tonic-gate panic("segkmem_free_one_lp: page not found"); 11277c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 11287c478bd9Sstevel@tonic-gate pp->p_lckcnt = 0; 11297c478bd9Sstevel@tonic-gate if (rootpp == NULL) 11307c478bd9Sstevel@tonic-gate rootpp = pp; 11317c478bd9Sstevel@tonic-gate } 11327c478bd9Sstevel@tonic-gate ASSERT(rootpp != NULL); 11337c478bd9Sstevel@tonic-gate page_destroy_pages(rootpp); 11347c478bd9Sstevel@tonic-gate 11357c478bd9Sstevel@tonic-gate /* page_unresv() is done by the caller */ 11367c478bd9Sstevel@tonic-gate } 11377c478bd9Sstevel@tonic-gate 11387c478bd9Sstevel@tonic-gate /* 11397c478bd9Sstevel@tonic-gate * This function is called to import new spans into the vmem arenas like 11407c478bd9Sstevel@tonic-gate * kmem_default_arena and kmem_oversize_arena. It first tries to import 11417c478bd9Sstevel@tonic-gate * spans from large page arena - kmem_lp_arena. In order to do this it might 11427c478bd9Sstevel@tonic-gate * have to "upgrade the requested size" to kmem_lp_arena quantum. If 11437c478bd9Sstevel@tonic-gate * it was not able to satisfy the upgraded request it then calls regular 11447c478bd9Sstevel@tonic-gate * segkmem_alloc() that satisfies the request by importing from "*vmp" arena 11457c478bd9Sstevel@tonic-gate */ 11467c478bd9Sstevel@tonic-gate void * 11477c478bd9Sstevel@tonic-gate segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, int vmflag) 11487c478bd9Sstevel@tonic-gate { 11497c478bd9Sstevel@tonic-gate size_t size; 11507c478bd9Sstevel@tonic-gate kthread_t *t = curthread; 11517c478bd9Sstevel@tonic-gate segkmem_lpcb_t *lpcb = &segkmem_lpcb; 11527c478bd9Sstevel@tonic-gate 11537c478bd9Sstevel@tonic-gate ASSERT(sizep != NULL); 11547c478bd9Sstevel@tonic-gate 11557c478bd9Sstevel@tonic-gate size = *sizep; 11567c478bd9Sstevel@tonic-gate 11577c478bd9Sstevel@tonic-gate if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) && 11587c478bd9Sstevel@tonic-gate !(vmflag & SEGKMEM_SHARELOCKED)) { 11597c478bd9Sstevel@tonic-gate 11607c478bd9Sstevel@tonic-gate size_t kmemlp_qnt = segkmem_kmemlp_quantum; 11617c478bd9Sstevel@tonic-gate size_t asize = P2ROUNDUP(size, kmemlp_qnt); 11627c478bd9Sstevel@tonic-gate void *addr = NULL; 11637c478bd9Sstevel@tonic-gate ulong_t *lpthrtp = &lpcb->lp_throttle; 11647c478bd9Sstevel@tonic-gate ulong_t lpthrt = *lpthrtp; 11657c478bd9Sstevel@tonic-gate int dowakeup = 0; 11667c478bd9Sstevel@tonic-gate int doalloc = 1; 11677c478bd9Sstevel@tonic-gate 11687c478bd9Sstevel@tonic-gate ASSERT(kmem_lp_arena != NULL); 11697c478bd9Sstevel@tonic-gate ASSERT(asize >= size); 11707c478bd9Sstevel@tonic-gate 11717c478bd9Sstevel@tonic-gate if (lpthrt != 0) { 11727c478bd9Sstevel@tonic-gate /* try to update the throttle value */ 11737c478bd9Sstevel@tonic-gate lpthrt = atomic_add_long_nv(lpthrtp, 1); 11747c478bd9Sstevel@tonic-gate if (lpthrt >= segkmem_lpthrottle_max) { 11757c478bd9Sstevel@tonic-gate lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 11767c478bd9Sstevel@tonic-gate segkmem_lpthrottle_max / 4); 11777c478bd9Sstevel@tonic-gate } 11787c478bd9Sstevel@tonic-gate 11797c478bd9Sstevel@tonic-gate /* 11807c478bd9Sstevel@tonic-gate * when we get above throttle start do an exponential 11817c478bd9Sstevel@tonic-gate * backoff at trying large pages and reaping 11827c478bd9Sstevel@tonic-gate */ 11837c478bd9Sstevel@tonic-gate if (lpthrt > segkmem_lpthrottle_start && 11847c478bd9Sstevel@tonic-gate (lpthrt & (lpthrt - 1))) { 11857c478bd9Sstevel@tonic-gate atomic_add_64(&lpcb->allocs_throttled, 1L); 11867c478bd9Sstevel@tonic-gate lpthrt--; 11877c478bd9Sstevel@tonic-gate if ((lpthrt & (lpthrt - 1)) == 0) 11887c478bd9Sstevel@tonic-gate kmem_reap(); 11897c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, vmflag)); 11907c478bd9Sstevel@tonic-gate } 11917c478bd9Sstevel@tonic-gate } 11927c478bd9Sstevel@tonic-gate 11937c478bd9Sstevel@tonic-gate if (!(vmflag & VM_NOSLEEP) && 11947c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum >= (8 * kmemlp_qnt) && 11957c478bd9Sstevel@tonic-gate vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt && 11967c478bd9Sstevel@tonic-gate asize < (segkmem_heaplp_quantum - kmemlp_qnt)) { 11977c478bd9Sstevel@tonic-gate 11987c478bd9Sstevel@tonic-gate /* 11997c478bd9Sstevel@tonic-gate * we are low on free memory in kmem_lp_arena 12007c478bd9Sstevel@tonic-gate * we let only one guy to allocate heap_lp 12017c478bd9Sstevel@tonic-gate * quantum size chunk that everybody is going to 12027c478bd9Sstevel@tonic-gate * share 12037c478bd9Sstevel@tonic-gate */ 12047c478bd9Sstevel@tonic-gate mutex_enter(&lpcb->lp_lock); 12057c478bd9Sstevel@tonic-gate 12067c478bd9Sstevel@tonic-gate if (lpcb->lp_wait) { 12077c478bd9Sstevel@tonic-gate 12087c478bd9Sstevel@tonic-gate /* we are not the first one - wait */ 12097c478bd9Sstevel@tonic-gate cv_wait(&lpcb->lp_cv, &lpcb->lp_lock); 12107c478bd9Sstevel@tonic-gate if (vmem_size(kmem_lp_arena, VMEM_FREE) < 12117c478bd9Sstevel@tonic-gate kmemlp_qnt) { 12127c478bd9Sstevel@tonic-gate doalloc = 0; 12137c478bd9Sstevel@tonic-gate } 12147c478bd9Sstevel@tonic-gate } else if (vmem_size(kmem_lp_arena, VMEM_FREE) <= 12157c478bd9Sstevel@tonic-gate kmemlp_qnt) { 12167c478bd9Sstevel@tonic-gate 12177c478bd9Sstevel@tonic-gate /* 12187c478bd9Sstevel@tonic-gate * we are the first one, make sure we import 12197c478bd9Sstevel@tonic-gate * a large page 12207c478bd9Sstevel@tonic-gate */ 12217c478bd9Sstevel@tonic-gate if (asize == kmemlp_qnt) 12227c478bd9Sstevel@tonic-gate asize += kmemlp_qnt; 12237c478bd9Sstevel@tonic-gate dowakeup = 1; 12247c478bd9Sstevel@tonic-gate lpcb->lp_wait = 1; 12257c478bd9Sstevel@tonic-gate } 12267c478bd9Sstevel@tonic-gate 12277c478bd9Sstevel@tonic-gate mutex_exit(&lpcb->lp_lock); 12287c478bd9Sstevel@tonic-gate } 12297c478bd9Sstevel@tonic-gate 12307c478bd9Sstevel@tonic-gate /* 12317c478bd9Sstevel@tonic-gate * VM_ABORT flag prevents sleeps in vmem_xalloc when 12327c478bd9Sstevel@tonic-gate * large pages are not available. In that case this allocation 12337c478bd9Sstevel@tonic-gate * attempt will fail and we will retry allocation with small 12347c478bd9Sstevel@tonic-gate * pages. We also do not want to panic if this allocation fails 12357c478bd9Sstevel@tonic-gate * because we are going to retry. 12367c478bd9Sstevel@tonic-gate */ 12377c478bd9Sstevel@tonic-gate if (doalloc) { 12387c478bd9Sstevel@tonic-gate addr = vmem_alloc(kmem_lp_arena, asize, 12397c478bd9Sstevel@tonic-gate (vmflag | VM_ABORT) & ~VM_PANIC); 12407c478bd9Sstevel@tonic-gate 12417c478bd9Sstevel@tonic-gate if (dowakeup) { 12427c478bd9Sstevel@tonic-gate mutex_enter(&lpcb->lp_lock); 12437c478bd9Sstevel@tonic-gate ASSERT(lpcb->lp_wait != 0); 12447c478bd9Sstevel@tonic-gate lpcb->lp_wait = 0; 12457c478bd9Sstevel@tonic-gate cv_broadcast(&lpcb->lp_cv); 12467c478bd9Sstevel@tonic-gate mutex_exit(&lpcb->lp_lock); 12477c478bd9Sstevel@tonic-gate } 12487c478bd9Sstevel@tonic-gate } 12497c478bd9Sstevel@tonic-gate 12507c478bd9Sstevel@tonic-gate if (addr != NULL) { 12517c478bd9Sstevel@tonic-gate *sizep = asize; 12527c478bd9Sstevel@tonic-gate *lpthrtp = 0; 12537c478bd9Sstevel@tonic-gate return (addr); 12547c478bd9Sstevel@tonic-gate } 12557c478bd9Sstevel@tonic-gate 12567c478bd9Sstevel@tonic-gate if (vmflag & VM_NOSLEEP) 12577c478bd9Sstevel@tonic-gate atomic_add_64(&lpcb->nosleep_allocs_failed, 1L); 12587c478bd9Sstevel@tonic-gate else 12597c478bd9Sstevel@tonic-gate atomic_add_64(&lpcb->sleep_allocs_failed, 1L); 12607c478bd9Sstevel@tonic-gate atomic_add_64(&lpcb->alloc_bytes_failed, size); 12617c478bd9Sstevel@tonic-gate 12627c478bd9Sstevel@tonic-gate /* if large page throttling is not started yet do it */ 12637c478bd9Sstevel@tonic-gate if (segkmem_use_lpthrottle && lpthrt == 0) { 12647c478bd9Sstevel@tonic-gate lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1); 12657c478bd9Sstevel@tonic-gate } 12667c478bd9Sstevel@tonic-gate } 12677c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, vmflag)); 12687c478bd9Sstevel@tonic-gate } 12697c478bd9Sstevel@tonic-gate 12707c478bd9Sstevel@tonic-gate void 12717c478bd9Sstevel@tonic-gate segkmem_free_lp(vmem_t *vmp, void *inaddr, size_t size) 12727c478bd9Sstevel@tonic-gate { 12737c478bd9Sstevel@tonic-gate if (kmem_lp_arena == NULL || !IS_KMEM_VA_LARGEPAGE((caddr_t)inaddr)) { 12747c478bd9Sstevel@tonic-gate segkmem_free(vmp, inaddr, size); 12757c478bd9Sstevel@tonic-gate } else { 12767c478bd9Sstevel@tonic-gate vmem_free(kmem_lp_arena, inaddr, size); 12777c478bd9Sstevel@tonic-gate } 12787c478bd9Sstevel@tonic-gate } 12797c478bd9Sstevel@tonic-gate 12807c478bd9Sstevel@tonic-gate /* 12817c478bd9Sstevel@tonic-gate * segkmem_alloc_lpi() imports virtual memory from large page heap arena 12827c478bd9Sstevel@tonic-gate * into kmem_lp arena. In the process it maps the imported segment with 12837c478bd9Sstevel@tonic-gate * large pages 12847c478bd9Sstevel@tonic-gate */ 12857c478bd9Sstevel@tonic-gate static void * 12867c478bd9Sstevel@tonic-gate segkmem_alloc_lpi(vmem_t *vmp, size_t size, int vmflag) 12877c478bd9Sstevel@tonic-gate { 12887c478bd9Sstevel@tonic-gate segkmem_lpcb_t *lpcb = &segkmem_lpcb; 12897c478bd9Sstevel@tonic-gate void *addr; 12907c478bd9Sstevel@tonic-gate 12917c478bd9Sstevel@tonic-gate ASSERT(size != 0); 12927c478bd9Sstevel@tonic-gate ASSERT(vmp == heap_lp_arena); 12937c478bd9Sstevel@tonic-gate 12947c478bd9Sstevel@tonic-gate /* do not allow large page heap grow beyound limits */ 12957c478bd9Sstevel@tonic-gate if (vmem_size(vmp, VMEM_ALLOC) >= segkmem_kmemlp_max) { 12967c478bd9Sstevel@tonic-gate atomic_add_64(&lpcb->allocs_limited, 1); 12977c478bd9Sstevel@tonic-gate return (NULL); 12987c478bd9Sstevel@tonic-gate } 12997c478bd9Sstevel@tonic-gate 13007c478bd9Sstevel@tonic-gate addr = segkmem_xalloc_lp(vmp, NULL, size, vmflag, 0, 13017c478bd9Sstevel@tonic-gate segkmem_page_create_large, NULL); 13027c478bd9Sstevel@tonic-gate return (addr); 13037c478bd9Sstevel@tonic-gate } 13047c478bd9Sstevel@tonic-gate 13057c478bd9Sstevel@tonic-gate /* 13067c478bd9Sstevel@tonic-gate * segkmem_free_lpi() returns virtual memory back into large page heap arena 13077c478bd9Sstevel@tonic-gate * from kmem_lp arena. Beore doing this it unmaps the segment and frees 13087c478bd9Sstevel@tonic-gate * large pages used to map it. 13097c478bd9Sstevel@tonic-gate */ 13107c478bd9Sstevel@tonic-gate static void 13117c478bd9Sstevel@tonic-gate segkmem_free_lpi(vmem_t *vmp, void *inaddr, size_t size) 13127c478bd9Sstevel@tonic-gate { 13137c478bd9Sstevel@tonic-gate pgcnt_t nlpages = size >> segkmem_lpshift; 13147c478bd9Sstevel@tonic-gate size_t lpsize = segkmem_lpsize; 13157c478bd9Sstevel@tonic-gate caddr_t addr = inaddr; 13167c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size); 13177c478bd9Sstevel@tonic-gate int i; 13187c478bd9Sstevel@tonic-gate 13197c478bd9Sstevel@tonic-gate ASSERT(vmp == heap_lp_arena); 13207c478bd9Sstevel@tonic-gate ASSERT(IS_KMEM_VA_LARGEPAGE(addr)); 13217c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)inaddr & (lpsize - 1)) == 0); 13227c478bd9Sstevel@tonic-gate 13237c478bd9Sstevel@tonic-gate for (i = 0; i < nlpages; i++) { 13247c478bd9Sstevel@tonic-gate segkmem_free_one_lp(addr, lpsize); 13257c478bd9Sstevel@tonic-gate addr += lpsize; 13267c478bd9Sstevel@tonic-gate } 13277c478bd9Sstevel@tonic-gate 13287c478bd9Sstevel@tonic-gate page_unresv(npages); 13297c478bd9Sstevel@tonic-gate 13307c478bd9Sstevel@tonic-gate vmem_free(vmp, inaddr, size); 13317c478bd9Sstevel@tonic-gate } 13327c478bd9Sstevel@tonic-gate 13337c478bd9Sstevel@tonic-gate /* 13347c478bd9Sstevel@tonic-gate * This function is called at system boot time by kmem_init right after 13357c478bd9Sstevel@tonic-gate * /etc/system file has been read. It checks based on hardware configuration 13367c478bd9Sstevel@tonic-gate * and /etc/system settings if system is going to use large pages. The 13377c478bd9Sstevel@tonic-gate * initialiazation necessary to actually start using large pages 13387c478bd9Sstevel@tonic-gate * happens later in the process after segkmem_heap_lp_init() is called. 13397c478bd9Sstevel@tonic-gate */ 13407c478bd9Sstevel@tonic-gate int 13417c478bd9Sstevel@tonic-gate segkmem_lpsetup() 13427c478bd9Sstevel@tonic-gate { 13437c478bd9Sstevel@tonic-gate int use_large_pages = 0; 13447c478bd9Sstevel@tonic-gate 13457c478bd9Sstevel@tonic-gate #ifdef __sparc 13467c478bd9Sstevel@tonic-gate 13477c478bd9Sstevel@tonic-gate size_t memtotal = physmem * PAGESIZE; 13487c478bd9Sstevel@tonic-gate 13497c478bd9Sstevel@tonic-gate if (heap_lp_base == NULL) { 13507c478bd9Sstevel@tonic-gate segkmem_lpsize = PAGESIZE; 13517c478bd9Sstevel@tonic-gate return (0); 13527c478bd9Sstevel@tonic-gate } 13537c478bd9Sstevel@tonic-gate 13547c478bd9Sstevel@tonic-gate /* get a platform dependent value of large page size for kernel heap */ 13557c478bd9Sstevel@tonic-gate segkmem_lpsize = get_segkmem_lpsize(segkmem_lpsize); 13567c478bd9Sstevel@tonic-gate 13577c478bd9Sstevel@tonic-gate if (segkmem_lpsize <= PAGESIZE) { 13587c478bd9Sstevel@tonic-gate /* 13597c478bd9Sstevel@tonic-gate * put virtual space reserved for the large page kernel 13607c478bd9Sstevel@tonic-gate * back to the regular heap 13617c478bd9Sstevel@tonic-gate */ 13627c478bd9Sstevel@tonic-gate vmem_xfree(heap_arena, heap_lp_base, 13637c478bd9Sstevel@tonic-gate heap_lp_end - heap_lp_base); 13647c478bd9Sstevel@tonic-gate heap_lp_base = NULL; 13657c478bd9Sstevel@tonic-gate heap_lp_end = NULL; 13667c478bd9Sstevel@tonic-gate segkmem_lpsize = PAGESIZE; 13677c478bd9Sstevel@tonic-gate return (0); 13687c478bd9Sstevel@tonic-gate } 13697c478bd9Sstevel@tonic-gate 13707c478bd9Sstevel@tonic-gate /* set heap_lp quantum if necessary */ 13717c478bd9Sstevel@tonic-gate if (segkmem_heaplp_quantum == 0 || 13727c478bd9Sstevel@tonic-gate (segkmem_heaplp_quantum & (segkmem_heaplp_quantum - 1)) || 13737c478bd9Sstevel@tonic-gate P2PHASE(segkmem_heaplp_quantum, segkmem_lpsize)) { 13747c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum = segkmem_lpsize; 13757c478bd9Sstevel@tonic-gate } 13767c478bd9Sstevel@tonic-gate 13777c478bd9Sstevel@tonic-gate /* set kmem_lp quantum if necessary */ 13787c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_quantum == 0 || 13797c478bd9Sstevel@tonic-gate (segkmem_kmemlp_quantum & (segkmem_kmemlp_quantum - 1)) || 13807c478bd9Sstevel@tonic-gate segkmem_kmemlp_quantum > segkmem_heaplp_quantum) { 13817c478bd9Sstevel@tonic-gate segkmem_kmemlp_quantum = segkmem_heaplp_quantum; 13827c478bd9Sstevel@tonic-gate } 13837c478bd9Sstevel@tonic-gate 13847c478bd9Sstevel@tonic-gate /* set total amount of memory allowed for large page kernel heap */ 13857c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_max == 0) { 13867c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_pcnt == 0 || segkmem_kmemlp_pcnt > 100) 13877c478bd9Sstevel@tonic-gate segkmem_kmemlp_pcnt = 25; 13887c478bd9Sstevel@tonic-gate segkmem_kmemlp_max = (memtotal * 100) / segkmem_kmemlp_pcnt; 13897c478bd9Sstevel@tonic-gate } 13907c478bd9Sstevel@tonic-gate segkmem_kmemlp_max = P2ROUNDUP(segkmem_kmemlp_max, 13917c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum); 13927c478bd9Sstevel@tonic-gate 13937c478bd9Sstevel@tonic-gate /* fix lp kmem preallocation request if necesssary */ 13947c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_min) { 13957c478bd9Sstevel@tonic-gate segkmem_kmemlp_min = P2ROUNDUP(segkmem_kmemlp_min, 13967c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum); 13977c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_min > segkmem_kmemlp_max) 13987c478bd9Sstevel@tonic-gate segkmem_kmemlp_min = segkmem_kmemlp_max; 13997c478bd9Sstevel@tonic-gate } 14007c478bd9Sstevel@tonic-gate 14017c478bd9Sstevel@tonic-gate use_large_pages = 1; 14027c478bd9Sstevel@tonic-gate segkmem_lpshift = page_get_shift(page_szc(segkmem_lpsize)); 14037c478bd9Sstevel@tonic-gate 14047c478bd9Sstevel@tonic-gate #endif 14057c478bd9Sstevel@tonic-gate return (use_large_pages); 14067c478bd9Sstevel@tonic-gate } 14077c478bd9Sstevel@tonic-gate 14087c478bd9Sstevel@tonic-gate #ifdef __sparc 14097c478bd9Sstevel@tonic-gate 14107c478bd9Sstevel@tonic-gate 14117c478bd9Sstevel@tonic-gate static void * 14127c478bd9Sstevel@tonic-gate segkmem_alloc_ppa(vmem_t *vmp, size_t size, int vmflag) 14137c478bd9Sstevel@tonic-gate { 14147c478bd9Sstevel@tonic-gate size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *); 14157c478bd9Sstevel@tonic-gate void *addr; 14167c478bd9Sstevel@tonic-gate 14177c478bd9Sstevel@tonic-gate if (ppaquantum <= PAGESIZE) 14187c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, vmflag)); 14197c478bd9Sstevel@tonic-gate 14207c478bd9Sstevel@tonic-gate ASSERT((size & (ppaquantum - 1)) == 0); 14217c478bd9Sstevel@tonic-gate 14227c478bd9Sstevel@tonic-gate addr = vmem_xalloc(vmp, size, ppaquantum, 0, 0, NULL, NULL, vmflag); 14237c478bd9Sstevel@tonic-gate if (addr != NULL && segkmem_xalloc(vmp, addr, size, vmflag, 0, 14247c478bd9Sstevel@tonic-gate segkmem_page_create, NULL) == NULL) { 14257c478bd9Sstevel@tonic-gate vmem_xfree(vmp, addr, size); 14267c478bd9Sstevel@tonic-gate addr = NULL; 14277c478bd9Sstevel@tonic-gate } 14287c478bd9Sstevel@tonic-gate 14297c478bd9Sstevel@tonic-gate return (addr); 14307c478bd9Sstevel@tonic-gate } 14317c478bd9Sstevel@tonic-gate 14327c478bd9Sstevel@tonic-gate static void 14337c478bd9Sstevel@tonic-gate segkmem_free_ppa(vmem_t *vmp, void *addr, size_t size) 14347c478bd9Sstevel@tonic-gate { 14357c478bd9Sstevel@tonic-gate size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *); 14367c478bd9Sstevel@tonic-gate 14377c478bd9Sstevel@tonic-gate ASSERT(addr != NULL); 14387c478bd9Sstevel@tonic-gate 14397c478bd9Sstevel@tonic-gate if (ppaquantum <= PAGESIZE) { 14407c478bd9Sstevel@tonic-gate segkmem_free(vmp, addr, size); 14417c478bd9Sstevel@tonic-gate } else { 14427c478bd9Sstevel@tonic-gate segkmem_free(NULL, addr, size); 14437c478bd9Sstevel@tonic-gate vmem_xfree(vmp, addr, size); 14447c478bd9Sstevel@tonic-gate } 14457c478bd9Sstevel@tonic-gate } 14467c478bd9Sstevel@tonic-gate 14477c478bd9Sstevel@tonic-gate void 14487c478bd9Sstevel@tonic-gate segkmem_heap_lp_init() 14497c478bd9Sstevel@tonic-gate { 14507c478bd9Sstevel@tonic-gate segkmem_lpcb_t *lpcb = &segkmem_lpcb; 14517c478bd9Sstevel@tonic-gate size_t heap_lp_size = heap_lp_end - heap_lp_base; 14527c478bd9Sstevel@tonic-gate size_t lpsize = segkmem_lpsize; 14537c478bd9Sstevel@tonic-gate size_t ppaquantum; 14547c478bd9Sstevel@tonic-gate void *addr; 14557c478bd9Sstevel@tonic-gate 14567c478bd9Sstevel@tonic-gate if (segkmem_lpsize <= PAGESIZE) { 14577c478bd9Sstevel@tonic-gate ASSERT(heap_lp_base == NULL); 14587c478bd9Sstevel@tonic-gate ASSERT(heap_lp_end == NULL); 14597c478bd9Sstevel@tonic-gate return; 14607c478bd9Sstevel@tonic-gate } 14617c478bd9Sstevel@tonic-gate 14627c478bd9Sstevel@tonic-gate ASSERT(segkmem_heaplp_quantum >= lpsize); 14637c478bd9Sstevel@tonic-gate ASSERT((segkmem_heaplp_quantum & (lpsize - 1)) == 0); 14647c478bd9Sstevel@tonic-gate ASSERT(lpcb->lp_uselp == 0); 14657c478bd9Sstevel@tonic-gate ASSERT(heap_lp_base != NULL); 14667c478bd9Sstevel@tonic-gate ASSERT(heap_lp_end != NULL); 14677c478bd9Sstevel@tonic-gate ASSERT(heap_lp_base < heap_lp_end); 14687c478bd9Sstevel@tonic-gate ASSERT(heap_lp_arena == NULL); 14697c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)heap_lp_base & (lpsize - 1)) == 0); 14707c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)heap_lp_end & (lpsize - 1)) == 0); 14717c478bd9Sstevel@tonic-gate 14727c478bd9Sstevel@tonic-gate /* create large page heap arena */ 14737c478bd9Sstevel@tonic-gate heap_lp_arena = vmem_create("heap_lp", heap_lp_base, heap_lp_size, 14747c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum, NULL, NULL, NULL, 0, VM_SLEEP); 14757c478bd9Sstevel@tonic-gate 14767c478bd9Sstevel@tonic-gate ASSERT(heap_lp_arena != NULL); 14777c478bd9Sstevel@tonic-gate 14787c478bd9Sstevel@tonic-gate /* This arena caches memory already mapped by large pages */ 14797c478bd9Sstevel@tonic-gate kmem_lp_arena = vmem_create("kmem_lp", NULL, 0, segkmem_kmemlp_quantum, 14807c478bd9Sstevel@tonic-gate segkmem_alloc_lpi, segkmem_free_lpi, heap_lp_arena, 0, VM_SLEEP); 14817c478bd9Sstevel@tonic-gate 14827c478bd9Sstevel@tonic-gate ASSERT(kmem_lp_arena != NULL); 14837c478bd9Sstevel@tonic-gate 14847c478bd9Sstevel@tonic-gate mutex_init(&lpcb->lp_lock, NULL, MUTEX_DEFAULT, NULL); 14857c478bd9Sstevel@tonic-gate cv_init(&lpcb->lp_cv, NULL, CV_DEFAULT, NULL); 14867c478bd9Sstevel@tonic-gate 14877c478bd9Sstevel@tonic-gate /* 14887c478bd9Sstevel@tonic-gate * this arena is used for the array of page_t pointers necessary 14897c478bd9Sstevel@tonic-gate * to call hat_mem_load_array 14907c478bd9Sstevel@tonic-gate */ 14917c478bd9Sstevel@tonic-gate ppaquantum = btopr(lpsize) * sizeof (page_t *); 14927c478bd9Sstevel@tonic-gate segkmem_ppa_arena = vmem_create("segkmem_ppa", NULL, 0, ppaquantum, 14937c478bd9Sstevel@tonic-gate segkmem_alloc_ppa, segkmem_free_ppa, heap_arena, ppaquantum, 14947c478bd9Sstevel@tonic-gate VM_SLEEP); 14957c478bd9Sstevel@tonic-gate 14967c478bd9Sstevel@tonic-gate ASSERT(segkmem_ppa_arena != NULL); 14977c478bd9Sstevel@tonic-gate 14987c478bd9Sstevel@tonic-gate /* prealloacate some memory for the lp kernel heap */ 14997c478bd9Sstevel@tonic-gate if (segkmem_kmemlp_min) { 15007c478bd9Sstevel@tonic-gate 15017c478bd9Sstevel@tonic-gate ASSERT(P2PHASE(segkmem_kmemlp_min, 15027c478bd9Sstevel@tonic-gate segkmem_heaplp_quantum) == 0); 15037c478bd9Sstevel@tonic-gate 15047c478bd9Sstevel@tonic-gate if ((addr = segkmem_alloc_lpi(heap_lp_arena, 15057c478bd9Sstevel@tonic-gate segkmem_kmemlp_min, VM_SLEEP)) != NULL) { 15067c478bd9Sstevel@tonic-gate 15077c478bd9Sstevel@tonic-gate addr = vmem_add(kmem_lp_arena, addr, 15087c478bd9Sstevel@tonic-gate segkmem_kmemlp_min, VM_SLEEP); 15097c478bd9Sstevel@tonic-gate ASSERT(addr != NULL); 15107c478bd9Sstevel@tonic-gate } 15117c478bd9Sstevel@tonic-gate } 15127c478bd9Sstevel@tonic-gate 15137c478bd9Sstevel@tonic-gate lpcb->lp_uselp = 1; 15147c478bd9Sstevel@tonic-gate } 15157c478bd9Sstevel@tonic-gate 15167c478bd9Sstevel@tonic-gate #endif 1517