1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/machsystm.h> 30 #include <sys/archsystm.h> 31 #include <sys/vm.h> 32 #include <sys/cpu.h> 33 #include <sys/atomic.h> 34 #include <sys/reboot.h> 35 #include <sys/kdi.h> 36 #include <sys/bootconf.h> 37 #include <sys/memlist_plat.h> 38 #include <sys/memlist_impl.h> 39 #include <sys/prom_plat.h> 40 #include <sys/prom_isa.h> 41 #include <sys/autoconf.h> 42 #include <sys/intreg.h> 43 #include <sys/ivintr.h> 44 #include <sys/fpu/fpusystm.h> 45 #include <sys/iommutsb.h> 46 #include <vm/vm_dep.h> 47 #include <vm/seg_dev.h> 48 #include <vm/seg_kmem.h> 49 #include <vm/seg_kpm.h> 50 #include <vm/seg_map.h> 51 #include <vm/seg_kp.h> 52 #include <sys/sysconf.h> 53 #include <vm/hat_sfmmu.h> 54 #include <sys/kobj.h> 55 #include <sys/sun4asi.h> 56 #include <sys/clconf.h> 57 #include <sys/platform_module.h> 58 #include <sys/panic.h> 59 #include <sys/cpu_sgnblk_defs.h> 60 #include <sys/clock.h> 61 #include <sys/cmn_err.h> 62 #include <sys/promif.h> 63 #include <sys/prom_debug.h> 64 #include <sys/traptrace.h> 65 #include <sys/memnode.h> 66 #include <sys/mem_cage.h> 67 68 extern void setup_trap_table(void); 69 extern void cpu_intrq_setup(struct cpu *); 70 extern void cpu_intrq_register(struct cpu *); 71 extern void contig_mem_init(void); 72 extern void mach_dump_buffer_init(void); 73 extern void mach_descrip_init(void); 74 extern void mach_memscrub(void); 75 extern void mach_fpras(void); 76 extern void mach_cpu_halt_idle(void); 77 extern void mach_hw_copy_limit(void); 78 extern void load_tod_module(void); 79 #pragma weak load_tod_module 80 81 extern int ndata_alloc_mmfsa(struct memlist *ndata); 82 #pragma weak ndata_alloc_mmfsa 83 84 extern void parse_idprom(void); 85 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 86 extern void mem_config_init(void); 87 extern void memseg_remap_init(void); 88 89 /* 90 * External Data: 91 */ 92 extern int vac_size; /* cache size in bytes */ 93 extern uint_t vac_mask; /* VAC alignment consistency mask */ 94 extern uint_t vac_colors; 95 96 /* 97 * Global Data Definitions: 98 */ 99 100 /* 101 * XXX - Don't port this to new architectures 102 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 103 * 'romp' has no use with a prom with an IEEE 1275 client interface. 104 * The driver doesn't use the value, but it depends on the symbol. 105 */ 106 void *romp; /* veritas driver won't load without romp 4154976 */ 107 /* 108 * Declare these as initialized data so we can patch them. 109 */ 110 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 111 pgcnt_t segkpsize = 112 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 113 uint_t segmap_percent = 12; /* Size of segmap segment */ 114 115 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 116 int vac_copyback = 1; 117 char *cache_mode = NULL; 118 int use_mix = 1; 119 int prom_debug = 0; 120 int usb_node_debug = 0; 121 122 struct bootops *bootops = 0; /* passed in from boot in %o2 */ 123 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 124 uint_t tba_taken_over = 0; 125 126 caddr_t s_text; /* start of kernel text segment */ 127 caddr_t e_text; /* end of kernel text segment */ 128 caddr_t s_data; /* start of kernel data segment */ 129 caddr_t e_data; /* end of kernel data segment */ 130 131 caddr_t modtext; /* beginning of module text */ 132 size_t modtext_sz; /* size of module text */ 133 caddr_t moddata; /* beginning of module data reserve */ 134 caddr_t e_moddata; /* end of module data reserve */ 135 136 /* 137 * End of first block of contiguous kernel in 32-bit virtual address space 138 */ 139 caddr_t econtig32; /* end of first blk of contiguous kernel */ 140 141 caddr_t ncbase; /* beginning of non-cached segment */ 142 caddr_t ncend; /* end of non-cached segment */ 143 caddr_t sdata; /* beginning of data segment */ 144 145 caddr_t extra_etva; /* beginning of unused nucleus text */ 146 pgcnt_t extra_etpg; /* number of pages of unused nucleus text */ 147 148 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 149 caddr_t nalloc_base; /* beginning of nucleus allocation */ 150 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 151 caddr_t valloc_base; /* beginning of kvalloc segment */ 152 153 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 154 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 155 156 uintptr_t shm_alignment = 0; /* VAC address consistency modulus */ 157 struct memlist *phys_install; /* Total installed physical memory */ 158 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 159 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 160 struct memlist ndata; /* memlist of nucleus allocatable memory */ 161 int memexp_flag; /* memory expansion card flag */ 162 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 163 pgcnt_t obp_pages; /* Physical pages used by OBP */ 164 165 /* 166 * VM data structures 167 */ 168 long page_hashsz; /* Size of page hash table (power of two) */ 169 struct page *pp_base; /* Base of system page struct array */ 170 size_t pp_sz; /* Size in bytes of page struct array */ 171 struct page **page_hash; /* Page hash table */ 172 struct seg ktextseg; /* Segment used for kernel executable image */ 173 struct seg kvalloc; /* Segment used for "valloc" mapping */ 174 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 175 struct seg ktexthole; /* Segment used for nucleus text hole */ 176 struct seg kmapseg; /* Segment used for generic kernel mappings */ 177 struct seg kpmseg; /* Segment used for physical mapping */ 178 struct seg kdebugseg; /* Segment used for the kernel debugger */ 179 180 uintptr_t kpm_pp_base; /* Base of system kpm_page array */ 181 size_t kpm_pp_sz; /* Size of system kpm_page array */ 182 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 183 184 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 185 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 186 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 187 188 /* 189 * debugger pages (if allocated) 190 */ 191 struct vnode kdebugvp; 192 193 /* 194 * Segment for relocated kernel structures in 64-bit large RAM kernels 195 */ 196 struct seg kmem64; 197 198 struct memseg *memseg_base; 199 size_t memseg_sz; /* Used to translate a va to page */ 200 struct vnode unused_pages_vp; 201 202 /* 203 * VM data structures allocated early during boot. 204 */ 205 size_t pagehash_sz; 206 uint64_t memlist_sz; 207 208 char tbr_wr_addr_inited = 0; 209 210 211 /* 212 * Static Routines: 213 */ 214 static void memlist_add(uint64_t, uint64_t, struct memlist **, 215 struct memlist **); 216 static void kphysm_init(page_t *, struct memseg *, pgcnt_t, uintptr_t, 217 pgcnt_t); 218 static void kvm_init(void); 219 220 static void startup_init(void); 221 static void startup_memlist(void); 222 static void startup_modules(void); 223 static void startup_bop_gone(void); 224 static void startup_vm(void); 225 static void startup_end(void); 226 static void setup_cage_params(void); 227 static void startup_create_input_node(void); 228 229 static pgcnt_t npages; 230 static struct memlist *memlist; 231 void *memlist_end; 232 233 static pgcnt_t bop_alloc_pages; 234 static caddr_t hblk_base; 235 uint_t hblk_alloc_dynamic = 0; 236 uint_t hblk1_min = H1MIN; 237 uint_t hblk8_min; 238 239 240 /* 241 * Hooks for unsupported platforms and down-rev firmware 242 */ 243 int iam_positron(void); 244 #pragma weak iam_positron 245 static void do_prom_version_check(void); 246 static void kpm_init(void); 247 static void kpm_npages_setup(int); 248 static void kpm_memseg_init(void); 249 250 /* 251 * After receiving a thermal interrupt, this is the number of seconds 252 * to delay before shutting off the system, assuming 253 * shutdown fails. Use /etc/system to change the delay if this isn't 254 * large enough. 255 */ 256 int thermal_powerdown_delay = 1200; 257 258 /* 259 * Used to hold off page relocations into the cage until OBP has completed 260 * its boot-time handoff of its resources to the kernel. 261 */ 262 int page_relocate_ready = 0; 263 264 /* 265 * Enable some debugging messages concerning memory usage... 266 */ 267 #ifdef DEBUGGING_MEM 268 static int debugging_mem; 269 static void 270 printmemlist(char *title, struct memlist *list) 271 { 272 if (!debugging_mem) 273 return; 274 275 printf("%s\n", title); 276 277 while (list) { 278 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 279 (uint32_t)(list->address >> 32), (uint32_t)list->address, 280 (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 281 list = list->next; 282 } 283 } 284 285 void 286 printmemseg(struct memseg *memseg) 287 { 288 if (!debugging_mem) 289 return; 290 291 printf("memseg\n"); 292 293 while (memseg) { 294 prom_printf("\tpage = 0x%p, epage = 0x%p, " 295 "pfn = 0x%x, epfn = 0x%x\n", 296 memseg->pages, memseg->epages, 297 memseg->pages_base, memseg->pages_end); 298 memseg = memseg->next; 299 } 300 } 301 302 #define debug_pause(str) halt((str)) 303 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 304 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 305 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 306 #define MPRINTF3(str, a, b, c) \ 307 if (debugging_mem) prom_printf((str), (a), (b), (c)) 308 #else /* DEBUGGING_MEM */ 309 #define MPRINTF(str) 310 #define MPRINTF1(str, a) 311 #define MPRINTF2(str, a, b) 312 #define MPRINTF3(str, a, b, c) 313 #endif /* DEBUGGING_MEM */ 314 315 /* Simple message to indicate that the bootops pointer has been zeroed */ 316 #ifdef DEBUG 317 static int bootops_gone_on = 0; 318 #define BOOTOPS_GONE() \ 319 if (bootops_gone_on) \ 320 prom_printf("The bootops vec is zeroed now!\n"); 321 #else 322 #define BOOTOPS_GONE() 323 #endif /* DEBUG */ 324 325 /* 326 * Monitor pages may not be where this says they are. 327 * and the debugger may not be there either. 328 * 329 * Note that 'pages' here are *physical* pages, which are 8k on sun4u. 330 * 331 * Physical memory layout 332 * (not necessarily contiguous) 333 * (THIS IS SOMEWHAT WRONG) 334 * /-----------------------\ 335 * | monitor pages | 336 * availmem -|-----------------------| 337 * | | 338 * | page pool | 339 * | | 340 * |-----------------------| 341 * | configured tables | 342 * | buffers | 343 * firstaddr -|-----------------------| 344 * | hat data structures | 345 * |-----------------------| 346 * | kernel data, bss | 347 * |-----------------------| 348 * | interrupt stack | 349 * |-----------------------| 350 * | kernel text (RO) | 351 * |-----------------------| 352 * | trap table (4k) | 353 * |-----------------------| 354 * page 1 | panicbuf | 355 * |-----------------------| 356 * page 0 | reclaimed | 357 * |_______________________| 358 * 359 * 360 * 361 * Kernel's Virtual Memory Layout. 362 * /-----------------------\ 363 * 0xFFFFFFFF.FFFFFFFF -| |- 364 * | OBP's virtual page | 365 * | tables | 366 * 0xFFFFFFFC.00000000 -|-----------------------|- 367 * : : 368 * : : 369 * 0xFFFFFE00.00000000 -|-----------------------|- 370 * | | Ultrasparc I/II support 371 * | segkpm segment | up to 2TB of physical 372 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 373 * | | 374 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 375 * : : 376 * : : 377 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 378 * | | ^ 379 * | UltraSPARC I/II call | | 380 * | bug requires an extra | | 381 * | 4 GB of space between | | 382 * | hole and used RAM | | 383 * | | | 384 * 0xFFFFF800.00000000 -|-----------------------|- | 385 * | | | 386 * | Virtual Address Hole | UltraSPARC 387 * | on UltraSPARC I/II | I/II * ONLY * 388 * | | | 389 * 0x00000800.00000000 -|-----------------------|- | 390 * | | | 391 * | UltraSPARC I/II call | | 392 * | bug requires an extra | | 393 * | 4 GB of space between | | 394 * | hole and used RAM | | 395 * | | v 396 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 397 * : : ^ 398 * : : | 399 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 400 * | | | 401 * | 64-bit kernel ONLY | | 402 * | | | 403 * | kmem64 segment | | 404 * | | | 405 * | (Relocated extra HME | Approximately 406 * | block allocations, | 1 TB of virtual 407 * | memnode freelists, | address space 408 * | HME hash buckets, | | 409 * | mml_table, kpmp_table,| | 410 * | page_t array and | | 411 * | hashblock pool to | | 412 * | avoid hard-coded | | 413 * | 32-bit vaddr | | 414 * | limitations) | | 415 * | | v 416 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 417 * | | 418 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 419 * | | 420 * 0x00000300.00000000 -|-----------------------|- SYSBASE 421 * : : 422 * : : 423 * -|-----------------------|- 424 * | | 425 * | segmap segment | SEGMAPSIZE (1/8th physmem, 426 * | | 256G MAX) 427 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 428 * : : 429 * : : 430 * -|-----------------------|- 431 * | | 432 * | segkp | SEGKPSIZE (2GB) 433 * | | 434 * | | 435 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 436 * | | 437 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 438 * | | (SEGKPBASE - 0x400000) 439 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 440 * | | (MEMSCRUBBASE - NCARGS) 441 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 442 * | | (ARGSBASE - PPMAPSIZE) 443 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 444 * | | 445 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 446 * | | 447 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 448 * : : 449 * : : 450 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 451 * | | 452 * | OBP | 453 * | | 454 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 455 * | kmdb | 456 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 457 * : : 458 * : : 459 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 460 * | | 461 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 462 * | | ~64MB) 463 * 0x00000000.78002000 -|-----------------------| 464 * | panicbuf | 465 * 0x00000000.78000000 -|-----------------------|- SYSBASE32 466 * : : 467 * : : 468 * | | 469 * |-----------------------|- econtig32 470 * | vm structures | 471 * 0x00000000.01C00000 |-----------------------|- nalloc_end 472 * | TSBs | 473 * |-----------------------|- end/nalloc_base 474 * | kernel data & bss | 475 * 0x00000000.01800000 -|-----------------------| 476 * : nucleus text hole : 477 * 0x00000000.01400000 -|-----------------------| 478 * : : 479 * |-----------------------| 480 * | module text | 481 * |-----------------------|- e_text/modtext 482 * | kernel text | 483 * |-----------------------| 484 * | trap table (48k) | 485 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 486 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 487 * |-----------------------| 488 * | | 489 * | invalid | 490 * | | 491 * 0x00000000.00000000 _|_______________________| 492 * 493 * 494 * 495 * 32-bit User Virtual Memory Layout. 496 * /-----------------------\ 497 * | | 498 * | invalid | 499 * | | 500 * 0xFFC00000 -|-----------------------|- USERLIMIT 501 * | user stack | 502 * : : 503 * : : 504 * : : 505 * | user data | 506 * -|-----------------------|- 507 * | user text | 508 * 0x00002000 -|-----------------------|- 509 * | invalid | 510 * 0x00000000 _|_______________________| 511 * 512 * 513 * 514 * 64-bit User Virtual Memory Layout. 515 * /-----------------------\ 516 * | | 517 * | invalid | 518 * | | 519 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 520 * | user stack | 521 * : : 522 * : : 523 * : : 524 * | user data | 525 * -|-----------------------|- 526 * | user text | 527 * 0x00000000.00100000 -|-----------------------|- 528 * | invalid | 529 * 0x00000000.00000000 _|_______________________| 530 */ 531 532 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 533 extern uint64_t ecache_flush_address(void); 534 535 #pragma weak load_platform_modules 536 #pragma weak starcat_startup_memlist 537 #pragma weak ecache_init_scrub_flush_area 538 #pragma weak ecache_flush_address 539 540 541 /* 542 * By default the DR Cage is enabled for maximum OS 543 * MPSS performance. Users needing to disable the cage mechanism 544 * can set this variable to zero via /etc/system. 545 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 546 * will result in loss of DR functionality. 547 * Platforms wishing to disable kernel Cage by default 548 * should do so in their set_platform_defaults() routine. 549 */ 550 int kernel_cage_enable = 1; 551 552 static void 553 setup_cage_params(void) 554 { 555 void (*func)(void); 556 557 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 558 if (func != NULL) { 559 (*func)(); 560 return; 561 } 562 563 if (kernel_cage_enable == 0) { 564 return; 565 } 566 kcage_range_lock(); 567 if (kcage_range_init(phys_avail, 1) == 0) { 568 kcage_init(total_pages / 256); 569 } 570 kcage_range_unlock(); 571 572 if (kcage_on) { 573 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 574 } else { 575 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 576 } 577 578 } 579 580 /* 581 * Machine-dependent startup code 582 */ 583 void 584 startup(void) 585 { 586 startup_init(); 587 if (&startup_platform) 588 startup_platform(); 589 startup_memlist(); 590 startup_modules(); 591 setup_cage_params(); 592 startup_bop_gone(); 593 startup_vm(); 594 startup_end(); 595 } 596 597 struct regs sync_reg_buf; 598 uint64_t sync_tt; 599 600 void 601 sync_handler(void) 602 { 603 struct trap_info ti; 604 int i; 605 606 /* 607 * Prevent trying to talk to the other CPUs since they are 608 * sitting in the prom and won't reply. 609 */ 610 for (i = 0; i < NCPU; i++) { 611 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 612 cpu[i]->cpu_flags &= ~CPU_READY; 613 cpu[i]->cpu_flags |= CPU_QUIESCED; 614 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 615 } 616 } 617 618 /* 619 * We've managed to get here without going through the 620 * normal panic code path. Try and save some useful 621 * information. 622 */ 623 if (!panicstr && (curthread->t_panic_trap == NULL)) { 624 ti.trap_type = sync_tt; 625 ti.trap_regs = &sync_reg_buf; 626 ti.trap_addr = NULL; 627 ti.trap_mmu_fsr = 0x0; 628 629 curthread->t_panic_trap = &ti; 630 } 631 632 /* 633 * If we're re-entering the panic path, update the signature 634 * block so that the SC knows we're in the second part of panic. 635 */ 636 if (panicstr) 637 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 638 639 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 640 panic("sync initiated"); 641 } 642 643 644 static void 645 startup_init(void) 646 { 647 /* 648 * We want to save the registers while we're still in OBP 649 * so that we know they haven't been fiddled with since. 650 * (In principle, OBP can't change them just because it 651 * makes a callback, but we'd rather not depend on that 652 * behavior.) 653 */ 654 char sync_str[] = 655 "warning @ warning off : sync " 656 "%%tl-c %%tstate h# %p x! " 657 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 658 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 659 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 660 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 661 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 662 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 663 "%%y h# %p l! %%tl-c %%tt h# %p x! " 664 "sync ; warning !"; 665 666 /* 667 * 20 == num of %p substrings 668 * 16 == max num of chars %p will expand to. 669 */ 670 char bp[sizeof (sync_str) + 16 * 20]; 671 672 (void) check_boot_version(BOP_GETVERSION(bootops)); 673 674 /* 675 * Initialize ptl1 stack for the 1st CPU. 676 */ 677 ptl1_init_cpu(&cpu0); 678 679 /* 680 * Initialize the address map for cache consistent mappings 681 * to random pages; must be done after vac_size is set. 682 */ 683 ppmapinit(); 684 685 /* 686 * Initialize the PROM callback handler. 687 */ 688 init_vx_handler(); 689 690 /* 691 * have prom call sync_callback() to handle the sync and 692 * save some useful information which will be stored in the 693 * core file later. 694 */ 695 (void) sprintf((char *)bp, sync_str, 696 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 697 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 698 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 699 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 700 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 701 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 702 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 703 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 704 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 705 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 706 prom_interpret(bp, 0, 0, 0, 0, 0); 707 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 708 } 709 710 static u_longlong_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 711 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 712 713 #define IVSIZE ((MAXIVNUM + 1) * sizeof (struct intr_vector)) 714 715 /* 716 * As OBP takes up some RAM when the system boots, pages will already be "lost" 717 * to the system and reflected in npages by the time we see it. 718 * 719 * We only want to allocate kernel structures in the 64-bit virtual address 720 * space on systems with enough RAM to make the overhead of keeping track of 721 * an extra kernel memory segment worthwhile. 722 * 723 * Since OBP has already performed its memory allocations by this point, if we 724 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 725 * memory in the 64-bit virtual address space; otherwise keep allocations 726 * contiguous with we've mapped so far in the 32-bit virtual address space. 727 */ 728 #define MINMOVE_RAM_MB ((size_t)1900) 729 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 730 731 pgcnt_t tune_npages = (pgcnt_t) 732 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 733 734 static void 735 startup_memlist(void) 736 { 737 size_t alloc_sz; 738 size_t ctrs_sz; 739 caddr_t alloc_base; 740 caddr_t ctrs_base, ctrs_end; 741 caddr_t memspace; 742 caddr_t va; 743 int memblocks = 0; 744 struct memlist *cur; 745 size_t syslimit = (size_t)SYSLIMIT; 746 size_t sysbase = (size_t)SYSBASE; 747 int alloc_alignsize = MMU_PAGESIZE; 748 extern void page_coloring_init(void); 749 750 /* 751 * Initialize enough of the system to allow kmem_alloc to work by 752 * calling boot to allocate its memory until the time that 753 * kvm_init is completed. The page structs are allocated after 754 * rounding up end to the nearest page boundary; the memsegs are 755 * initialized and the space they use comes from the kernel heap. 756 * With appropriate initialization, they can be reallocated later 757 * to a size appropriate for the machine's configuration. 758 * 759 * At this point, memory is allocated for things that will never 760 * need to be freed, this used to be "valloced". This allows a 761 * savings as the pages don't need page structures to describe 762 * them because them will not be managed by the vm system. 763 */ 764 765 /* 766 * We're loaded by boot with the following configuration (as 767 * specified in the sun4u/conf/Mapfile): 768 * 769 * text: 4 MB chunk aligned on a 4MB boundary 770 * data & bss: 4 MB chunk aligned on a 4MB boundary 771 * 772 * These two chunks will eventually be mapped by 2 locked 4MB 773 * ttes and will represent the nucleus of the kernel. This gives 774 * us some free space that is already allocated, some or all of 775 * which is made available to kernel module text. 776 * 777 * The free space in the data-bss chunk is used for nucleus 778 * allocatable data structures and we reserve it using the 779 * nalloc_base and nalloc_end variables. This space is currently 780 * being used for hat data structures required for tlb miss 781 * handling operations. We align nalloc_base to a l2 cache 782 * linesize because this is the line size the hardware uses to 783 * maintain cache coherency. 784 * 256K is carved out for module data. 785 */ 786 787 nalloc_base = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 788 moddata = nalloc_base; 789 e_moddata = nalloc_base + MODDATA; 790 nalloc_base = e_moddata; 791 792 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 793 valloc_base = nalloc_base; 794 795 /* 796 * Calculate the start of the data segment. 797 */ 798 sdata = (caddr_t)((uintptr_t)e_data & MMU_PAGEMASK4M); 799 800 PRM_DEBUG(moddata); 801 PRM_DEBUG(nalloc_base); 802 PRM_DEBUG(nalloc_end); 803 PRM_DEBUG(sdata); 804 805 /* 806 * Remember any slop after e_text so we can give it to the modules. 807 */ 808 PRM_DEBUG(e_text); 809 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 810 if (((uintptr_t)modtext & MMU_PAGEMASK4M) != (uintptr_t)s_text) 811 panic("nucleus text overflow"); 812 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 813 modtext; 814 PRM_DEBUG(modtext); 815 PRM_DEBUG(modtext_sz); 816 817 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 818 &boot_physavail, &boot_physavail_len, 819 &boot_virtavail, &boot_virtavail_len); 820 /* 821 * Remember what the physically available highest page is 822 * so that dumpsys works properly, and find out how much 823 * memory is installed. 824 */ 825 installed_top_size_memlist_array(boot_physinstalled, 826 boot_physinstalled_len, &physmax, &physinstalled); 827 PRM_DEBUG(physinstalled); 828 PRM_DEBUG(physmax); 829 830 /* Fill out memory nodes config structure */ 831 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 832 833 /* 834 * Get the list of physically available memory to size 835 * the number of page structures needed. 836 */ 837 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 838 /* 839 * This first snap shot of npages can represent the pages used 840 * by OBP's text and data approximately. This is used in the 841 * the calculation of the kernel size 842 */ 843 obp_pages = physinstalled - npages; 844 845 846 /* 847 * On small-memory systems (<MODTEXT_SM_SIZE MB, currently 256MB), the 848 * in-nucleus module text is capped to MODTEXT_SM_CAP bytes (currently 849 * 2MB) and any excess pages are put on physavail. The assumption is 850 * that small-memory systems will need more pages more than they'll 851 * need efficiently-mapped module texts. 852 */ 853 if ((physinstalled < mmu_btop(MODTEXT_SM_SIZE << 20)) && 854 modtext_sz > MODTEXT_SM_CAP) { 855 extra_etpg = mmu_btop(modtext_sz - MODTEXT_SM_CAP); 856 modtext_sz = MODTEXT_SM_CAP; 857 } else 858 extra_etpg = 0; 859 PRM_DEBUG(extra_etpg); 860 PRM_DEBUG(modtext_sz); 861 extra_etva = modtext + modtext_sz; 862 PRM_DEBUG(extra_etva); 863 864 /* 865 * Account for any pages after e_text and e_data. 866 */ 867 npages += extra_etpg; 868 npages += mmu_btopr(nalloc_end - nalloc_base); 869 PRM_DEBUG(npages); 870 871 /* 872 * npages is the maximum of available physical memory possible. 873 * (ie. it will never be more than this) 874 */ 875 876 /* 877 * initialize the nucleus memory allocator. 878 */ 879 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 880 881 /* 882 * Allocate mmu fault status area from the nucleus data area. 883 */ 884 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 885 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 886 887 /* 888 * Allocate kernel TSBs from the nucleus data area. 889 */ 890 if (ndata_alloc_tsbs(&ndata, npages) != 0) 891 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 892 893 /* 894 * Allocate cpus structs from the nucleus data area. 895 */ 896 if (ndata_alloc_cpus(&ndata) != 0) 897 cmn_err(CE_PANIC, "no more nucleus memory after cpu alloc"); 898 899 /* 900 * Allocate dmv dispatch table from the nucleus data area. 901 */ 902 if (ndata_alloc_dmv(&ndata) != 0) 903 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 904 905 906 page_coloring_init(); 907 908 /* 909 * Allocate page_freelists bin headers for memnode 0 from the 910 * nucleus data area. 911 */ 912 if (ndata_alloc_page_freelists(&ndata, 0) != 0) 913 cmn_err(CE_PANIC, 914 "no more nucleus memory after page free lists alloc"); 915 916 if (kpm_enable) { 917 kpm_init(); 918 /* 919 * kpm page space -- Update kpm_npages and make the 920 * same assumption about fragmenting as it is done 921 * for memseg_sz. 922 */ 923 kpm_npages_setup(memblocks + 4); 924 } 925 926 /* 927 * Allocate hat related structs from the nucleus data area. 928 */ 929 if (ndata_alloc_hat(&ndata, npages, kpm_npages) != 0) 930 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 931 932 /* 933 * We want to do the BOP_ALLOCs before the real allocation of page 934 * structs in order to not have to allocate page structs for this 935 * memory. We need to calculate a virtual address because we want 936 * the page structs to come before other allocations in virtual address 937 * space. This is so some (if not all) of page structs can actually 938 * live in the nucleus. 939 */ 940 941 /* 942 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 943 * 944 * There are comments all over the SFMMU code warning of dire 945 * consequences if the TSBs are moved out of 32-bit space. This 946 * is largely because the asm code uses "sethi %hi(addr)"-type 947 * instructions which will not provide the expected result if the 948 * address is a 64-bit one. 949 * 950 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 951 */ 952 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 953 alloc_base = sfmmu_ktsb_alloc(alloc_base); 954 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 955 PRM_DEBUG(alloc_base); 956 957 /* 958 * Allocate IOMMU TSB array. We do this here so that the physical 959 * memory gets deducted from the PROM's physical memory list. 960 */ 961 alloc_base = iommu_tsb_init(alloc_base); 962 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 963 ecache_alignsize); 964 PRM_DEBUG(alloc_base); 965 966 /* 967 * Starcat needs its special structures assigned in 32-bit virtual 968 * address space because its probing routines execute FCode, and FCode 969 * can't handle 64-bit virtual addresses... 970 */ 971 if (&starcat_startup_memlist) { 972 alloc_base = starcat_startup_memlist(alloc_base); 973 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 974 ecache_alignsize); 975 PRM_DEBUG(alloc_base); 976 } 977 978 /* 979 * If we have enough memory, use 4M pages for alignment because it 980 * greatly reduces the number of TLB misses we take albeit at the cost 981 * of possible RAM wastage (degenerate case of 4 MB - MMU_PAGESIZE per 982 * allocation.) Still, the speedup on large memory systems (e.g. > 64 983 * GB) is quite noticeable, so it is worth the effort to do if we can. 984 * 985 * Note, however, that this speedup will only occur if the boot PROM 986 * uses the largest possible MMU page size possible to map memory 987 * requests that are properly aligned and sized (for example, a request 988 * for a multiple of 4MB of memory aligned to a 4MB boundary will 989 * result in a mapping using a 4MB MMU page.) 990 * 991 * Even then, the large page mappings will only speed things up until 992 * the startup process proceeds a bit further, as when 993 * sfmmu_map_prom_mappings() copies page mappings from the PROM to the 994 * kernel it remaps everything but the TSBs using 8K pages anyway... 995 * 996 * At some point in the future, sfmmu_map_prom_mappings() will be 997 * rewritten to copy memory mappings to the kernel using the same MMU 998 * page sizes the PROM used. When that occurs, if the PROM did use 999 * large MMU pages to map memory, the alignment/sizing work we're 1000 * doing now should give us a nice extra performance boost, albeit at 1001 * the cost of greater RAM usage... 1002 */ 1003 alloc_alignsize = ((npages >= tune_npages) ? MMU_PAGESIZE4M : 1004 MMU_PAGESIZE); 1005 1006 PRM_DEBUG(tune_npages); 1007 PRM_DEBUG(alloc_alignsize); 1008 1009 /* 1010 * Save off where the contiguous allocations to date have ended 1011 * in econtig32. 1012 */ 1013 econtig32 = alloc_base; 1014 PRM_DEBUG(econtig32); 1015 1016 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1017 cmn_err(CE_PANIC, "econtig32 too big"); 1018 1019 /* 1020 * To avoid memory allocation collisions in the 32-bit virtual address 1021 * space, make allocations from this point forward in 64-bit virtual 1022 * address space starting at syslimit and working up. Also use the 1023 * alignment specified by alloc_alignsize, as we may be able to save 1024 * ourselves TLB misses by using larger page sizes if they're 1025 * available. 1026 * 1027 * All this is needed because on large memory systems, the default 1028 * Solaris allocations will collide with SYSBASE32, which is hard 1029 * coded to be at the virtual address 0x78000000. Therefore, on 64-bit 1030 * kernels, move the allocations to a location in the 64-bit virtual 1031 * address space space, allowing those structures to grow without 1032 * worry. 1033 * 1034 * On current CPUs we'll run out of physical memory address bits before 1035 * we need to worry about the allocations running into anything else in 1036 * VM or the virtual address holes on US-I and II, as there's currently 1037 * about 1 TB of addressable space before the US-I/II VA hole. 1038 */ 1039 kmem64_base = (caddr_t)syslimit; 1040 PRM_DEBUG(kmem64_base); 1041 1042 alloc_base = (caddr_t)roundup((uintptr_t)kmem64_base, alloc_alignsize); 1043 1044 /* 1045 * If KHME and/or UHME hash buckets won't fit in the nucleus, allocate 1046 * them here. 1047 */ 1048 if (khme_hash == NULL || uhme_hash == NULL) { 1049 /* 1050 * alloc_hme_buckets() will align alloc_base properly before 1051 * assigning the hash buckets, so we don't need to do it 1052 * before the call... 1053 */ 1054 alloc_base = alloc_hme_buckets(alloc_base, alloc_alignsize); 1055 1056 PRM_DEBUG(alloc_base); 1057 PRM_DEBUG(khme_hash); 1058 PRM_DEBUG(uhme_hash); 1059 } 1060 1061 /* 1062 * Allocate the remaining page freelists. NUMA systems can 1063 * have lots of page freelists, one per node, which quickly 1064 * outgrow the amount of nucleus memory available. 1065 */ 1066 if (max_mem_nodes > 1) { 1067 int mnode; 1068 caddr_t alloc_start = alloc_base; 1069 1070 for (mnode = 1; mnode < max_mem_nodes; mnode++) { 1071 alloc_base = alloc_page_freelists(mnode, alloc_base, 1072 ecache_alignsize); 1073 } 1074 1075 if (alloc_base > alloc_start) { 1076 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1077 alloc_alignsize); 1078 if ((caddr_t)BOP_ALLOC(bootops, alloc_start, 1079 alloc_base - alloc_start, 1080 alloc_alignsize) != alloc_start) 1081 cmn_err(CE_PANIC, 1082 "Unable to alloc page freelists\n"); 1083 } 1084 1085 PRM_DEBUG(alloc_base); 1086 } 1087 1088 if (!mml_table) { 1089 size_t mmltable_sz; 1090 1091 /* 1092 * We need to allocate the mml_table here because there 1093 * was not enough space within the nucleus. 1094 */ 1095 mmltable_sz = sizeof (kmutex_t) * mml_table_sz; 1096 alloc_sz = roundup(mmltable_sz, alloc_alignsize); 1097 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1098 alloc_alignsize); 1099 1100 if ((mml_table = (kmutex_t *)BOP_ALLOC(bootops, alloc_base, 1101 alloc_sz, alloc_alignsize)) != (kmutex_t *)alloc_base) 1102 panic("mml_table alloc failure"); 1103 1104 alloc_base += alloc_sz; 1105 PRM_DEBUG(mml_table); 1106 PRM_DEBUG(alloc_base); 1107 } 1108 1109 if (kpm_enable && !(kpmp_table || kpmp_stable)) { 1110 size_t kpmptable_sz; 1111 caddr_t table; 1112 1113 /* 1114 * We need to allocate either kpmp_table or kpmp_stable here 1115 * because there was not enough space within the nucleus. 1116 */ 1117 kpmptable_sz = (kpm_smallpages == 0) ? 1118 sizeof (kpm_hlk_t) * kpmp_table_sz : 1119 sizeof (kpm_shlk_t) * kpmp_stable_sz; 1120 1121 alloc_sz = roundup(kpmptable_sz, alloc_alignsize); 1122 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1123 alloc_alignsize); 1124 1125 table = BOP_ALLOC(bootops, alloc_base, alloc_sz, 1126 alloc_alignsize); 1127 1128 if (table != alloc_base) 1129 panic("kpmp_table or kpmp_stable alloc failure"); 1130 1131 if (kpm_smallpages == 0) { 1132 kpmp_table = (kpm_hlk_t *)table; 1133 PRM_DEBUG(kpmp_table); 1134 } else { 1135 kpmp_stable = (kpm_shlk_t *)table; 1136 PRM_DEBUG(kpmp_stable); 1137 } 1138 1139 alloc_base += alloc_sz; 1140 PRM_DEBUG(alloc_base); 1141 } 1142 1143 if (&ecache_init_scrub_flush_area) { 1144 /* 1145 * Pass alloc_base directly, as the routine itself is 1146 * responsible for any special alignment requirements... 1147 */ 1148 alloc_base = ecache_init_scrub_flush_area(alloc_base); 1149 PRM_DEBUG(alloc_base); 1150 } 1151 1152 /* 1153 * Take the most current snapshot we can by calling mem-update. 1154 */ 1155 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1156 &boot_physavail, &boot_physavail_len, 1157 &boot_virtavail, &boot_virtavail_len); 1158 1159 /* 1160 * Reset npages and memblocks based on boot_physavail list. 1161 */ 1162 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 1163 PRM_DEBUG(npages); 1164 1165 /* 1166 * Account for extra memory after e_text. 1167 */ 1168 npages += extra_etpg; 1169 1170 /* 1171 * Calculate the largest free memory chunk in the nucleus data area. 1172 * We need to figure out if page structs can fit in there or not. 1173 * We also make sure enough page structs get created for any physical 1174 * memory we might be returning to the system. 1175 */ 1176 ndata_remain_sz = ndata_maxsize(&ndata); 1177 PRM_DEBUG(ndata_remain_sz); 1178 1179 pp_sz = sizeof (struct page) * npages; 1180 1181 /* 1182 * Here's a nice bit of code based on somewhat recursive logic: 1183 * 1184 * If the page array would fit within the nucleus, we want to 1185 * add npages to cover any extra memory we may be returning back 1186 * to the system. 1187 * 1188 * HOWEVER, the page array is sized by calculating the size of 1189 * (struct page * npages), as are the pagehash table, ctrs and 1190 * memseg_list, so the very act of performing the calculation below may 1191 * in fact make the array large enough that it no longer fits in the 1192 * nucleus, meaning there would now be a much larger area of the 1193 * nucleus free that should really be added to npages, which would 1194 * make the page array that much larger, and so on. 1195 * 1196 * This also ignores the memory possibly used in the nucleus for the 1197 * the page hash, ctrs and memseg list and the fact that whether they 1198 * fit there or not varies with the npages calculation below, but we 1199 * don't even factor them into the equation at this point; perhaps we 1200 * should or perhaps we should just take the approach that the few 1201 * extra pages we could add via this calculation REALLY aren't worth 1202 * the hassle... 1203 */ 1204 if (ndata_remain_sz > pp_sz) { 1205 size_t spare = ndata_spare(&ndata, pp_sz, ecache_alignsize); 1206 1207 npages += mmu_btop(spare); 1208 1209 pp_sz = npages * sizeof (struct page); 1210 1211 pp_base = ndata_alloc(&ndata, pp_sz, ecache_alignsize); 1212 } 1213 1214 /* 1215 * If physmem is patched to be non-zero, use it instead of 1216 * the monitor value unless physmem is larger than the total 1217 * amount of memory on hand. 1218 */ 1219 if (physmem == 0 || physmem > npages) 1220 physmem = npages; 1221 1222 /* 1223 * If pp_base is NULL that means the routines above have determined 1224 * the page array will not fit in the nucleus; we'll have to 1225 * BOP_ALLOC() ourselves some space for them. 1226 */ 1227 if (pp_base == NULL) { 1228 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1229 alloc_alignsize); 1230 1231 alloc_sz = roundup(pp_sz, alloc_alignsize); 1232 1233 if ((pp_base = (struct page *)BOP_ALLOC(bootops, 1234 alloc_base, alloc_sz, alloc_alignsize)) != 1235 (struct page *)alloc_base) 1236 panic("page alloc failure"); 1237 1238 alloc_base += alloc_sz; 1239 } 1240 1241 /* 1242 * The page structure hash table size is a power of 2 1243 * such that the average hash chain length is PAGE_HASHAVELEN. 1244 */ 1245 page_hashsz = npages / PAGE_HASHAVELEN; 1246 page_hashsz = 1 << highbit((ulong_t)page_hashsz); 1247 pagehash_sz = sizeof (struct page *) * page_hashsz; 1248 1249 /* 1250 * We want to TRY to fit the page structure hash table, 1251 * the page size free list counters, the memseg list and 1252 * and the kpm page space in the nucleus if possible. 1253 * 1254 * alloc_sz counts how much memory needs to be allocated by 1255 * BOP_ALLOC(). 1256 */ 1257 page_hash = ndata_alloc(&ndata, pagehash_sz, ecache_alignsize); 1258 1259 alloc_sz = (page_hash == NULL ? pagehash_sz : 0); 1260 1261 /* 1262 * Size up per page size free list counters. 1263 */ 1264 ctrs_sz = page_ctrs_sz(); 1265 ctrs_base = ndata_alloc(&ndata, ctrs_sz, ecache_alignsize); 1266 1267 if (ctrs_base == NULL) 1268 alloc_sz = roundup(alloc_sz, ecache_alignsize) + ctrs_sz; 1269 1270 /* 1271 * The memseg list is for the chunks of physical memory that 1272 * will be managed by the vm system. The number calculated is 1273 * a guess as boot may fragment it more when memory allocations 1274 * are made before kphysm_init(). Currently, there are two 1275 * allocations before then, so we assume each causes fragmen- 1276 * tation, and add a couple more for good measure. 1277 */ 1278 memseg_sz = sizeof (struct memseg) * (memblocks + 4); 1279 memseg_base = ndata_alloc(&ndata, memseg_sz, ecache_alignsize); 1280 1281 if (memseg_base == NULL) 1282 alloc_sz = roundup(alloc_sz, ecache_alignsize) + memseg_sz; 1283 1284 1285 if (kpm_enable) { 1286 /* 1287 * kpm page space -- Update kpm_npages and make the 1288 * same assumption about fragmenting as it is done 1289 * for memseg_sz above. 1290 */ 1291 kpm_npages_setup(memblocks + 4); 1292 kpm_pp_sz = (kpm_smallpages == 0) ? 1293 kpm_npages * sizeof (kpm_page_t): 1294 kpm_npages * sizeof (kpm_spage_t); 1295 1296 kpm_pp_base = (uintptr_t)ndata_alloc(&ndata, kpm_pp_sz, 1297 ecache_alignsize); 1298 1299 if (kpm_pp_base == NULL) 1300 alloc_sz = roundup(alloc_sz, ecache_alignsize) + 1301 kpm_pp_sz; 1302 } 1303 1304 if (alloc_sz > 0) { 1305 uintptr_t bop_base; 1306 1307 /* 1308 * We need extra memory allocated through BOP_ALLOC. 1309 */ 1310 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1311 alloc_alignsize); 1312 1313 alloc_sz = roundup(alloc_sz, alloc_alignsize); 1314 1315 if ((bop_base = (uintptr_t)BOP_ALLOC(bootops, alloc_base, 1316 alloc_sz, alloc_alignsize)) != (uintptr_t)alloc_base) 1317 panic("system page struct alloc failure"); 1318 1319 alloc_base += alloc_sz; 1320 1321 if (page_hash == NULL) { 1322 page_hash = (struct page **)bop_base; 1323 bop_base = roundup(bop_base + pagehash_sz, 1324 ecache_alignsize); 1325 } 1326 1327 if (ctrs_base == NULL) { 1328 ctrs_base = (caddr_t)bop_base; 1329 bop_base = roundup(bop_base + ctrs_sz, 1330 ecache_alignsize); 1331 } 1332 1333 if (memseg_base == NULL) { 1334 memseg_base = (struct memseg *)bop_base; 1335 bop_base = roundup(bop_base + memseg_sz, 1336 ecache_alignsize); 1337 } 1338 1339 if (kpm_enable && kpm_pp_base == NULL) { 1340 kpm_pp_base = (uintptr_t)bop_base; 1341 bop_base = roundup(bop_base + kpm_pp_sz, 1342 ecache_alignsize); 1343 } 1344 1345 ASSERT(bop_base <= (uintptr_t)alloc_base); 1346 } 1347 1348 /* 1349 * Initialize per page size free list counters. 1350 */ 1351 ctrs_end = page_ctrs_alloc(ctrs_base); 1352 ASSERT(ctrs_base + ctrs_sz >= ctrs_end); 1353 1354 PRM_DEBUG(page_hash); 1355 PRM_DEBUG(memseg_base); 1356 PRM_DEBUG(kpm_pp_base); 1357 PRM_DEBUG(kpm_pp_sz); 1358 PRM_DEBUG(pp_base); 1359 PRM_DEBUG(pp_sz); 1360 PRM_DEBUG(alloc_base); 1361 1362 #ifdef TRAPTRACE 1363 /* 1364 * Allocate trap trace buffer last so as not to affect 1365 * the 4M alignments of the allocations above on V9 SPARCs... 1366 */ 1367 alloc_base = trap_trace_alloc(alloc_base); 1368 PRM_DEBUG(alloc_base); 1369 #endif /* TRAPTRACE */ 1370 1371 if (kmem64_base) { 1372 /* 1373 * Set the end of the kmem64 segment for V9 SPARCs, if 1374 * appropriate... 1375 */ 1376 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, 1377 alloc_alignsize); 1378 1379 PRM_DEBUG(kmem64_base); 1380 PRM_DEBUG(kmem64_end); 1381 } 1382 1383 /* 1384 * Allocate space for the interrupt vector table. 1385 */ 1386 memspace = (caddr_t)BOP_ALLOC(bootops, (caddr_t)intr_vector, 1387 IVSIZE, MMU_PAGESIZE); 1388 if (memspace != (caddr_t)intr_vector) 1389 panic("interrupt table allocation failure"); 1390 1391 /* 1392 * The memory lists from boot are allocated from the heap arena 1393 * so that later they can be freed and/or reallocated. 1394 */ 1395 if (BOP_GETPROP(bootops, "extent", &memlist_sz) == -1) 1396 panic("could not retrieve property \"extent\""); 1397 1398 /* 1399 * Between now and when we finish copying in the memory lists, 1400 * allocations happen so the space gets fragmented and the 1401 * lists longer. Leave enough space for lists twice as long 1402 * as what boot says it has now; roundup to a pagesize. 1403 * Also add space for the final phys-avail copy in the fixup 1404 * routine. 1405 */ 1406 va = (caddr_t)(sysbase + PAGESIZE + PANICBUFSIZE + 1407 roundup(IVSIZE, MMU_PAGESIZE)); 1408 memlist_sz *= 4; 1409 memlist_sz = roundup(memlist_sz, MMU_PAGESIZE); 1410 memspace = (caddr_t)BOP_ALLOC(bootops, va, memlist_sz, BO_NO_ALIGN); 1411 if (memspace == NULL) 1412 halt("Boot allocation failed."); 1413 1414 memlist = (struct memlist *)memspace; 1415 memlist_end = (char *)memspace + memlist_sz; 1416 1417 PRM_DEBUG(memlist); 1418 PRM_DEBUG(memlist_end); 1419 PRM_DEBUG(sysbase); 1420 PRM_DEBUG(syslimit); 1421 1422 kernelheap_init((void *)sysbase, (void *)syslimit, 1423 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1424 1425 /* 1426 * Take the most current snapshot we can by calling mem-update. 1427 */ 1428 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1429 &boot_physavail, &boot_physavail_len, 1430 &boot_virtavail, &boot_virtavail_len); 1431 1432 /* 1433 * Remove the space used by BOP_ALLOC from the kernel heap 1434 * plus the area actually used by the OBP (if any) 1435 * ignoring virtual addresses in virt_avail, above syslimit. 1436 */ 1437 virt_avail = memlist; 1438 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1439 1440 for (cur = virt_avail; cur->next; cur = cur->next) { 1441 uint64_t range_base, range_size; 1442 1443 if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1444 continue; 1445 if (range_base >= (uint64_t)syslimit) 1446 break; 1447 /* 1448 * Limit the range to end at syslimit. 1449 */ 1450 range_size = MIN(cur->next->address, 1451 (uint64_t)syslimit) - range_base; 1452 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1453 0, 0, (void *)range_base, (void *)(range_base + range_size), 1454 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1455 } 1456 1457 phys_avail = memlist; 1458 (void) copy_physavail(boot_physavail, boot_physavail_len, 1459 &memlist, 0, 0); 1460 1461 /* 1462 * Add any extra memory after e_text to the phys_avail list, as long 1463 * as there's at least a page to add. 1464 */ 1465 if (extra_etpg) 1466 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1467 &memlist, &phys_avail); 1468 1469 /* 1470 * Add any extra memory after e_data to the phys_avail list as long 1471 * as there's at least a page to add. Usually, there isn't any, 1472 * since extra HME blocks typically get allocated there first before 1473 * using RAM elsewhere. 1474 */ 1475 if ((nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE)) == NULL) 1476 nalloc_base = nalloc_end; 1477 ndata_remain_sz = nalloc_end - nalloc_base; 1478 1479 if (ndata_remain_sz >= MMU_PAGESIZE) 1480 memlist_add(va_to_pa(nalloc_base), 1481 (uint64_t)ndata_remain_sz, &memlist, &phys_avail); 1482 1483 PRM_DEBUG(memlist); 1484 PRM_DEBUG(memlist_sz); 1485 PRM_DEBUG(memspace); 1486 1487 if ((caddr_t)memlist > (memspace + memlist_sz)) 1488 panic("memlist overflow"); 1489 1490 PRM_DEBUG(pp_base); 1491 PRM_DEBUG(memseg_base); 1492 PRM_DEBUG(npages); 1493 1494 /* 1495 * Initialize the page structures from the memory lists. 1496 */ 1497 kphysm_init(pp_base, memseg_base, npages, kpm_pp_base, kpm_npages); 1498 1499 availrmem_initial = availrmem = freemem; 1500 PRM_DEBUG(availrmem); 1501 1502 /* 1503 * Some of the locks depend on page_hashsz being set! 1504 * kmem_init() depends on this; so, keep it here. 1505 */ 1506 page_lock_init(); 1507 1508 /* 1509 * Initialize kernel memory allocator. 1510 */ 1511 kmem_init(); 1512 1513 /* 1514 * Initialize bp_mapin(). 1515 */ 1516 bp_init(shm_alignment, HAT_STRICTORDER); 1517 1518 /* 1519 * Reserve space for panicbuf and intr_vector from the 32-bit heap 1520 */ 1521 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1522 panicbuf, panicbuf + PANICBUFSIZE, 1523 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1524 1525 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1526 intr_vector, (caddr_t)intr_vector + IVSIZE, 1527 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1528 1529 mem_config_init(); 1530 } 1531 1532 static void 1533 startup_modules(void) 1534 { 1535 int proplen, nhblk1, nhblk8; 1536 size_t nhblksz; 1537 pgcnt_t hblk_pages, pages_per_hblk; 1538 size_t hme8blk_sz, hme1blk_sz; 1539 1540 /* 1541 * Log any optional messages from the boot program 1542 */ 1543 proplen = (size_t)BOP_GETPROPLEN(bootops, "boot-message"); 1544 if (proplen > 0) { 1545 char *msg; 1546 size_t len = (size_t)proplen; 1547 1548 msg = kmem_zalloc(len, KM_SLEEP); 1549 (void) BOP_GETPROP(bootops, "boot-message", msg); 1550 cmn_err(CE_CONT, "?%s\n", msg); 1551 kmem_free(msg, len); 1552 } 1553 1554 /* 1555 * Let the platforms have a chance to change default 1556 * values before reading system file. 1557 */ 1558 if (&set_platform_defaults) 1559 set_platform_defaults(); 1560 1561 /* 1562 * Calculate default settings of system parameters based upon 1563 * maxusers, yet allow to be overridden via the /etc/system file. 1564 */ 1565 param_calc(0); 1566 1567 mod_setup(); 1568 1569 /* 1570 * If this is a positron, complain and halt. 1571 */ 1572 if (&iam_positron && iam_positron()) { 1573 cmn_err(CE_WARN, "This hardware platform is not supported" 1574 " by this release of Solaris.\n"); 1575 #ifdef DEBUG 1576 prom_enter_mon(); /* Type 'go' to resume */ 1577 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1578 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1579 1580 #else /* DEBUG */ 1581 halt(0); 1582 #endif /* DEBUG */ 1583 } 1584 1585 /* 1586 * If we are running firmware that isn't 64-bit ready 1587 * then complain and halt. 1588 */ 1589 do_prom_version_check(); 1590 1591 /* 1592 * Initialize system parameters 1593 */ 1594 param_init(); 1595 1596 /* 1597 * maxmem is the amount of physical memory we're playing with. 1598 */ 1599 maxmem = physmem; 1600 1601 /* Set segkp limits. */ 1602 ncbase = (caddr_t)SEGDEBUGBASE; 1603 ncend = (caddr_t)SEGDEBUGBASE; 1604 1605 /* 1606 * Initialize the hat layer. 1607 */ 1608 hat_init(); 1609 1610 /* 1611 * Initialize segment management stuff. 1612 */ 1613 seg_init(); 1614 1615 /* 1616 * Create the va>tte handler, so the prom can understand 1617 * kernel translations. The handler is installed later, just 1618 * as we are about to take over the trap table from the prom. 1619 */ 1620 create_va_to_tte(); 1621 1622 /* 1623 * Load the forthdebugger (optional) 1624 */ 1625 forthdebug_init(); 1626 1627 /* 1628 * Create OBP node for console input callbacks 1629 * if it is needed. 1630 */ 1631 startup_create_input_node(); 1632 1633 if (modloadonly("fs", "specfs") == -1) 1634 halt("Can't load specfs"); 1635 1636 if (modloadonly("fs", "devfs") == -1) 1637 halt("Can't load devfs"); 1638 1639 if (modloadonly("misc", "swapgeneric") == -1) 1640 halt("Can't load swapgeneric"); 1641 1642 dispinit(); 1643 1644 /* 1645 * Infer meanings to the members of the idprom buffer. 1646 */ 1647 parse_idprom(); 1648 1649 /* Read cluster configuration data. */ 1650 clconf_init(); 1651 1652 setup_ddi(); 1653 1654 /* 1655 * Lets take this opportunity to load the root device. 1656 */ 1657 if (loadrootmodules() != 0) 1658 debug_enter("Can't load the root filesystem"); 1659 1660 /* 1661 * Load tod driver module for the tod part found on this system. 1662 * Recompute the cpu frequency/delays based on tod as tod part 1663 * tends to keep time more accurately. 1664 */ 1665 if (&load_tod_module) 1666 load_tod_module(); 1667 1668 /* 1669 * Allow platforms to load modules which might 1670 * be needed after bootops are gone. 1671 */ 1672 if (&load_platform_modules) 1673 load_platform_modules(); 1674 1675 setcpudelay(); 1676 1677 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1678 &boot_physavail, &boot_physavail_len, 1679 &boot_virtavail, &boot_virtavail_len); 1680 1681 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1682 1683 /* 1684 * Calculation and allocation of hmeblks needed to remap 1685 * the memory allocated by PROM till now: 1686 * 1687 * (1) calculate how much virtual memory has been bop_alloc'ed. 1688 * (2) roundup this memory to span of hme8blk, i.e. 64KB 1689 * (3) calculate number of hme8blk's needed to remap this memory 1690 * (4) calculate amount of memory that's consumed by these hme8blk's 1691 * (5) add memory calculated in steps (2) and (4) above. 1692 * (6) roundup this memory to span of hme8blk, i.e. 64KB 1693 * (7) calculate number of hme8blk's needed to remap this memory 1694 * (8) calculate amount of memory that's consumed by these hme8blk's 1695 * (9) allocate additional hme1blk's to hold large mappings. 1696 * H8TOH1 determines this. The current SWAG gives enough hblk1's 1697 * to remap everything with 4M mappings. 1698 * (10) account for partially used hblk8's due to non-64K aligned 1699 * PROM mapping entries. 1700 * (11) add memory calculated in steps (8), (9), and (10) above. 1701 * (12) kmem_zalloc the memory calculated in (11); since segkmem 1702 * is not ready yet, this gets bop_alloc'ed. 1703 * (13) there will be very few bop_alloc's after this point before 1704 * trap table takes over 1705 */ 1706 1707 /* sfmmu_init_nucleus_hblks expects properly aligned data structures. */ 1708 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1709 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1710 1711 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1712 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1713 nhblk8 = bop_alloc_pages / pages_per_hblk; 1714 nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 1715 hblk_pages = btopr(nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz); 1716 bop_alloc_pages += hblk_pages; 1717 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1718 nhblk8 = bop_alloc_pages / pages_per_hblk; 1719 nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 1720 if (nhblk1 < hblk1_min) 1721 nhblk1 = hblk1_min; 1722 if (nhblk8 < hblk8_min) 1723 nhblk8 = hblk8_min; 1724 1725 /* 1726 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1727 * boundary, the number of hblk8's needed to map the entries in the 1728 * boot_virtavail list needs to be adjusted to take this into 1729 * consideration. Thus, we need to add additional hblk8's since it 1730 * is possible that an hblk8 will not have all 8 slots used due to 1731 * alignment constraints. Since there were boot_virtavail_len entries 1732 * in that list, we need to add that many hblk8's to the number 1733 * already calculated to make sure we don't underestimate. 1734 */ 1735 nhblk8 += boot_virtavail_len; 1736 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1737 1738 /* Allocate in pagesize chunks */ 1739 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1740 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1741 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1742 } 1743 1744 static void 1745 startup_bop_gone(void) 1746 { 1747 extern int bop_io_quiesced; 1748 1749 /* 1750 * Call back into boot and release boots resources. 1751 */ 1752 BOP_QUIESCE_IO(bootops); 1753 bop_io_quiesced = 1; 1754 1755 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1756 &boot_physavail, &boot_physavail_len, 1757 &boot_virtavail, &boot_virtavail_len); 1758 /* 1759 * Copy physinstalled list into kernel space. 1760 */ 1761 phys_install = memlist; 1762 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1763 1764 /* 1765 * setup physically contiguous area twice as large as the ecache. 1766 * this is used while doing displacement flush of ecaches 1767 */ 1768 if (&ecache_flush_address) { 1769 ecache_flushaddr = ecache_flush_address(); 1770 if (ecache_flushaddr == (uint64_t)-1) { 1771 cmn_err(CE_PANIC, 1772 "startup: no memory to set ecache_flushaddr"); 1773 } 1774 } 1775 1776 /* 1777 * Virtual available next. 1778 */ 1779 ASSERT(virt_avail != NULL); 1780 memlist_free_list(virt_avail); 1781 virt_avail = memlist; 1782 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1783 1784 /* 1785 * Last chance to ask our booter questions .. 1786 */ 1787 } 1788 1789 1790 /* 1791 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1792 * allocations have been performed. We can't call it in startup_bop_gone 1793 * since later operations can cause obp to allocate more memory. 1794 */ 1795 void 1796 startup_fixup_physavail(void) 1797 { 1798 struct memlist *cur; 1799 1800 /* 1801 * take the most current snapshot we can by calling mem-update 1802 */ 1803 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1804 &boot_physavail, &boot_physavail_len, 1805 &boot_virtavail, &boot_virtavail_len); 1806 1807 /* 1808 * Copy phys_avail list, again. 1809 * Both the kernel/boot and the prom have been allocating 1810 * from the original list we copied earlier. 1811 */ 1812 cur = memlist; 1813 (void) copy_physavail(boot_physavail, boot_physavail_len, 1814 &memlist, 0, 0); 1815 1816 /* 1817 * Add any extra memory after e_text we added to the phys_avail list 1818 * back to the old list. 1819 */ 1820 if (extra_etpg) 1821 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1822 &memlist, &cur); 1823 if (ndata_remain_sz >= MMU_PAGESIZE) 1824 memlist_add(va_to_pa(nalloc_base), 1825 (uint64_t)ndata_remain_sz, &memlist, &cur); 1826 1827 /* 1828 * There isn't any bounds checking on the memlist area 1829 * so ensure it hasn't overgrown. 1830 */ 1831 if ((caddr_t)memlist > (caddr_t)memlist_end) 1832 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1833 1834 /* 1835 * The kernel removes the pages that were allocated for it from 1836 * the freelist, but we now have to find any -extra- pages that 1837 * the prom has allocated for it's own book-keeping, and remove 1838 * them from the freelist too. sigh. 1839 */ 1840 fix_prom_pages(phys_avail, cur); 1841 1842 ASSERT(phys_avail != NULL); 1843 memlist_free_list(phys_avail); 1844 phys_avail = cur; 1845 1846 /* 1847 * We're done with boot. Just after this point in time, boot 1848 * gets unmapped, so we can no longer rely on its services. 1849 * Zero the bootops to indicate this fact. 1850 */ 1851 bootops = (struct bootops *)NULL; 1852 BOOTOPS_GONE(); 1853 } 1854 1855 static void 1856 startup_vm(void) 1857 { 1858 size_t i; 1859 struct segmap_crargs a; 1860 struct segkpm_crargs b; 1861 1862 uint64_t avmem; 1863 caddr_t va; 1864 pgcnt_t max_phys_segkp; 1865 int mnode; 1866 1867 extern int exec_lpg_disable, use_brk_lpg, use_stk_lpg, use_zmap_lpg; 1868 1869 /* 1870 * get prom's mappings, create hments for them and switch 1871 * to the kernel context. 1872 */ 1873 hat_kern_setup(); 1874 1875 /* 1876 * Take over trap table 1877 */ 1878 setup_trap_table(); 1879 1880 /* 1881 * Install the va>tte handler, so that the prom can handle 1882 * misses and understand the kernel table layout in case 1883 * we need call into the prom. 1884 */ 1885 install_va_to_tte(); 1886 1887 /* 1888 * Set a flag to indicate that the tba has been taken over. 1889 */ 1890 tba_taken_over = 1; 1891 1892 /* initialize MMU primary context register */ 1893 mmu_init_kcontext(); 1894 1895 /* 1896 * The boot cpu can now take interrupts, x-calls, x-traps 1897 */ 1898 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1899 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1900 1901 /* 1902 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1903 */ 1904 tbr_wr_addr_inited = 1; 1905 1906 /* 1907 * Initialize VM system, and map kernel address space. 1908 */ 1909 kvm_init(); 1910 1911 /* 1912 * XXX4U: previously, we initialized and turned on 1913 * the caches at this point. But of course we have 1914 * nothing to do, as the prom has already done this 1915 * for us -- main memory must be E$able at all times. 1916 */ 1917 1918 /* 1919 * If the following is true, someone has patched 1920 * phsymem to be less than the number of pages that 1921 * the system actually has. Remove pages until system 1922 * memory is limited to the requested amount. Since we 1923 * have allocated page structures for all pages, we 1924 * correct the amount of memory we want to remove 1925 * by the size of the memory used to hold page structures 1926 * for the non-used pages. 1927 */ 1928 if (physmem < npages) { 1929 pgcnt_t diff, off; 1930 struct page *pp; 1931 struct seg kseg; 1932 1933 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1934 1935 off = 0; 1936 diff = npages - physmem; 1937 diff -= mmu_btopr(diff * sizeof (struct page)); 1938 kseg.s_as = &kas; 1939 while (diff--) { 1940 pp = page_create_va(&unused_pages_vp, (offset_t)off, 1941 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1942 &kseg, (caddr_t)off); 1943 if (pp == NULL) 1944 cmn_err(CE_PANIC, "limited physmem too much!"); 1945 page_io_unlock(pp); 1946 page_downgrade(pp); 1947 availrmem--; 1948 off += MMU_PAGESIZE; 1949 } 1950 } 1951 1952 /* 1953 * When printing memory, show the total as physmem less 1954 * that stolen by a debugger. 1955 */ 1956 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1957 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1958 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1959 1960 avmem = (uint64_t)freemem << PAGESHIFT; 1961 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1962 1963 /* For small memory systems disable automatic large pages. */ 1964 if (physmem < auto_lpg_min_physmem) { 1965 exec_lpg_disable = 1; 1966 use_brk_lpg = 0; 1967 use_stk_lpg = 0; 1968 use_zmap_lpg = 0; 1969 } 1970 1971 /* 1972 * Perform platform specific freelist processing 1973 */ 1974 if (&plat_freelist_process) { 1975 for (mnode = 0; mnode < max_mem_nodes; mnode++) 1976 if (mem_node_config[mnode].exists) 1977 plat_freelist_process(mnode); 1978 } 1979 1980 /* 1981 * Initialize the segkp segment type. We position it 1982 * after the configured tables and buffers (whose end 1983 * is given by econtig) and before V_WKBASE_ADDR. 1984 * Also in this area is segkmap (size SEGMAPSIZE). 1985 */ 1986 1987 /* XXX - cache alignment? */ 1988 va = (caddr_t)SEGKPBASE; 1989 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 1990 1991 max_phys_segkp = (physmem * 2); 1992 1993 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 1994 segkpsize = btop(SEGKPDEFSIZE); 1995 cmn_err(CE_WARN, "Illegal value for segkpsize. " 1996 "segkpsize has been reset to %ld pages", segkpsize); 1997 } 1998 1999 i = ptob(MIN(segkpsize, max_phys_segkp)); 2000 2001 rw_enter(&kas.a_lock, RW_WRITER); 2002 if (seg_attach(&kas, va, i, segkp) < 0) 2003 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 2004 if (segkp_create(segkp) != 0) 2005 cmn_err(CE_PANIC, "startup: segkp_create failed"); 2006 rw_exit(&kas.a_lock); 2007 2008 /* 2009 * kpm segment 2010 */ 2011 segmap_kpm = kpm_enable && 2012 segmap_kpm && PAGESIZE == MAXBSIZE; 2013 2014 if (kpm_enable) { 2015 rw_enter(&kas.a_lock, RW_WRITER); 2016 2017 /* 2018 * The segkpm virtual range range is larger than the 2019 * actual physical memory size and also covers gaps in 2020 * the physical address range for the following reasons: 2021 * . keep conversion between segkpm and physical addresses 2022 * simple, cheap and unambiguous. 2023 * . avoid extension/shrink of the the segkpm in case of DR. 2024 * . avoid complexity for handling of virtual addressed 2025 * caches, segkpm and the regular mapping scheme must be 2026 * kept in sync wrt. the virtual color of mapped pages. 2027 * Any accesses to virtual segkpm ranges not backed by 2028 * physical memory will fall through the memseg pfn hash 2029 * and will be handled in segkpm_fault. 2030 * Additional kpm_size spaces needed for vac alias prevention. 2031 */ 2032 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 2033 segkpm) < 0) 2034 cmn_err(CE_PANIC, "cannot attach segkpm"); 2035 2036 b.prot = PROT_READ | PROT_WRITE; 2037 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 2038 2039 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 2040 panic("segkpm_create segkpm"); 2041 2042 rw_exit(&kas.a_lock); 2043 } 2044 2045 /* 2046 * Now create generic mapping segment. This mapping 2047 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2048 * virtual address is greater than the amount of free 2049 * memory that is available, then we trim back the 2050 * segment size to that amount 2051 */ 2052 va = (caddr_t)SEGMAPBASE; 2053 2054 /* 2055 * 1201049: segkmap base address must be MAXBSIZE aligned 2056 */ 2057 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2058 2059 /* 2060 * Set size of segmap to percentage of freemem at boot, 2061 * but stay within the allowable range 2062 * Note we take percentage before converting from pages 2063 * to bytes to avoid an overflow on 32-bit kernels. 2064 */ 2065 i = mmu_ptob((freemem * segmap_percent) / 100); 2066 2067 if (i < MINMAPSIZE) 2068 i = MINMAPSIZE; 2069 2070 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2071 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2072 2073 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2074 2075 rw_enter(&kas.a_lock, RW_WRITER); 2076 if (seg_attach(&kas, va, i, segkmap) < 0) 2077 cmn_err(CE_PANIC, "cannot attach segkmap"); 2078 2079 a.prot = PROT_READ | PROT_WRITE; 2080 a.shmsize = shm_alignment; 2081 a.nfreelist = 0; /* use segmap driver defaults */ 2082 2083 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2084 panic("segmap_create segkmap"); 2085 rw_exit(&kas.a_lock); 2086 2087 segdev_init(); 2088 } 2089 2090 static void 2091 startup_end(void) 2092 { 2093 if ((caddr_t)memlist > (caddr_t)memlist_end) 2094 panic("memlist overflow 2"); 2095 memlist_free_block((caddr_t)memlist, 2096 ((caddr_t)memlist_end - (caddr_t)memlist)); 2097 memlist = NULL; 2098 2099 /* enable page_relocation since OBP is now done */ 2100 page_relocate_ready = 1; 2101 2102 /* 2103 * Perform tasks that get done after most of the VM 2104 * initialization has been done but before the clock 2105 * and other devices get started. 2106 */ 2107 kern_setup1(); 2108 2109 /* 2110 * Intialize the VM arenas for allocating physically 2111 * contiguus memory chunk for interrupt queues snd 2112 * allocate/register boot cpu's queues, if any and 2113 * allocate dump buffer for sun4v systems to store 2114 * extra crash information during crash dump 2115 */ 2116 contig_mem_init(); 2117 mach_descrip_init(); 2118 cpu_intrq_setup(CPU); 2119 cpu_intrq_register(CPU); 2120 mach_htraptrace_init(); 2121 mach_htraptrace_setup(CPU->cpu_id); 2122 mach_htraptrace_configure(CPU->cpu_id); 2123 mach_dump_buffer_init(); 2124 2125 /* 2126 * Initialize interrupt related stuff 2127 */ 2128 init_intr_threads(CPU); 2129 2130 (void) splzs(); /* allow hi clock ints but not zs */ 2131 2132 /* 2133 * Initialize errors. 2134 */ 2135 error_init(); 2136 2137 /* 2138 * Note that we may have already used kernel bcopy before this 2139 * point - but if you really care about this, adb the use_hw_* 2140 * variables to 0 before rebooting. 2141 */ 2142 mach_hw_copy_limit(); 2143 2144 /* 2145 * Install the "real" preemption guards before DDI services 2146 * are available. 2147 */ 2148 (void) prom_set_preprom(kern_preprom); 2149 (void) prom_set_postprom(kern_postprom); 2150 CPU->cpu_m.mutex_ready = 1; 2151 2152 /* 2153 * Initialize segnf (kernel support for non-faulting loads). 2154 */ 2155 segnf_init(); 2156 2157 /* 2158 * Configure the root devinfo node. 2159 */ 2160 configure(); /* set up devices */ 2161 mach_cpu_halt_idle(); 2162 } 2163 2164 2165 void 2166 post_startup(void) 2167 { 2168 #ifdef PTL1_PANIC_DEBUG 2169 extern void init_ptl1_thread(void); 2170 #endif /* PTL1_PANIC_DEBUG */ 2171 extern void abort_sequence_init(void); 2172 2173 /* 2174 * Set the system wide, processor-specific flags to be passed 2175 * to userland via the aux vector for performance hints and 2176 * instruction set extensions. 2177 */ 2178 bind_hwcap(); 2179 2180 /* 2181 * Startup memory scrubber (if any) 2182 */ 2183 mach_memscrub(); 2184 2185 /* 2186 * Allocate soft interrupt to handle abort sequence. 2187 */ 2188 abort_sequence_init(); 2189 2190 /* 2191 * Configure the rest of the system. 2192 * Perform forceloading tasks for /etc/system. 2193 */ 2194 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2195 /* 2196 * ON4.0: Force /proc module in until clock interrupt handle fixed 2197 * ON4.0: This must be fixed or restated in /etc/systems. 2198 */ 2199 (void) modload("fs", "procfs"); 2200 2201 if (&load_platform_drivers) 2202 load_platform_drivers(); 2203 2204 /* load vis simulation module, if we are running w/fpu off */ 2205 if (!fpu_exists) { 2206 if (modload("misc", "vis") == -1) 2207 halt("Can't load vis"); 2208 } 2209 2210 mach_fpras(); 2211 2212 maxmem = freemem; 2213 2214 #ifdef PTL1_PANIC_DEBUG 2215 init_ptl1_thread(); 2216 #endif /* PTL1_PANIC_DEBUG */ 2217 } 2218 2219 #ifdef PTL1_PANIC_DEBUG 2220 int ptl1_panic_test = 0; 2221 int ptl1_panic_xc_one_test = 0; 2222 int ptl1_panic_xc_all_test = 0; 2223 int ptl1_panic_xt_one_test = 0; 2224 int ptl1_panic_xt_all_test = 0; 2225 kthread_id_t ptl1_thread_p = NULL; 2226 kcondvar_t ptl1_cv; 2227 kmutex_t ptl1_mutex; 2228 int ptl1_recurse_count_threshold = 0x40; 2229 int ptl1_recurse_trap_threshold = 0x3d; 2230 extern void ptl1_recurse(int, int); 2231 extern void ptl1_panic_xt(int, int); 2232 2233 /* 2234 * Called once per second by timeout() to wake up 2235 * the ptl1_panic thread to see if it should cause 2236 * a trap to the ptl1_panic() code. 2237 */ 2238 /* ARGSUSED */ 2239 static void 2240 ptl1_wakeup(void *arg) 2241 { 2242 mutex_enter(&ptl1_mutex); 2243 cv_signal(&ptl1_cv); 2244 mutex_exit(&ptl1_mutex); 2245 } 2246 2247 /* 2248 * ptl1_panic cross call function: 2249 * Needed because xc_one() and xc_some() can pass 2250 * 64 bit args but ptl1_recurse() expects ints. 2251 */ 2252 static void 2253 ptl1_panic_xc(void) 2254 { 2255 ptl1_recurse(ptl1_recurse_count_threshold, 2256 ptl1_recurse_trap_threshold); 2257 } 2258 2259 /* 2260 * The ptl1 thread waits for a global flag to be set 2261 * and uses the recurse thresholds to set the stack depth 2262 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2263 * or indirectly via the cross call and cross trap functions. 2264 * 2265 * This is useful testing stack overflows and normal 2266 * ptl1_panic() states with a know stack frame. 2267 * 2268 * ptl1_recurse() is an asm function in ptl1_panic.s that 2269 * sets the {In, Local, Out, and Global} registers to a 2270 * know state on the stack and just prior to causing a 2271 * test ptl1_panic trap. 2272 */ 2273 static void 2274 ptl1_thread(void) 2275 { 2276 mutex_enter(&ptl1_mutex); 2277 while (ptl1_thread_p) { 2278 cpuset_t other_cpus; 2279 int cpu_id; 2280 int my_cpu_id; 2281 int target_cpu_id; 2282 int target_found; 2283 2284 if (ptl1_panic_test) { 2285 ptl1_recurse(ptl1_recurse_count_threshold, 2286 ptl1_recurse_trap_threshold); 2287 } 2288 2289 /* 2290 * Find potential targets for x-call and x-trap, 2291 * if any exist while preempt is disabled we 2292 * start a ptl1_panic if requested via a 2293 * globals. 2294 */ 2295 kpreempt_disable(); 2296 my_cpu_id = CPU->cpu_id; 2297 other_cpus = cpu_ready_set; 2298 CPUSET_DEL(other_cpus, CPU->cpu_id); 2299 target_found = 0; 2300 if (!CPUSET_ISNULL(other_cpus)) { 2301 /* 2302 * Pick the first one 2303 */ 2304 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2305 if (cpu_id == my_cpu_id) 2306 continue; 2307 2308 if (CPU_XCALL_READY(cpu_id)) { 2309 target_cpu_id = cpu_id; 2310 target_found = 1; 2311 break; 2312 } 2313 } 2314 ASSERT(target_found); 2315 2316 if (ptl1_panic_xc_one_test) { 2317 xc_one(target_cpu_id, 2318 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2319 } 2320 if (ptl1_panic_xc_all_test) { 2321 xc_some(other_cpus, 2322 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2323 } 2324 if (ptl1_panic_xt_one_test) { 2325 xt_one(target_cpu_id, 2326 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2327 } 2328 if (ptl1_panic_xt_all_test) { 2329 xt_some(other_cpus, 2330 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2331 } 2332 } 2333 kpreempt_enable(); 2334 (void) timeout(ptl1_wakeup, NULL, hz); 2335 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2336 } 2337 mutex_exit(&ptl1_mutex); 2338 } 2339 2340 /* 2341 * Called during early startup to create the ptl1_thread 2342 */ 2343 void 2344 init_ptl1_thread(void) 2345 { 2346 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2347 &p0, TS_RUN, 0); 2348 } 2349 #endif /* PTL1_PANIC_DEBUG */ 2350 2351 2352 /* 2353 * Add to a memory list. 2354 * start = start of new memory segment 2355 * len = length of new memory segment in bytes 2356 * memlistp = pointer to array of available memory segment structures 2357 * curmemlistp = memory list to which to add segment. 2358 */ 2359 static void 2360 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2361 struct memlist **curmemlistp) 2362 { 2363 struct memlist *new; 2364 2365 new = *memlistp; 2366 new->address = start; 2367 new->size = len; 2368 *memlistp = new + 1; 2369 2370 memlist_insert(new, curmemlistp); 2371 } 2372 2373 /* 2374 * In the case of architectures that support dynamic addition of 2375 * memory at run-time there are two cases where memsegs need to 2376 * be initialized and added to the memseg list. 2377 * 1) memsegs that are constructed at startup. 2378 * 2) memsegs that are constructed at run-time on 2379 * hot-plug capable architectures. 2380 * This code was originally part of the function kphysm_init(). 2381 */ 2382 2383 static void 2384 memseg_list_add(struct memseg *memsegp) 2385 { 2386 struct memseg **prev_memsegp; 2387 pgcnt_t num; 2388 2389 /* insert in memseg list, decreasing number of pages order */ 2390 2391 num = MSEG_NPAGES(memsegp); 2392 2393 for (prev_memsegp = &memsegs; *prev_memsegp; 2394 prev_memsegp = &((*prev_memsegp)->next)) { 2395 if (num > MSEG_NPAGES(*prev_memsegp)) 2396 break; 2397 } 2398 2399 memsegp->next = *prev_memsegp; 2400 *prev_memsegp = memsegp; 2401 2402 if (kpm_enable) { 2403 memsegp->nextpa = (memsegp->next) ? 2404 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2405 2406 if (prev_memsegp != &memsegs) { 2407 struct memseg *msp; 2408 msp = (struct memseg *)((caddr_t)prev_memsegp - 2409 offsetof(struct memseg, next)); 2410 msp->nextpa = va_to_pa(memsegp); 2411 } else { 2412 memsegspa = va_to_pa(memsegs); 2413 } 2414 } 2415 } 2416 2417 /* 2418 * PSM add_physmem_cb(). US-II and newer processors have some 2419 * flavor of the prefetch capability implemented. We exploit 2420 * this capability for optimum performance. 2421 */ 2422 #define PREFETCH_BYTES 64 2423 2424 void 2425 add_physmem_cb(page_t *pp, pfn_t pnum) 2426 { 2427 extern void prefetch_page_w(void *); 2428 2429 pp->p_pagenum = pnum; 2430 2431 /* 2432 * Prefetch one more page_t into E$. To prevent future 2433 * mishaps with the sizeof(page_t) changing on us, we 2434 * catch this on debug kernels if we can't bring in the 2435 * entire hpage with 2 PREFETCH_BYTES reads. See 2436 * also, sun4u/cpu/cpu_module.c 2437 */ 2438 /*LINTED*/ 2439 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2440 prefetch_page_w((char *)pp); 2441 } 2442 2443 /* 2444 * kphysm_init() tackles the problem of initializing physical memory. 2445 * The old startup made some assumptions about the kernel living in 2446 * physically contiguous space which is no longer valid. 2447 */ 2448 static void 2449 kphysm_init(page_t *pp, struct memseg *memsegp, pgcnt_t npages, 2450 uintptr_t kpm_pp, pgcnt_t kpm_npages) 2451 { 2452 struct memlist *pmem; 2453 struct memseg *msp; 2454 pfn_t base; 2455 pgcnt_t num; 2456 pfn_t lastseg_pages_end = 0; 2457 pgcnt_t nelem_used = 0; 2458 2459 ASSERT(page_hash != NULL && page_hashsz != 0); 2460 2461 msp = memsegp; 2462 for (pmem = phys_avail; pmem && npages; pmem = pmem->next) { 2463 2464 /* 2465 * Build the memsegs entry 2466 */ 2467 num = btop(pmem->size); 2468 if (num > npages) 2469 num = npages; 2470 npages -= num; 2471 base = btop(pmem->address); 2472 2473 msp->pages = pp; 2474 msp->epages = pp + num; 2475 msp->pages_base = base; 2476 msp->pages_end = base + num; 2477 2478 if (kpm_enable) { 2479 pfn_t pbase_a; 2480 pfn_t pend_a; 2481 pfn_t prev_pend_a; 2482 pgcnt_t nelem; 2483 2484 msp->pagespa = va_to_pa(pp); 2485 msp->epagespa = va_to_pa(pp + num); 2486 pbase_a = kpmptop(ptokpmp(base)); 2487 pend_a = kpmptop(ptokpmp(base + num - 1)) + kpmpnpgs; 2488 nelem = ptokpmp(pend_a - pbase_a); 2489 msp->kpm_nkpmpgs = nelem; 2490 msp->kpm_pbase = pbase_a; 2491 if (lastseg_pages_end) { 2492 /* 2493 * Assume phys_avail is in ascending order 2494 * of physical addresses. 2495 */ 2496 ASSERT(base + num > lastseg_pages_end); 2497 prev_pend_a = kpmptop( 2498 ptokpmp(lastseg_pages_end - 1)) + kpmpnpgs; 2499 2500 if (prev_pend_a > pbase_a) { 2501 /* 2502 * Overlap, more than one memseg may 2503 * point to the same kpm_page range. 2504 */ 2505 if (kpm_smallpages == 0) { 2506 msp->kpm_pages = 2507 (kpm_page_t *)kpm_pp - 1; 2508 kpm_pp = (uintptr_t) 2509 ((kpm_page_t *)kpm_pp 2510 + nelem - 1); 2511 } else { 2512 msp->kpm_spages = 2513 (kpm_spage_t *)kpm_pp - 1; 2514 kpm_pp = (uintptr_t) 2515 ((kpm_spage_t *)kpm_pp 2516 + nelem - 1); 2517 } 2518 nelem_used += nelem - 1; 2519 2520 } else { 2521 if (kpm_smallpages == 0) { 2522 msp->kpm_pages = 2523 (kpm_page_t *)kpm_pp; 2524 kpm_pp = (uintptr_t) 2525 ((kpm_page_t *)kpm_pp 2526 + nelem); 2527 } else { 2528 msp->kpm_spages = 2529 (kpm_spage_t *)kpm_pp; 2530 kpm_pp = (uintptr_t) 2531 ((kpm_spage_t *) 2532 kpm_pp + nelem); 2533 } 2534 nelem_used += nelem; 2535 } 2536 2537 } else { 2538 if (kpm_smallpages == 0) { 2539 msp->kpm_pages = (kpm_page_t *)kpm_pp; 2540 kpm_pp = (uintptr_t) 2541 ((kpm_page_t *)kpm_pp + nelem); 2542 } else { 2543 msp->kpm_spages = (kpm_spage_t *)kpm_pp; 2544 kpm_pp = (uintptr_t) 2545 ((kpm_spage_t *)kpm_pp + nelem); 2546 } 2547 nelem_used = nelem; 2548 } 2549 2550 if (nelem_used > kpm_npages) 2551 panic("kphysm_init: kpm_pp overflow\n"); 2552 2553 msp->kpm_pagespa = va_to_pa(msp->kpm_pages); 2554 lastseg_pages_end = msp->pages_end; 2555 } 2556 2557 memseg_list_add(msp); 2558 2559 /* 2560 * add_physmem() initializes the PSM part of the page 2561 * struct by calling the PSM back with add_physmem_cb(). 2562 * In addition it coalesces pages into larger pages as 2563 * it initializes them. 2564 */ 2565 add_physmem(pp, num, base); 2566 pp += num; 2567 msp++; 2568 } 2569 2570 build_pfn_hash(); 2571 } 2572 2573 /* 2574 * Kernel VM initialization. 2575 * Assumptions about kernel address space ordering: 2576 * (1) gap (user space) 2577 * (2) kernel text 2578 * (3) kernel data/bss 2579 * (4) gap 2580 * (5) kernel data structures 2581 * (6) gap 2582 * (7) debugger (optional) 2583 * (8) monitor 2584 * (9) gap (possibly null) 2585 * (10) dvma 2586 * (11) devices 2587 */ 2588 static void 2589 kvm_init(void) 2590 { 2591 /* 2592 * Put the kernel segments in kernel address space. 2593 */ 2594 rw_enter(&kas.a_lock, RW_WRITER); 2595 as_avlinit(&kas); 2596 2597 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2598 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2599 (void) segkmem_create(&ktextseg); 2600 2601 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2602 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2603 (void) segkmem_create(&ktexthole); 2604 2605 (void) seg_attach(&kas, (caddr_t)valloc_base, 2606 (size_t)(econtig32 - valloc_base), &kvalloc); 2607 (void) segkmem_create(&kvalloc); 2608 2609 if (kmem64_base) { 2610 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2611 (size_t)(kmem64_end - kmem64_base), &kmem64); 2612 (void) segkmem_create(&kmem64); 2613 } 2614 2615 /* 2616 * We're about to map out /boot. This is the beginning of the 2617 * system resource management transition. We can no longer 2618 * call into /boot for I/O or memory allocations. 2619 */ 2620 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2621 (void) segkmem_create(&kvseg); 2622 hblk_alloc_dynamic = 1; 2623 2624 /* 2625 * we need to preallocate pages for DR operations before enabling large 2626 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2627 */ 2628 memseg_remap_init(); 2629 2630 /* at this point we are ready to use large page heap */ 2631 segkmem_heap_lp_init(); 2632 2633 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2634 &kvseg32); 2635 (void) segkmem_create(&kvseg32); 2636 2637 /* 2638 * Create a segment for the debugger. 2639 */ 2640 (void) seg_attach(&kas, (caddr_t)SEGDEBUGBASE, (size_t)SEGDEBUGSIZE, 2641 &kdebugseg); 2642 (void) segkmem_create(&kdebugseg); 2643 2644 rw_exit(&kas.a_lock); 2645 } 2646 2647 char obp_tte_str[] = 2648 "h# %x constant MMU_PAGESHIFT " 2649 "h# %x constant TTE8K " 2650 "h# %x constant SFHME_SIZE " 2651 "h# %x constant SFHME_TTE " 2652 "h# %x constant HMEBLK_TAG " 2653 "h# %x constant HMEBLK_NEXT " 2654 "h# %x constant HMEBLK_MISC " 2655 "h# %x constant HMEBLK_HME1 " 2656 "h# %x constant NHMENTS " 2657 "h# %x constant HBLK_SZMASK " 2658 "h# %x constant HBLK_RANGE_SHIFT " 2659 "h# %x constant HMEBP_HBLK " 2660 "h# %x constant HMEBUCKET_SIZE " 2661 "h# %x constant HTAG_SFMMUPSZ " 2662 "h# %x constant HTAG_REHASHSZ " 2663 "h# %x constant mmu_hashcnt " 2664 "h# %p constant uhme_hash " 2665 "h# %p constant khme_hash " 2666 "h# %x constant UHMEHASH_SZ " 2667 "h# %x constant KHMEHASH_SZ " 2668 "h# %p constant KHATID " 2669 "h# %x constant CTX_SIZE " 2670 "h# %x constant CTX_SFMMU " 2671 "h# %p constant ctxs " 2672 "h# %x constant ASI_MEM " 2673 2674 ": PHYS-X@ ( phys -- data ) " 2675 " ASI_MEM spacex@ " 2676 "; " 2677 2678 ": PHYS-W@ ( phys -- data ) " 2679 " ASI_MEM spacew@ " 2680 "; " 2681 2682 ": PHYS-L@ ( phys -- data ) " 2683 " ASI_MEM spaceL@ " 2684 "; " 2685 2686 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2687 " 3 * MMU_PAGESHIFT + " 2688 "; " 2689 2690 ": TTE_IS_VALID ( ttep -- flag ) " 2691 " PHYS-X@ 0< " 2692 "; " 2693 2694 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2695 " dup TTE8K = if " 2696 " drop HBLK_RANGE_SHIFT " 2697 " else " 2698 " TTE_PAGE_SHIFT " 2699 " then " 2700 "; " 2701 2702 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2703 " tuck >> swap MMU_PAGESHIFT - << " 2704 "; " 2705 2706 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2707 " >> over xor swap ( hash sfmmup ) " 2708 " KHATID <> if ( hash ) " 2709 " UHMEHASH_SZ and ( bucket ) " 2710 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2711 " else ( hash ) " 2712 " KHMEHASH_SZ and ( bucket ) " 2713 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2714 " then ( hmebp ) " 2715 "; " 2716 2717 ": HME_HASH_TABLE_SEARCH " 2718 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2719 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2720 " dup if ( sfmmup hmeblkp ) ( r: hblktag ) " 2721 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2722 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2723 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2724 " else " 2725 " hmeblk_next + phys-x@ false " 2726 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2727 " then " 2728 " else " 2729 " hmeblk_next + phys-x@ false " 2730 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2731 " then " 2732 " else " 2733 " true " 2734 " then " 2735 " until r> drop " 2736 "; " 2737 2738 ": CNUM_TO_SFMMUP ( cnum -- sfmmup ) " 2739 " CTX_SIZE * ctxs + CTX_SFMMU + " 2740 "x@ " 2741 "; " 2742 2743 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2744 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2745 " HTAG_REHASHSZ << or nip ( hblktag ) " 2746 "; " 2747 2748 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2749 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2750 " TTE8K = if ( hmeblkp addr ) " 2751 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2752 " else ( hmeblkp addr ) " 2753 " drop 0 ( hmeblkp 0 ) " 2754 " then ( hmeblkp hme-index ) " 2755 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2756 " SFHME_TTE + ( ttep ) " 2757 "; " 2758 2759 ": unix-tte ( addr cnum -- false | tte-data true ) " 2760 " CNUM_TO_SFMMUP ( addr sfmmup ) " 2761 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2762 " 2dup swap i HME_HASH_SHIFT " 2763 "( addr sfmmup sfmmup addr hmeshift ) " 2764 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2765 " over i 4 pick " 2766 "( addr sfmmup hmebp sfmmup rehash addr ) " 2767 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2768 " HME_HASH_TABLE_SEARCH " 2769 "( addr sfmmup { null | hmeblkp } ) " 2770 " ?dup if ( addr sfmmup hmeblkp ) " 2771 " nip swap HBLK_TO_TTEP ( ttep ) " 2772 " dup TTE_IS_VALID if ( valid-ttep ) " 2773 " PHYS-X@ true ( tte-data true ) " 2774 " else ( invalid-tte ) " 2775 " drop false ( false ) " 2776 " then ( false | tte-data true ) " 2777 " unloop exit ( false | tte-data true ) " 2778 " then ( addr sfmmup ) " 2779 " loop ( addr sfmmup ) " 2780 " 2drop false ( false ) " 2781 "; " 2782 ; 2783 2784 void 2785 create_va_to_tte(void) 2786 { 2787 char *bp; 2788 extern int khmehash_num, uhmehash_num; 2789 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2790 2791 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2792 2793 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2794 2795 /* 2796 * Teach obp how to parse our sw ttes. 2797 */ 2798 (void) sprintf(bp, obp_tte_str, 2799 MMU_PAGESHIFT, 2800 TTE8K, 2801 sizeof (struct sf_hment), 2802 OFFSET(struct sf_hment, hme_tte), 2803 OFFSET(struct hme_blk, hblk_tag), 2804 OFFSET(struct hme_blk, hblk_nextpa), 2805 OFFSET(struct hme_blk, hblk_misc), 2806 OFFSET(struct hme_blk, hblk_hme), 2807 NHMENTS, 2808 HBLK_SZMASK, 2809 HBLK_RANGE_SHIFT, 2810 OFFSET(struct hmehash_bucket, hmeh_nextpa), 2811 sizeof (struct hmehash_bucket), 2812 HTAG_SFMMUPSZ, 2813 HTAG_REHASHSZ, 2814 mmu_hashcnt, 2815 (caddr_t)va_to_pa((caddr_t)uhme_hash), 2816 (caddr_t)va_to_pa((caddr_t)khme_hash), 2817 UHMEHASH_SZ, 2818 KHMEHASH_SZ, 2819 KHATID, 2820 sizeof (struct ctx), 2821 OFFSET(struct ctx, ctx_sfmmu), 2822 ctxs, 2823 ASI_MEM); 2824 prom_interpret(bp, 0, 0, 0, 0, 0); 2825 2826 kobj_free(bp, MMU_PAGESIZE); 2827 } 2828 2829 void 2830 install_va_to_tte(void) 2831 { 2832 /* 2833 * advise prom that he can use unix-tte 2834 */ 2835 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 2836 } 2837 2838 2839 static char *create_node = 2840 "root-device " 2841 "new-device " 2842 "\" os-io\" device-name " 2843 ": cb-r/w ( adr,len method$ -- #read/#written ) " 2844 " 2>r swap 2 2r> ['] $callback catch if " 2845 " 2drop 3drop 0 " 2846 " then " 2847 "; " 2848 ": read ( adr,len -- #read ) " 2849 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 2850 " ( retN ... ret1 N ) " 2851 " ?dup if " 2852 " swap >r 1- 0 ?do drop loop r> " 2853 " else " 2854 " -2 " 2855 " then l->n " 2856 "; " 2857 ": write ( adr,len -- #written ) " 2858 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 2859 " ( retN ... ret1 N ) " 2860 " ?dup if " 2861 " swap >r 1- 0 ?do drop loop r> " 2862 " else " 2863 " 0 " 2864 " then " 2865 "; " 2866 ": poll-tty ( -- ) ; " 2867 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 2868 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 2869 ": cb-give/take ( $method -- ) " 2870 " 0 -rot ['] $callback catch ?dup if " 2871 " >r 2drop 2drop r> throw " 2872 " else " 2873 " 0 ?do drop loop " 2874 " then " 2875 "; " 2876 ": give ( -- ) \" exit-input\" cb-give/take ; " 2877 ": take ( -- ) \" enter-input\" cb-give/take ; " 2878 ": open ( -- ok? ) true ; " 2879 ": close ( -- ) ; " 2880 "finish-device " 2881 "device-end "; 2882 2883 /* 2884 * Create the obp input/output node only if the USB keyboard is the 2885 * standard input device. When the USB software takes over the 2886 * input device at the time consconfig runs, it will switch OBP's 2887 * notion of the input device to this node. Whenever the 2888 * forth user interface is used after this switch, the node will 2889 * call back into the kernel for console input. 2890 * 2891 * This callback mechanism is currently only used when the USB keyboard 2892 * is the input device. If a serial device such as ttya or 2893 * a UART with a Type 5 keyboard attached is used, obp takes over the 2894 * serial device when the system goes to the debugger after the system is 2895 * booted. This sharing of the relatively simple serial device is difficult 2896 * but possible. Sharing the USB host controller is impossible due 2897 * its complexity 2898 */ 2899 static void 2900 startup_create_input_node(void) 2901 { 2902 char *stdin_path; 2903 2904 /* 2905 * If usb_node_debug is set in /etc/system 2906 * then the user would like to test the callbacks 2907 * from the input node regardless of whether or 2908 * not the USB keyboard is the console input. 2909 * This variable is useful for debugging. 2910 */ 2911 if (usb_node_debug) { 2912 2913 prom_interpret(create_node, 0, 0, 0, 0, 0); 2914 2915 return; 2916 } 2917 2918 /* Obtain the console input device */ 2919 stdin_path = prom_stdinpath(); 2920 2921 /* 2922 * If the string "usb" and "keyboard" are in the path 2923 * then a USB keyboard is the console input device, 2924 * create the node. 2925 */ 2926 if ((strstr(stdin_path, "usb") != 0) && 2927 (strstr(stdin_path, "keyboard") != 0)) { 2928 2929 prom_interpret(create_node, 0, 0, 0, 0, 0); 2930 } 2931 } 2932 2933 2934 static void 2935 do_prom_version_check(void) 2936 { 2937 int i; 2938 pnode_t node; 2939 char buf[64]; 2940 static char drev[] = "Down-rev firmware detected%s\n" 2941 "\tPlease upgrade to the following minimum version:\n" 2942 "\t\t%s\n"; 2943 2944 i = prom_version_check(buf, sizeof (buf), &node); 2945 2946 if (i == PROM_VER64_OK) 2947 return; 2948 2949 if (i == PROM_VER64_UPGRADE) { 2950 cmn_err(CE_WARN, drev, "", buf); 2951 2952 #ifdef DEBUG 2953 prom_enter_mon(); /* Type 'go' to continue */ 2954 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 2955 return; 2956 #else 2957 halt(0); 2958 #endif 2959 } 2960 2961 /* 2962 * The other possibility is that this is a server running 2963 * good firmware, but down-rev firmware was detected on at 2964 * least one other cpu board. We just complain if we see 2965 * that. 2966 */ 2967 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 2968 } 2969 2970 static void 2971 kpm_init() 2972 { 2973 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 2974 kpm_pgsz = 1ull << kpm_pgshft; 2975 kpm_pgoff = kpm_pgsz - 1; 2976 kpmp2pshft = kpm_pgshft - PAGESHIFT; 2977 kpmpnpgs = 1 << kpmp2pshft; 2978 ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0); 2979 } 2980 2981 void 2982 kpm_npages_setup(int memblocks) 2983 { 2984 /* 2985 * npages can be scattered in a maximum of 'memblocks' 2986 */ 2987 kpm_npages = ptokpmpr(npages) + memblocks; 2988 } 2989 2990 /* 2991 * Must be defined in platform dependent code. 2992 */ 2993 extern caddr_t modtext; 2994 extern size_t modtext_sz; 2995 extern caddr_t moddata; 2996 2997 #define HEAPTEXT_ARENA(addr) \ 2998 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 2999 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3000 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3001 3002 #define HEAPTEXT_OVERSIZED(addr) \ 3003 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3004 3005 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3006 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3007 kmutex_t texthole_lock; 3008 3009 char kern_bootargs[OBP_MAXPATHLEN]; 3010 3011 void 3012 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3013 { 3014 uintptr_t addr, limit; 3015 3016 addr = HEAPTEXT_BASE; 3017 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3018 3019 /* 3020 * Before we initialize the text_arena, we want to punch holes in the 3021 * underlying heaptext_arena. This guarantees that for any text 3022 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3023 */ 3024 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3025 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3026 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3027 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3028 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3029 } 3030 3031 /* 3032 * Allocate one page at the oversize to break up the text region 3033 * from the oversized region. 3034 */ 3035 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3036 (void *)limit, (void *)(limit + PAGESIZE), 3037 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3038 3039 *text_arena = vmem_create("module_text", modtext, modtext_sz, 3040 sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3041 heaptext_arena, 0, VM_SLEEP); 3042 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3043 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3044 } 3045 3046 caddr_t 3047 kobj_text_alloc(vmem_t *arena, size_t size) 3048 { 3049 caddr_t rval, better; 3050 3051 /* 3052 * First, try a sleeping allocation. 3053 */ 3054 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3055 3056 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3057 return (rval); 3058 3059 /* 3060 * We didn't get the area that we wanted. We're going to try to do an 3061 * allocation with explicit constraints. 3062 */ 3063 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3064 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3065 VM_NOSLEEP | VM_BESTFIT); 3066 3067 if (better != NULL) { 3068 /* 3069 * That worked. Free our first attempt and return. 3070 */ 3071 vmem_free(arena, rval, size); 3072 return (better); 3073 } 3074 3075 /* 3076 * That didn't work; we'll have to return our first attempt. 3077 */ 3078 return (rval); 3079 } 3080 3081 caddr_t 3082 kobj_texthole_alloc(caddr_t addr, size_t size) 3083 { 3084 int arena = HEAPTEXT_ARENA(addr); 3085 char c[30]; 3086 uintptr_t base; 3087 3088 if (HEAPTEXT_OVERSIZED(addr)) { 3089 /* 3090 * If this is an oversized allocation, there is no text hole 3091 * available for it; return NULL. 3092 */ 3093 return (NULL); 3094 } 3095 3096 mutex_enter(&texthole_lock); 3097 3098 if (texthole_arena[arena] == NULL) { 3099 ASSERT(texthole_source[arena] == NULL); 3100 3101 if (arena == 0) { 3102 texthole_source[0] = vmem_create("module_text_holesrc", 3103 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3104 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3105 0, VM_SLEEP); 3106 } else { 3107 base = HEAPTEXT_BASE + 3108 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3109 3110 (void) snprintf(c, sizeof (c), 3111 "heaptext_holesrc_%d", arena); 3112 3113 texthole_source[arena] = vmem_create(c, (void *)base, 3114 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3115 0, VM_SLEEP); 3116 } 3117 3118 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3119 3120 texthole_arena[arena] = vmem_create(c, NULL, 0, 3121 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3122 texthole_source[arena], 0, VM_SLEEP); 3123 } 3124 3125 mutex_exit(&texthole_lock); 3126 3127 ASSERT(texthole_arena[arena] != NULL); 3128 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3129 return (vmem_alloc(texthole_arena[arena], size, 3130 VM_BESTFIT | VM_NOSLEEP)); 3131 } 3132 3133 void 3134 kobj_texthole_free(caddr_t addr, size_t size) 3135 { 3136 int arena = HEAPTEXT_ARENA(addr); 3137 3138 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3139 ASSERT(texthole_arena[arena] != NULL); 3140 vmem_free(texthole_arena[arena], addr, size); 3141 } 3142