1 /* $OpenBSD: machdep.c,v 1.38 2024/04/06 18:33:54 kettenis Exp $ */
2
3 /*
4 * Copyright (c) 2014 Patrick Wildt <patrick@blueri.se>
5 * Copyright (c) 2021 Mark Kettenis <kettenis@openbsd.org>
6 *
7 * Permission to use, copy, modify, and distribute this software for any
8 * purpose with or without fee is hereby granted, provided that the above
9 * copyright notice and this permission notice appear in all copies.
10 *
11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18 */
19
20 #include <sys/param.h>
21 #include <sys/systm.h>
22 #include <sys/sched.h>
23 #include <sys/proc.h>
24 #include <sys/sysctl.h>
25 #include <sys/reboot.h>
26 #include <sys/mount.h>
27 #include <sys/exec.h>
28 #include <sys/user.h>
29 #include <sys/conf.h>
30 #include <sys/kcore.h>
31 #include <sys/msgbuf.h>
32 #include <sys/buf.h>
33 #include <sys/sensors.h>
34 #include <sys/malloc.h>
35 #include <sys/syscallargs.h>
36
37 #include <net/if.h>
38 #include <uvm/uvm.h>
39 #include <dev/cons.h>
40 #include <dev/ofw/fdt.h>
41 #include <dev/ofw/openfirm.h>
42 #include <machine/param.h>
43 #include <machine/bootconfig.h>
44 #include <machine/bus.h>
45 #include <machine/riscv64var.h>
46 #include <machine/sbi.h>
47 #include <machine/sysarch.h>
48
49 #include <machine/db_machdep.h>
50 #include <ddb/db_extern.h>
51
52 #include <dev/efi/efi.h>
53
54 #include "softraid.h"
55 #if NSOFTRAID > 0
56 #include <dev/softraidvar.h>
57 #endif
58
59 extern vaddr_t virtual_avail;
60 extern uint64_t esym;
61
62 extern char _start[];
63
64 char *boot_args = NULL;
65 uint8_t *bootmac = NULL;
66
67 int stdout_node;
68 int stdout_speed;
69
70 void (*cpuresetfn)(void);
71 void (*powerdownfn)(void);
72
73 int cold = 1;
74
75 struct vm_map *exec_map = NULL;
76 struct vm_map *phys_map = NULL;
77
78 int physmem;
79
80 caddr_t msgbufaddr;
81 paddr_t msgbufphys;
82
83 struct user *proc0paddr;
84
85 struct uvm_constraint_range dma_constraint = { 0x0, (paddr_t)-1 };
86 struct uvm_constraint_range *uvm_md_constraints[] = {
87 &dma_constraint,
88 NULL,
89 };
90
91 /* the following is used externally (sysctl_hw) */
92 char machine[] = MACHINE; /* from <machine/param.h> */
93
94 int safepri = 0;
95
96 uint32_t boot_hart; /* The hart we booted on. */
97 struct cpu_info cpu_info_primary;
98 struct cpu_info *cpu_info[MAXCPUS] = { &cpu_info_primary };
99
100 uint64_t tb_freq = 1000000;
101
102 struct fdt_reg memreg[VM_PHYSSEG_MAX];
103 int nmemreg;
104
105 void memreg_add(const struct fdt_reg *);
106 void memreg_remove(const struct fdt_reg *);
107
108 static int
atoi(const char * s)109 atoi(const char *s)
110 {
111 int n, neg;
112
113 n = 0;
114 neg = 0;
115
116 while (*s == '-') {
117 s++;
118 neg = !neg;
119 }
120
121 while (*s != '\0') {
122 if (*s < '0' || *s > '9')
123 break;
124
125 n = (10 * n) + (*s - '0');
126 s++;
127 }
128
129 return (neg ? -n : n);
130 }
131
132 void *
fdt_find_cons(const char * name)133 fdt_find_cons(const char *name)
134 {
135 char *alias = "serial0";
136 char buf[128];
137 char *stdout = NULL;
138 char *p;
139 void *node;
140
141 /* First check if "stdout-path" is set. */
142 node = fdt_find_node("/chosen");
143 if (node) {
144 if (fdt_node_property(node, "stdout-path", &stdout) > 0) {
145 if (strchr(stdout, ':') != NULL) {
146 strlcpy(buf, stdout, sizeof(buf));
147 if ((p = strchr(buf, ':')) != NULL) {
148 *p++ = '\0';
149 stdout_speed = atoi(p);
150 }
151 stdout = buf;
152 }
153 if (stdout[0] != '/') {
154 /* It's an alias. */
155 alias = stdout;
156 stdout = NULL;
157 }
158 }
159 }
160
161 /* Perform alias lookup if necessary. */
162 if (stdout == NULL) {
163 node = fdt_find_node("/aliases");
164 if (node)
165 fdt_node_property(node, alias, &stdout);
166 }
167
168 /* Lookup the physical address of the interface. */
169 if (stdout) {
170 node = fdt_find_node(stdout);
171 if (node && fdt_is_compatible(node, name)) {
172 stdout_node = OF_finddevice(stdout);
173 return (node);
174 }
175 }
176
177 return (NULL);
178 }
179
180 void com_fdt_init_cons(void);
181 void sfuart_init_cons(void);
182
183 void
consinit(void)184 consinit(void)
185 {
186 static int consinit_called = 0;
187
188 if (consinit_called != 0)
189 return;
190
191 consinit_called = 1;
192
193 com_fdt_init_cons();
194 sfuart_init_cons();
195 }
196
197 void
cpu_idle_cycle(void)198 cpu_idle_cycle(void)
199 {
200 // Enable interrupts
201 intr_enable();
202 // XXX Data Sync Barrier? (Maybe SFENCE???)
203 __asm volatile("wfi");
204 }
205
206 /* Dummy trapframe for proc0. */
207 struct trapframe proc0tf;
208
209 void
cpu_startup(void)210 cpu_startup(void)
211 {
212 u_int loop;
213 paddr_t minaddr;
214 paddr_t maxaddr;
215
216 proc0.p_addr = proc0paddr;
217
218 /*
219 * Give pmap a chance to set up a few more things now the vm
220 * is initialised
221 */
222 pmap_postinit();
223
224 /*
225 * Initialize error message buffer (at end of core).
226 */
227
228 /* msgbufphys was setup during the secondary boot strap */
229 for (loop = 0; loop < atop(MSGBUFSIZE); ++loop)
230 pmap_kenter_pa((vaddr_t)msgbufaddr + loop * PAGE_SIZE,
231 msgbufphys + loop * PAGE_SIZE, PROT_READ | PROT_WRITE);
232 pmap_update(pmap_kernel());
233 initmsgbuf(msgbufaddr, round_page(MSGBUFSIZE));
234
235 /*
236 * Identify ourselves for the msgbuf (everything printed earlier will
237 * not be buffered).
238 */
239 printf("%s", version);
240
241 printf("real mem = %lu (%luMB)\n", ptoa(physmem),
242 ptoa(physmem) / 1024 / 1024);
243
244 /*
245 * Allocate a submap for exec arguments. This map effectively
246 * limits the number of processes exec'ing at any time.
247 */
248 minaddr = vm_map_min(kernel_map);
249 exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
250 16 * NCARGS, VM_MAP_PAGEABLE, FALSE, NULL);
251
252 /*
253 * Allocate a submap for physio
254 */
255 phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
256 VM_PHYS_SIZE, 0, FALSE, NULL);
257
258 /*
259 * Set up buffers, so they can be used to read disk labels.
260 */
261 bufinit();
262
263 printf("avail mem = %lu (%luMB)\n", ptoa(uvmexp.free),
264 ptoa(uvmexp.free) / 1024 / 1024);
265
266 sbi_print_version();
267
268 curpcb = &proc0.p_addr->u_pcb;
269 curpcb->pcb_flags = 0;
270 curpcb->pcb_tf = &proc0tf;
271
272 if (boothowto & RB_CONFIG) {
273 #ifdef BOOT_CONFIG
274 user_config();
275 #else
276 printf("kernel does not support -c; continuing..\n");
277 #endif
278 }
279 }
280
281 /*
282 * Move parts of cpu_switchto into C, too difficult in asm
283 */
284
285 void cpu_switchto_asm(struct proc *, struct proc *);
286
287 void
cpu_switchto(struct proc * old,struct proc * new)288 cpu_switchto(struct proc *old, struct proc *new)
289 {
290 if (old) {
291 struct pcb *pcb = &old->p_addr->u_pcb;
292 struct trapframe *tf = pcb->pcb_tf;
293
294 if (pcb->pcb_flags & PCB_FPU)
295 fpu_save(old, tf);
296
297 /* drop FPU state */
298 tf->tf_sstatus &= ~SSTATUS_FS_MASK;
299 tf->tf_sstatus |= SSTATUS_FS_OFF;
300 }
301
302 cpu_switchto_asm(old, new);
303 }
304
305 /*
306 * machine dependent system variables.
307 */
308
309 int
cpu_sysctl(int * name,u_int namelen,void * oldp,size_t * oldlenp,void * newp,size_t newlen,struct proc * p)310 cpu_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp,
311 size_t newlen, struct proc *p)
312 {
313 char *compatible;
314 int node, len, error;
315
316 /* all sysctl names at this level are terminal */
317 if (namelen != 1)
318 return (ENOTDIR); /* overloaded */
319
320 switch (name[0]) {
321 case CPU_COMPATIBLE:
322 node = OF_finddevice("/");
323 len = OF_getproplen(node, "compatible");
324 if (len <= 0)
325 return (EOPNOTSUPP);
326 compatible = malloc(len, M_TEMP, M_WAITOK | M_ZERO);
327 OF_getprop(node, "compatible", compatible, len);
328 compatible[len - 1] = 0;
329 error = sysctl_rdstring(oldp, oldlenp, newp, compatible);
330 free(compatible, M_TEMP, len);
331 return error;
332 default:
333 return (EOPNOTSUPP);
334 }
335 /* NOTREACHED */
336 }
337
338 int waittime = -1;
339
340 __dead void
boot(int howto)341 boot(int howto)
342 {
343 if ((howto & RB_RESET) != 0)
344 goto doreset;
345
346 if (cold) {
347 if ((howto & RB_USERREQ) == 0)
348 howto |= RB_HALT;
349 goto haltsys;
350 }
351
352 boothowto = howto;
353 if ((howto & RB_NOSYNC) == 0 && waittime < 0) {
354 waittime = 0;
355 vfs_shutdown(curproc);
356
357 if ((howto & RB_TIMEBAD) == 0) {
358 resettodr();
359 } else {
360 printf("WARNING: not updating battery clock\n");
361 }
362 }
363 if_downall();
364
365 uvm_shutdown();
366 splhigh();
367 cold = 1;
368
369 if ((howto & RB_DUMP) != 0)
370 printf("no dump so far\n");
371
372 haltsys:
373 config_suspend_all(DVACT_POWERDOWN);
374
375 if ((howto & RB_HALT) != 0) {
376 if ((howto & RB_POWERDOWN) != 0) {
377 printf("\nAttempting to power down...\n");
378 delay(500000);
379 if (powerdownfn)
380 (*powerdownfn)();
381 }
382
383 printf("\n");
384 printf("The operating system has halted.\n");
385 printf("Please press any key to reboot.\n\n");
386 cngetc();
387 }
388
389 doreset:
390 printf("rebooting...\n");
391 delay(500000);
392 if (cpuresetfn)
393 (*cpuresetfn)();
394 printf("reboot failed; spinning\n");
395 for (;;)
396 continue;
397 /* NOTREACHED */
398 }
399
400 void
setregs(struct proc * p,struct exec_package * pack,u_long stack,struct ps_strings * arginfo)401 setregs(struct proc *p, struct exec_package *pack, u_long stack,
402 struct ps_strings *arginfo)
403 {
404 struct trapframe *tf = p->p_addr->u_pcb.pcb_tf;
405 struct pcb *pcb = &p->p_addr->u_pcb;
406
407 /* If we were using the FPU, forget about it. */
408 pcb->pcb_flags &= ~PCB_FPU;
409 tf->tf_sstatus &= ~SSTATUS_FS_MASK;
410 tf->tf_sstatus |= SSTATUS_FS_OFF;
411
412 memset(tf, 0, sizeof *tf);
413 tf->tf_sp = STACKALIGN(stack);
414 tf->tf_ra = pack->ep_entry;
415 tf->tf_sepc = pack->ep_entry;
416 }
417
418 void
need_resched(struct cpu_info * ci)419 need_resched(struct cpu_info *ci)
420 {
421 ci->ci_want_resched = 1;
422
423 /* There's a risk we'll be called before the idle threads start */
424 if (ci->ci_curproc) {
425 aston(ci->ci_curproc);
426 cpu_kick(ci);
427 }
428 }
429
430
431 /*
432 * Size of memory segments, before any memory is stolen.
433 */
434 phys_ram_seg_t mem_clusters[VM_PHYSSEG_MAX];
435 int mem_cluster_cnt;
436
437 /*
438 * cpu_dumpsize: calculate size of machine-dependent kernel core dump headers.
439 */
440 int
cpu_dumpsize(void)441 cpu_dumpsize(void)
442 {
443 int size;
444
445 size = ALIGN(sizeof(kcore_seg_t)) +
446 ALIGN(mem_cluster_cnt * sizeof(phys_ram_seg_t));
447 if (roundup(size, dbtob(1)) != dbtob(1))
448 return (-1);
449
450 return (1);
451 }
452
453 u_long
cpu_dump_mempagecnt(void)454 cpu_dump_mempagecnt(void)
455 {
456 return 0;
457 }
458
459 /*
460 * These variables are needed by /sbin/savecore
461 */
462 u_long dumpmag = 0x8fca0101; /* magic number */
463 int dumpsize = 0; /* pages */
464 long dumplo = 0; /* blocks */
465
466 /*
467 * This is called by main to set dumplo and dumpsize.
468 * Dumps always skip the first PAGE_SIZE of disk space
469 * in case there might be a disk label stored there.
470 * If there is extra space, put dump at the end to
471 * reduce the chance that swapping trashes it.
472 */
473 void
dumpconf(void)474 dumpconf(void)
475 {
476 int nblks, dumpblks; /* size of dump area */
477
478 if (dumpdev == NODEV ||
479 (nblks = (bdevsw[major(dumpdev)].d_psize)(dumpdev)) == 0)
480 return;
481 if (nblks <= ctod(1))
482 return;
483
484 dumpblks = cpu_dumpsize();
485 if (dumpblks < 0)
486 return;
487 dumpblks += ctod(cpu_dump_mempagecnt());
488
489 /* If dump won't fit (incl. room for possible label), punt. */
490 if (dumpblks > (nblks - ctod(1)))
491 return;
492
493 /* Put dump at end of partition */
494 dumplo = nblks - dumpblks;
495
496 /* dumpsize is in page units, and doesn't include headers. */
497 dumpsize = cpu_dump_mempagecnt();
498 }
499
500 int
sys_sysarch(struct proc * p,void * v,register_t * retval)501 sys_sysarch(struct proc *p, void *v, register_t *retval)
502 {
503 struct sys_sysarch_args /* {
504 syscallarg(int) op;
505 syscallarg(void *) parms;
506 } */ *uap = v;
507 struct riscv_sync_icache_args args;
508 int error = 0;
509
510 switch (SCARG(uap, op)) {
511 case RISCV_SYNC_ICACHE:
512 if (SCARG(uap, parms) != NULL)
513 error = copyin(SCARG(uap, parms), &args, sizeof(args));
514 if (error)
515 break;
516 /*
517 * XXX Validate args.addr and args.len before using them.
518 */
519 pmap_proc_iflush(p->p_p, (vaddr_t)args.addr, args.len);
520 break;
521 default:
522 error = EINVAL;
523 break;
524 }
525
526 return (error);
527 }
528
529 uint64_t mmap_start;
530 uint32_t mmap_size;
531 uint32_t mmap_desc_size;
532 uint32_t mmap_desc_ver;
533
534 EFI_MEMORY_DESCRIPTOR *mmap;
535
536 void collect_kernel_args(const char *);
537 void process_kernel_args(void);
538
539 int pmap_bootstrap_bs_map(bus_space_tag_t, bus_addr_t,
540 bus_size_t, int, bus_space_handle_t *);
541
542 void
initriscv(struct riscv_bootparams * rbp)543 initriscv(struct riscv_bootparams *rbp)
544 {
545 paddr_t memstart, memend;
546 paddr_t startpa, endpa, pa;
547 vaddr_t vstart, va;
548 struct fdt_head *fh;
549 void *config = (void *)rbp->dtbp_phys;
550 void *fdt = NULL;
551 struct fdt_reg reg;
552 void *node;
553 EFI_PHYSICAL_ADDRESS system_table = 0;
554 int (*map_func_save)(bus_space_tag_t, bus_addr_t, bus_size_t, int,
555 bus_space_handle_t *);
556 int i;
557
558 /* Set the per-CPU pointer. */
559 __asm volatile("mv tp, %0" :: "r"(&cpu_info_primary));
560
561 sbi_init();
562
563 /* The bootloader has loaded us into a 64MB block. */
564 memstart = rbp->kern_phys;
565 memend = memstart + 64 * 1024 * 1024;
566
567 /* Bootstrap enough of pmap to enter the kernel proper. */
568 vstart = pmap_bootstrap(rbp->kern_phys - KERNBASE, rbp->kern_l1pt,
569 KERNBASE, esym, memstart, memend);
570
571 /* Map the FDT header to determine its size. */
572 va = vstart;
573 startpa = trunc_page((paddr_t)config);
574 endpa = round_page((paddr_t)config + sizeof(struct fdt_head));
575 for (pa = startpa; pa < endpa; pa += PAGE_SIZE, va += PAGE_SIZE)
576 pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE, PMAP_CACHE_WB);
577 fh = (void *)(vstart + ((paddr_t)config - startpa));
578 if (betoh32(fh->fh_magic) != FDT_MAGIC || betoh32(fh->fh_size) == 0)
579 panic("%s: no FDT", __func__);
580
581 /* Map the remainder of the FDT. */
582 endpa = round_page((paddr_t)config + betoh32(fh->fh_size));
583 for (; pa < endpa; pa += PAGE_SIZE, va += PAGE_SIZE)
584 pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE, PMAP_CACHE_WB);
585 config = (void *)(vstart + ((paddr_t)config - startpa));
586 vstart = va;
587
588 if (!fdt_init(config))
589 panic("%s: corrupt FDT", __func__);
590
591 node = fdt_find_node("/cpus");
592 if (node != NULL) {
593 char *prop;
594 int len;
595
596 len = fdt_node_property(node, "timebase-frequency", &prop);
597 if (len == sizeof(uint32_t))
598 tb_freq = bemtoh32((uint32_t *)prop);
599 }
600
601 node = fdt_find_node("/chosen");
602 if (node != NULL) {
603 char *prop;
604 int len;
605 static uint8_t lladdr[6];
606
607 len = fdt_node_property(node, "boot-hartid", &prop);
608 if (len == sizeof(boot_hart))
609 boot_hart = bemtoh32((uint32_t *)prop);
610
611 len = fdt_node_property(node, "bootargs", &prop);
612 if (len > 0)
613 collect_kernel_args(prop);
614
615 len = fdt_node_property(node, "openbsd,boothowto", &prop);
616 if (len == sizeof(boothowto))
617 boothowto = bemtoh32((uint32_t *)prop);
618
619 len = fdt_node_property(node, "openbsd,bootduid", &prop);
620 if (len == sizeof(bootduid))
621 memcpy(bootduid, prop, sizeof(bootduid));
622
623 len = fdt_node_property(node, "openbsd,bootmac", &prop);
624 if (len == sizeof(lladdr)) {
625 memcpy(lladdr, prop, sizeof(lladdr));
626 bootmac = lladdr;
627 }
628
629 len = fdt_node_property(node, "openbsd,sr-bootuuid", &prop);
630 #if NSOFTRAID > 0
631 if (len == sizeof(sr_bootuuid))
632 memcpy(&sr_bootuuid, prop, sizeof(sr_bootuuid));
633 #endif
634 if (len > 0)
635 explicit_bzero(prop, len);
636
637 len = fdt_node_property(node, "openbsd,sr-bootkey", &prop);
638 #if NSOFTRAID > 0
639 if (len == sizeof(sr_bootkey))
640 memcpy(&sr_bootkey, prop, sizeof(sr_bootkey));
641 #endif
642 if (len > 0)
643 explicit_bzero(prop, len);
644
645 len = fdt_node_property(node, "openbsd,uefi-mmap-start", &prop);
646 if (len == sizeof(mmap_start))
647 mmap_start = bemtoh64((uint64_t *)prop);
648 len = fdt_node_property(node, "openbsd,uefi-mmap-size", &prop);
649 if (len == sizeof(mmap_size))
650 mmap_size = bemtoh32((uint32_t *)prop);
651 len = fdt_node_property(node, "openbsd,uefi-mmap-desc-size", &prop);
652 if (len == sizeof(mmap_desc_size))
653 mmap_desc_size = bemtoh32((uint32_t *)prop);
654 len = fdt_node_property(node, "openbsd,uefi-mmap-desc-ver", &prop);
655 if (len == sizeof(mmap_desc_ver))
656 mmap_desc_ver = bemtoh32((uint32_t *)prop);
657
658 len = fdt_node_property(node, "openbsd,uefi-system-table", &prop);
659 if (len == sizeof(system_table))
660 system_table = bemtoh64((uint64_t *)prop);
661
662 len = fdt_node_property(node, "openbsd,dma-constraint", &prop);
663 if (len == sizeof(dma_constraint)) {
664 dma_constraint.ucr_low = bemtoh64((uint64_t *)prop);
665 dma_constraint.ucr_high = bemtoh64((uint64_t *)prop + 1);
666 }
667 }
668
669 process_kernel_args();
670
671 proc0paddr = (struct user *)rbp->kern_stack;
672
673 msgbufaddr = (caddr_t)vstart;
674 msgbufphys = pmap_steal_avail(round_page(MSGBUFSIZE), PAGE_SIZE, NULL);
675 vstart += round_page(MSGBUFSIZE);
676
677 zero_page = vstart;
678 vstart += MAXCPUS * PAGE_SIZE;
679 copy_src_page = vstart;
680 vstart += MAXCPUS * PAGE_SIZE;
681 copy_dst_page = vstart;
682 vstart += MAXCPUS * PAGE_SIZE;
683
684 /* Relocate the FDT to safe memory. */
685 if (fdt_get_size(config) != 0) {
686 uint32_t csize, size = round_page(fdt_get_size(config));
687 paddr_t pa;
688 vaddr_t va;
689
690 pa = pmap_steal_avail(size, PAGE_SIZE, NULL);
691 memcpy((void *)PHYS_TO_DMAP(pa), config, size);
692 for (va = vstart, csize = size; csize > 0;
693 csize -= PAGE_SIZE, va += PAGE_SIZE, pa += PAGE_SIZE)
694 pmap_kenter_cache(va, pa, PROT_READ, PMAP_CACHE_WB);
695
696 fdt = (void *)vstart;
697 vstart += size;
698 }
699
700 /* Relocate the EFI memory map too. */
701 if (mmap_start != 0) {
702 uint32_t csize, size = round_page(mmap_size);
703 paddr_t pa, startpa, endpa;
704 vaddr_t va;
705
706 startpa = trunc_page(mmap_start);
707 endpa = round_page(mmap_start + mmap_size);
708 for (pa = startpa, va = vstart; pa < endpa;
709 pa += PAGE_SIZE, va += PAGE_SIZE)
710 pmap_kenter_cache(va, pa, PROT_READ, PMAP_CACHE_WB);
711 pa = pmap_steal_avail(size, PAGE_SIZE, NULL);
712 memcpy((void *)PHYS_TO_DMAP(pa),
713 (caddr_t)vstart + (mmap_start - startpa), mmap_size);
714 pmap_kremove(vstart, endpa - startpa);
715
716 for (va = vstart, csize = size; csize > 0;
717 csize -= PAGE_SIZE, va += PAGE_SIZE, pa += PAGE_SIZE)
718 pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE, PMAP_CACHE_WB);
719
720 mmap = (void *)vstart;
721 vstart += size;
722 }
723
724 /* No more KVA stealing after this point. */
725 virtual_avail = vstart;
726
727 /* Now we can reinit the FDT, using the virtual address. */
728 if (fdt)
729 fdt_init(fdt);
730
731 map_func_save = riscv64_bs_tag._space_map;
732 riscv64_bs_tag._space_map = pmap_bootstrap_bs_map;
733
734 consinit();
735
736 riscv64_bs_tag._space_map = map_func_save;
737
738 pmap_avail_fixup();
739
740 uvmexp.pagesize = PAGE_SIZE;
741 uvm_setpagesize();
742
743 /* Make what's left of the initial 64MB block available to UVM. */
744 pmap_physload_avail();
745
746 /* Make all other physical memory available to UVM. */
747 if (mmap && mmap_desc_ver == EFI_MEMORY_DESCRIPTOR_VERSION) {
748 EFI_MEMORY_DESCRIPTOR *desc = mmap;
749
750 /*
751 * Load all memory marked as EfiConventionalMemory,
752 * EfiBootServicesCode or EfiBootServicesData.
753 * Don't bother with blocks smaller than 64KB. The
754 * initial 64MB memory block should be marked as
755 * EfiLoaderData so it won't be added here.
756 */
757 for (i = 0; i < mmap_size / mmap_desc_size; i++) {
758 printf("type 0x%x pa 0x%llx va 0x%llx pages 0x%llx attr 0x%llx\n",
759 desc->Type, desc->PhysicalStart,
760 desc->VirtualStart, desc->NumberOfPages,
761 desc->Attribute);
762 if ((desc->Type == EfiConventionalMemory ||
763 desc->Type == EfiBootServicesCode ||
764 desc->Type == EfiBootServicesData) &&
765 desc->NumberOfPages >= 16) {
766 reg.addr = desc->PhysicalStart;
767 reg.size = ptoa(desc->NumberOfPages);
768 memreg_add(®);
769 }
770 desc = NextMemoryDescriptor(desc, mmap_desc_size);
771 }
772 } else {
773 node = fdt_find_node("/memory");
774 if (node == NULL)
775 panic("%s: no memory specified", __func__);
776
777 for (i = 0; nmemreg < nitems(memreg); i++) {
778 if (fdt_get_reg(node, i, ®))
779 break;
780 if (reg.size == 0)
781 continue;
782 memreg_add(®);
783 }
784 }
785
786 /* Remove reserved memory. */
787 node = fdt_find_node("/reserved-memory");
788 if (node) {
789 for (node = fdt_child_node(node); node;
790 node = fdt_next_node(node)) {
791 if (fdt_get_reg(node, 0, ®))
792 continue;
793 if (reg.size == 0)
794 continue;
795 memreg_remove(®);
796 }
797 }
798
799 /* Remove the initial 64MB block. */
800 reg.addr = memstart;
801 reg.size = memend - memstart;
802 memreg_remove(®);
803
804 for (i = 0; i < nmemreg; i++) {
805 paddr_t start = memreg[i].addr;
806 paddr_t end = start + memreg[i].size;
807
808 uvm_page_physload(atop(start), atop(end),
809 atop(start), atop(end), 0);
810 }
811
812 /*
813 * Determine physical RAM size from the /memory nodes in the
814 * FDT. There can be multiple nodes and each node can contain
815 * multiple ranges.
816 */
817 node = fdt_find_node("/memory");
818 if (node == NULL)
819 panic("%s: no memory specified", __func__);
820 while (node) {
821 const char *s = fdt_node_name(node);
822 if (strncmp(s, "memory", 6) == 0 &&
823 (s[6] == '\0' || s[6] == '@')) {
824 for (i = 0; i < VM_PHYSSEG_MAX; i++) {
825 if (fdt_get_reg(node, i, ®))
826 break;
827 if (reg.size == 0)
828 continue;
829 physmem += atop(reg.size);
830 }
831 }
832
833 node = fdt_next_node(node);
834 }
835
836 kmeminit_nkmempages();
837
838 /*
839 * Make sure that we have enough KVA to initialize UVM. In
840 * particular, we need enough KVA to be able to allocate the
841 * vm_page structures and nkmempages for malloc(9).
842 */
843 pmap_growkernel(VM_MIN_KERNEL_ADDRESS + 1024 * 1024 * 1024 +
844 physmem * sizeof(struct vm_page) + ptoa(nkmempages));
845
846 #ifdef DDB
847 db_machine_init();
848
849 /* Firmware doesn't load symbols. */
850 ddb_init();
851
852 if (boothowto & RB_KDB)
853 db_enter();
854 #endif
855
856 softintr_init();
857 splraise(IPL_IPI);
858 }
859
860 char bootargs[256];
861
862 void
collect_kernel_args(const char * args)863 collect_kernel_args(const char *args)
864 {
865 /* Make a local copy of the bootargs */
866 strlcpy(bootargs, args, sizeof(bootargs));
867 }
868
869 void
process_kernel_args(void)870 process_kernel_args(void)
871 {
872 char *cp = bootargs;
873
874 if (*cp == 0)
875 return;
876
877 /* Skip the kernel image filename */
878 while (*cp != ' ' && *cp != 0)
879 cp++;
880
881 if (*cp != 0)
882 *cp++ = 0;
883
884 while (*cp == ' ')
885 cp++;
886
887 boot_args = cp;
888
889 printf("bootargs: %s\n", boot_args);
890
891 /* Setup pointer to boot flags */
892 while (*cp != '-')
893 if (*cp++ == '\0')
894 return;
895
896 while (*++cp != 0) {
897 switch (*cp) {
898 case 'a':
899 boothowto |= RB_ASKNAME;
900 break;
901 case 'c':
902 boothowto |= RB_CONFIG;
903 break;
904 case 'd':
905 boothowto |= RB_KDB;
906 break;
907 case 's':
908 boothowto |= RB_SINGLE;
909 break;
910 default:
911 printf("unknown option `%c'\n", *cp);
912 break;
913 }
914 }
915 }
916
917 /*
918 * Allow bootstrap to steal KVA after machdep has given it back to pmap.
919 */
920 int
pmap_bootstrap_bs_map(bus_space_tag_t t,bus_addr_t bpa,bus_size_t size,int flags,bus_space_handle_t * bshp)921 pmap_bootstrap_bs_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size,
922 int flags, bus_space_handle_t *bshp)
923 {
924 u_long startpa, pa, endpa;
925 vaddr_t va;
926
927 va = virtual_avail; /* steal memory from virtual avail. */
928
929 startpa = trunc_page(bpa);
930 endpa = round_page((bpa + size));
931
932 *bshp = (bus_space_handle_t)(va + (bpa - startpa));
933
934 for (pa = startpa; pa < endpa; pa += PAGE_SIZE, va += PAGE_SIZE)
935 pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE,
936 PMAP_CACHE_DEV);
937
938 virtual_avail = va;
939
940 return 0;
941 }
942
943 void
memreg_add(const struct fdt_reg * reg)944 memreg_add(const struct fdt_reg *reg)
945 {
946 int i;
947
948 for (i = 0; i < nmemreg; i++) {
949 if (reg->addr == memreg[i].addr + memreg[i].size) {
950 memreg[i].size += reg->size;
951 return;
952 }
953 if (reg->addr + reg->size == memreg[i].addr) {
954 memreg[i].addr = reg->addr;
955 memreg[i].size += reg->size;
956 return;
957 }
958 }
959
960 if (nmemreg >= nitems(memreg))
961 return;
962
963 memreg[nmemreg++] = *reg;
964 }
965
966 void
memreg_remove(const struct fdt_reg * reg)967 memreg_remove(const struct fdt_reg *reg)
968 {
969 uint64_t start = reg->addr;
970 uint64_t end = reg->addr + reg->size;
971 int i, j;
972
973 for (i = 0; i < nmemreg; i++) {
974 uint64_t memstart = memreg[i].addr;
975 uint64_t memend = memreg[i].addr + memreg[i].size;
976
977 if (end <= memstart)
978 continue;
979 if (start >= memend)
980 continue;
981
982 if (start <= memstart)
983 memstart = MIN(end, memend);
984 if (end >= memend)
985 memend = MAX(start, memstart);
986
987 if (start > memstart && end < memend) {
988 if (nmemreg < nitems(memreg)) {
989 memreg[nmemreg].addr = end;
990 memreg[nmemreg].size = memend - end;
991 nmemreg++;
992 }
993 memend = start;
994 }
995 memreg[i].addr = memstart;
996 memreg[i].size = memend - memstart;
997 }
998
999 /* Remove empty slots. */
1000 for (i = nmemreg - 1; i >= 0; i--) {
1001 if (memreg[i].size == 0) {
1002 for (j = i; (j + 1) < nmemreg; j++)
1003 memreg[j] = memreg[j + 1];
1004 nmemreg--;
1005 }
1006 }
1007 }
1008