1 2 #define _SYSTEM 1 3 4 #include <minix/callnr.h> 5 #include <minix/com.h> 6 #include <minix/config.h> 7 #include <minix/const.h> 8 #include <minix/ds.h> 9 #include <minix/endpoint.h> 10 #include <minix/minlib.h> 11 #include <minix/type.h> 12 #include <minix/ipc.h> 13 #include <minix/sysutil.h> 14 #include <minix/syslib.h> 15 #include <minix/safecopies.h> 16 #include <minix/cpufeature.h> 17 #include <minix/bitmap.h> 18 #include <minix/debug.h> 19 20 #include <errno.h> 21 #include <stdlib.h> 22 #include <assert.h> 23 #include <string.h> 24 #include <stdio.h> 25 #include <fcntl.h> 26 #include <stdlib.h> 27 28 #include "proto.h" 29 #include "glo.h" 30 #include "util.h" 31 #include "vm.h" 32 #include "sanitycheck.h" 33 34 static int vm_self_pages; 35 36 /* PDE used to map in kernel, kernel physical address. */ 37 #define MAX_PAGEDIR_PDES 5 38 static struct pdm { 39 int pdeno; 40 u32_t val; 41 phys_bytes phys; 42 u32_t *page_directories; 43 } pagedir_mappings[MAX_PAGEDIR_PDES]; 44 45 static multiboot_module_t *kern_mb_mod = NULL; 46 static size_t kern_size = 0; 47 static int kern_start_pde = -1; 48 49 /* big page size available in hardware? */ 50 static int bigpage_ok = 1; 51 52 /* Our process table entry. */ 53 struct vmproc *vmprocess = &vmproc[VM_PROC_NR]; 54 55 /* Spare memory, ready to go after initialization, to avoid a 56 * circular dependency on allocating memory and writing it into VM's 57 * page table. 58 */ 59 #if SANITYCHECKS 60 #define SPAREPAGES 200 61 #define STATIC_SPAREPAGES 190 62 #else 63 #ifdef __arm__ 64 # define SPAREPAGES 150 65 # define STATIC_SPAREPAGES 140 66 #else 67 # define SPAREPAGES 20 68 # define STATIC_SPAREPAGES 15 69 #endif /* __arm__ */ 70 #endif 71 72 #ifdef __i386__ 73 static u32_t global_bit = 0; 74 #endif 75 76 #define SPAREPAGEDIRS 1 77 #define STATIC_SPAREPAGEDIRS 1 78 79 int missing_sparedirs = SPAREPAGEDIRS; 80 static struct { 81 void *pagedir; 82 phys_bytes phys; 83 } sparepagedirs[SPAREPAGEDIRS]; 84 85 #define is_staticaddr(v) ((vir_bytes) (v) < VM_OWN_HEAPSTART) 86 87 #define MAX_KERNMAPPINGS 10 88 static struct { 89 phys_bytes phys_addr; /* Physical addr. */ 90 phys_bytes len; /* Length in bytes. */ 91 vir_bytes vir_addr; /* Offset in page table. */ 92 int flags; 93 } kern_mappings[MAX_KERNMAPPINGS]; 94 int kernmappings = 0; 95 96 /* Clicks must be pages, as 97 * - they must be page aligned to map them 98 * - they must be a multiple of the page size 99 * - it's inconvenient to have them bigger than pages, because we often want 100 * just one page 101 * May as well require them to be equal then. 102 */ 103 #if CLICK_SIZE != VM_PAGE_SIZE 104 #error CLICK_SIZE must be page size. 105 #endif 106 107 static void *spare_pagequeue; 108 static char static_sparepages[VM_PAGE_SIZE*STATIC_SPAREPAGES] 109 __aligned(VM_PAGE_SIZE); 110 111 #if defined(__arm__) 112 static char static_sparepagedirs[ARCH_PAGEDIR_SIZE*STATIC_SPAREPAGEDIRS + ARCH_PAGEDIR_SIZE] __aligned(ARCH_PAGEDIR_SIZE); 113 #endif 114 115 void pt_assert(pt_t *pt) 116 { 117 char dir[4096]; 118 pt_clearmapcache(); 119 if((sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) { 120 panic("VMCTL_FLUSHTLB failed"); 121 } 122 sys_physcopy(NONE, pt->pt_dir_phys, SELF, (vir_bytes) dir, sizeof(dir), 0); 123 assert(!memcmp(dir, pt->pt_dir, sizeof(dir))); 124 } 125 126 #if SANITYCHECKS 127 /*===========================================================================* 128 * pt_sanitycheck * 129 *===========================================================================*/ 130 void pt_sanitycheck(pt_t *pt, const char *file, int line) 131 { 132 /* Basic pt sanity check. */ 133 int slot; 134 135 MYASSERT(pt); 136 MYASSERT(pt->pt_dir); 137 MYASSERT(pt->pt_dir_phys); 138 139 for(slot = 0; slot < ELEMENTS(vmproc); slot++) { 140 if(pt == &vmproc[slot].vm_pt) 141 break; 142 } 143 144 if(slot >= ELEMENTS(vmproc)) { 145 panic("pt_sanitycheck: passed pt not in any proc"); 146 } 147 148 MYASSERT(usedpages_add(pt->pt_dir_phys, VM_PAGE_SIZE) == OK); 149 } 150 #endif 151 152 /*===========================================================================* 153 * findhole * 154 *===========================================================================*/ 155 static u32_t findhole(int pages) 156 { 157 /* Find a space in the virtual address space of VM. */ 158 u32_t curv; 159 int pde = 0, try_restart; 160 static void *lastv = 0; 161 pt_t *pt = &vmprocess->vm_pt; 162 vir_bytes vmin, vmax; 163 u32_t holev = NO_MEM; 164 int holesize = -1; 165 166 vmin = VM_OWN_MMAPBASE; 167 vmax = VM_OWN_MMAPTOP; 168 169 /* Input sanity check. */ 170 assert(vmin + VM_PAGE_SIZE >= vmin); 171 assert(vmax >= vmin + VM_PAGE_SIZE); 172 assert((vmin % VM_PAGE_SIZE) == 0); 173 assert((vmax % VM_PAGE_SIZE) == 0); 174 assert(pages > 0); 175 176 curv = (u32_t) lastv; 177 if(curv < vmin || curv >= vmax) 178 curv = vmin; 179 180 try_restart = 1; 181 182 /* Start looking for a free page starting at vmin. */ 183 while(curv < vmax) { 184 int pte; 185 186 assert(curv >= vmin); 187 assert(curv < vmax); 188 189 pde = ARCH_VM_PDE(curv); 190 pte = ARCH_VM_PTE(curv); 191 192 if((pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT) && 193 (pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) { 194 /* there is a page here - so keep looking for holes */ 195 holev = NO_MEM; 196 holesize = 0; 197 } else { 198 /* there is no page here - so we have a hole, a bigger 199 * one if we already had one 200 */ 201 if(holev == NO_MEM) { 202 holev = curv; 203 holesize = 1; 204 } else holesize++; 205 206 assert(holesize > 0); 207 assert(holesize <= pages); 208 209 /* if it's big enough, return it */ 210 if(holesize == pages) { 211 lastv = (void*) (curv + VM_PAGE_SIZE); 212 return holev; 213 } 214 } 215 216 curv+=VM_PAGE_SIZE; 217 218 /* if we reached the limit, start scanning from the beginning if 219 * we haven't looked there yet 220 */ 221 if(curv >= vmax && try_restart) { 222 try_restart = 0; 223 curv = vmin; 224 } 225 } 226 227 printf("VM: out of virtual address space in vm\n"); 228 229 return NO_MEM; 230 } 231 232 /*===========================================================================* 233 * vm_freepages * 234 *===========================================================================*/ 235 void vm_freepages(vir_bytes vir, int pages) 236 { 237 assert(!(vir % VM_PAGE_SIZE)); 238 239 if(is_staticaddr(vir)) { 240 printf("VM: not freeing static page\n"); 241 return; 242 } 243 244 if(pt_writemap(vmprocess, &vmprocess->vm_pt, vir, 245 MAP_NONE, pages*VM_PAGE_SIZE, 0, 246 WMF_OVERWRITE | WMF_FREE) != OK) 247 panic("vm_freepages: pt_writemap failed"); 248 249 vm_self_pages--; 250 251 #if SANITYCHECKS 252 /* If SANITYCHECKS are on, flush tlb so accessing freed pages is 253 * always trapped, also if not in tlb. 254 */ 255 if((sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) { 256 panic("VMCTL_FLUSHTLB failed"); 257 } 258 #endif 259 } 260 261 /*===========================================================================* 262 * vm_getsparepage * 263 *===========================================================================*/ 264 static void *vm_getsparepage(phys_bytes *phys) 265 { 266 void *ptr; 267 if(reservedqueue_alloc(spare_pagequeue, phys, &ptr) != OK) { 268 return NULL; 269 } 270 assert(ptr); 271 return ptr; 272 } 273 274 /*===========================================================================* 275 * vm_getsparepagedir * 276 *===========================================================================*/ 277 static void *vm_getsparepagedir(phys_bytes *phys) 278 { 279 int s; 280 assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS); 281 for(s = 0; s < SPAREPAGEDIRS; s++) { 282 if(sparepagedirs[s].pagedir) { 283 void *sp; 284 sp = sparepagedirs[s].pagedir; 285 *phys = sparepagedirs[s].phys; 286 sparepagedirs[s].pagedir = NULL; 287 missing_sparedirs++; 288 assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS); 289 return sp; 290 } 291 } 292 return NULL; 293 } 294 295 void *vm_mappages(phys_bytes p, int pages) 296 { 297 vir_bytes loc; 298 int r; 299 pt_t *pt = &vmprocess->vm_pt; 300 301 /* Where in our virtual address space can we put it? */ 302 loc = findhole(pages); 303 if(loc == NO_MEM) { 304 printf("vm_mappages: findhole failed\n"); 305 return NULL; 306 } 307 308 /* Map this page into our address space. */ 309 if((r=pt_writemap(vmprocess, pt, loc, p, VM_PAGE_SIZE*pages, 310 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW 311 #if defined(__arm__) 312 | ARM_VM_PTE_CACHED 313 #endif 314 , 0)) != OK) { 315 printf("vm_mappages writemap failed\n"); 316 return NULL; 317 } 318 319 if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) { 320 panic("VMCTL_FLUSHTLB failed: %d", r); 321 } 322 323 assert(loc); 324 325 return (void *) loc; 326 } 327 328 static int pt_init_done; 329 330 /*===========================================================================* 331 * vm_allocpage * 332 *===========================================================================*/ 333 void *vm_allocpages(phys_bytes *phys, int reason, int pages) 334 { 335 /* Allocate a page for use by VM itself. */ 336 phys_bytes newpage; 337 static int level = 0; 338 void *ret; 339 u32_t mem_flags = 0; 340 341 assert(reason >= 0 && reason < VMP_CATEGORIES); 342 343 assert(pages > 0); 344 345 level++; 346 347 assert(level >= 1); 348 assert(level <= 2); 349 350 if((level > 1) || !pt_init_done) { 351 void *s; 352 353 if(pages == 1) s=vm_getsparepage(phys); 354 else if(pages == 4) s=vm_getsparepagedir(phys); 355 else panic("%d pages", pages); 356 357 level--; 358 if(!s) { 359 util_stacktrace(); 360 printf("VM: warning: out of spare pages\n"); 361 } 362 if(!is_staticaddr(s)) vm_self_pages++; 363 return s; 364 } 365 366 #if defined(__arm__) 367 if (reason == VMP_PAGEDIR) { 368 mem_flags |= PAF_ALIGN16K; 369 } 370 #endif 371 372 /* Allocate page of memory for use by VM. As VM 373 * is trusted, we don't have to pre-clear it. 374 */ 375 if((newpage = alloc_mem(pages, mem_flags)) == NO_MEM) { 376 level--; 377 printf("VM: vm_allocpage: alloc_mem failed\n"); 378 return NULL; 379 } 380 381 *phys = CLICK2ABS(newpage); 382 383 if(!(ret = vm_mappages(*phys, pages))) { 384 level--; 385 printf("VM: vm_allocpage: vm_mappages failed\n"); 386 return NULL; 387 } 388 389 level--; 390 vm_self_pages++; 391 392 return ret; 393 } 394 395 void *vm_allocpage(phys_bytes *phys, int reason) 396 { 397 return vm_allocpages(phys, reason, 1); 398 } 399 400 /*===========================================================================* 401 * vm_pagelock * 402 *===========================================================================*/ 403 void vm_pagelock(void *vir, int lockflag) 404 { 405 /* Mark a page allocated by vm_allocpage() unwritable, i.e. only for VM. */ 406 vir_bytes m = (vir_bytes) vir; 407 int r; 408 u32_t flags = ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER; 409 pt_t *pt; 410 411 pt = &vmprocess->vm_pt; 412 413 assert(!(m % VM_PAGE_SIZE)); 414 415 if(!lockflag) 416 flags |= ARCH_VM_PTE_RW; 417 #if defined(__arm__) 418 else 419 flags |= ARCH_VM_PTE_RO; 420 421 flags |= ARM_VM_PTE_CACHED ; 422 #endif 423 424 /* Update flags. */ 425 if((r=pt_writemap(vmprocess, pt, m, 0, VM_PAGE_SIZE, 426 flags, WMF_OVERWRITE | WMF_WRITEFLAGSONLY)) != OK) { 427 panic("vm_lockpage: pt_writemap failed"); 428 } 429 430 if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) { 431 panic("VMCTL_FLUSHTLB failed: %d", r); 432 } 433 434 return; 435 } 436 437 /*===========================================================================* 438 * vm_addrok * 439 *===========================================================================*/ 440 int vm_addrok(void *vir, int writeflag) 441 { 442 pt_t *pt = &vmprocess->vm_pt; 443 int pde, pte; 444 vir_bytes v = (vir_bytes) vir; 445 446 pde = ARCH_VM_PDE(v); 447 pte = ARCH_VM_PTE(v); 448 449 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) { 450 printf("addr not ok: missing pde %d\n", pde); 451 return 0; 452 } 453 454 #if defined(__i386__) 455 if(writeflag && 456 !(pt->pt_dir[pde] & ARCH_VM_PTE_RW)) { 457 printf("addr not ok: pde %d present but pde unwritable\n", pde); 458 return 0; 459 } 460 #elif defined(__arm__) 461 if(writeflag && 462 (pt->pt_dir[pde] & ARCH_VM_PTE_RO)) { 463 printf("addr not ok: pde %d present but pde unwritable\n", pde); 464 return 0; 465 } 466 467 #endif 468 if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) { 469 printf("addr not ok: missing pde %d / pte %d\n", 470 pde, pte); 471 return 0; 472 } 473 474 #if defined(__i386__) 475 if(writeflag && 476 !(pt->pt_pt[pde][pte] & ARCH_VM_PTE_RW)) { 477 printf("addr not ok: pde %d / pte %d present but unwritable\n", 478 pde, pte); 479 #elif defined(__arm__) 480 if(writeflag && 481 (pt->pt_pt[pde][pte] & ARCH_VM_PTE_RO)) { 482 printf("addr not ok: pde %d / pte %d present but unwritable\n", 483 pde, pte); 484 #endif 485 return 0; 486 } 487 488 return 1; 489 } 490 491 /*===========================================================================* 492 * pt_ptalloc * 493 *===========================================================================*/ 494 static int pt_ptalloc(pt_t *pt, int pde, u32_t flags) 495 { 496 /* Allocate a page table and write its address into the page directory. */ 497 int i; 498 phys_bytes pt_phys; 499 u32_t *p; 500 501 /* Argument must make sense. */ 502 assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES); 503 assert(!(flags & ~(PTF_ALLFLAGS))); 504 505 /* We don't expect to overwrite page directory entry, nor 506 * storage for the page table. 507 */ 508 assert(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)); 509 assert(!pt->pt_pt[pde]); 510 511 /* Get storage for the page table. The allocation call may in fact 512 * recursively create the directory entry as a side effect. In that 513 * case, we free the newly allocated page and do nothing else. 514 */ 515 if (!(p = vm_allocpage(&pt_phys, VMP_PAGETABLE))) 516 return ENOMEM; 517 if (pt->pt_pt[pde]) { 518 vm_freepages((vir_bytes) p, 1); 519 assert(pt->pt_pt[pde]); 520 return OK; 521 } 522 pt->pt_pt[pde] = p; 523 524 for(i = 0; i < ARCH_VM_PT_ENTRIES; i++) 525 pt->pt_pt[pde][i] = 0; /* Empty entry. */ 526 527 /* Make page directory entry. 528 * The PDE is always 'present,' 'writable,' and 'user accessible,' 529 * relying on the PTE for protection. 530 */ 531 #if defined(__i386__) 532 pt->pt_dir[pde] = (pt_phys & ARCH_VM_ADDR_MASK) | flags 533 | ARCH_VM_PDE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW; 534 #elif defined(__arm__) 535 pt->pt_dir[pde] = (pt_phys & ARCH_VM_PDE_MASK) 536 | ARCH_VM_PDE_PRESENT | ARM_VM_PDE_DOMAIN; //LSC FIXME 537 #endif 538 539 return OK; 540 } 541 542 /*===========================================================================* 543 * pt_ptalloc_in_range * 544 *===========================================================================*/ 545 int pt_ptalloc_in_range(pt_t *pt, vir_bytes start, vir_bytes end, 546 u32_t flags, int verify) 547 { 548 /* Allocate all the page tables in the range specified. */ 549 int pde, first_pde, last_pde; 550 551 first_pde = ARCH_VM_PDE(start); 552 last_pde = ARCH_VM_PDE(end-1); 553 554 assert(first_pde >= 0); 555 assert(last_pde < ARCH_VM_DIR_ENTRIES); 556 557 /* Scan all page-directory entries in the range. */ 558 for(pde = first_pde; pde <= last_pde; pde++) { 559 assert(!(pt->pt_dir[pde] & ARCH_VM_BIGPAGE)); 560 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) { 561 int r; 562 if(verify) { 563 printf("pt_ptalloc_in_range: no pde %d\n", pde); 564 return EFAULT; 565 } 566 assert(!pt->pt_dir[pde]); 567 if((r=pt_ptalloc(pt, pde, flags)) != OK) { 568 /* Couldn't do (complete) mapping. 569 * Don't bother freeing any previously 570 * allocated page tables, they're 571 * still writable, don't point to nonsense, 572 * and pt_ptalloc leaves the directory 573 * and other data in a consistent state. 574 */ 575 return r; 576 } 577 assert(pt->pt_pt[pde]); 578 } 579 assert(pt->pt_pt[pde]); 580 assert(pt->pt_dir[pde]); 581 assert(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT); 582 } 583 584 return OK; 585 } 586 587 static const char *ptestr(u32_t pte) 588 { 589 #define FLAG(constant, name) { \ 590 if(pte & (constant)) { strcat(str, name); strcat(str, " "); } \ 591 } 592 593 static char str[30]; 594 if(!(pte & ARCH_VM_PTE_PRESENT)) { 595 return "not present"; 596 } 597 str[0] = '\0'; 598 #if defined(__i386__) 599 FLAG(ARCH_VM_PTE_RW, "W"); 600 #elif defined(__arm__) 601 if(pte & ARCH_VM_PTE_RO) { 602 strcat(str, "R "); 603 } else { 604 strcat(str, "W "); 605 } 606 #endif 607 FLAG(ARCH_VM_PTE_USER, "U"); 608 #if defined(__i386__) 609 FLAG(I386_VM_PWT, "PWT"); 610 FLAG(I386_VM_PCD, "PCD"); 611 FLAG(I386_VM_ACC, "ACC"); 612 FLAG(I386_VM_DIRTY, "DIRTY"); 613 FLAG(I386_VM_PS, "PS"); 614 FLAG(I386_VM_GLOBAL, "G"); 615 FLAG(I386_VM_PTAVAIL1, "AV1"); 616 FLAG(I386_VM_PTAVAIL2, "AV2"); 617 FLAG(I386_VM_PTAVAIL3, "AV3"); 618 #elif defined(__arm__) 619 FLAG(ARM_VM_PTE_SUPER, "S"); 620 FLAG(ARM_VM_PTE_S, "SH"); 621 FLAG(ARM_VM_PTE_WB, "WB"); 622 FLAG(ARM_VM_PTE_WT, "WT"); 623 #endif 624 625 return str; 626 } 627 628 /*===========================================================================* 629 * pt_map_in_range * 630 *===========================================================================*/ 631 int pt_map_in_range(struct vmproc *src_vmp, struct vmproc *dst_vmp, 632 vir_bytes start, vir_bytes end) 633 { 634 /* Transfer all the mappings from the pt of the source process to the pt of 635 * the destination process in the range specified. 636 */ 637 int pde, pte; 638 vir_bytes viraddr; 639 pt_t *pt, *dst_pt; 640 641 pt = &src_vmp->vm_pt; 642 dst_pt = &dst_vmp->vm_pt; 643 644 end = end ? end : VM_DATATOP; 645 assert(start % VM_PAGE_SIZE == 0); 646 assert(end % VM_PAGE_SIZE == 0); 647 648 assert( /* ARCH_VM_PDE(start) >= 0 && */ start <= end); 649 assert(ARCH_VM_PDE(end) < ARCH_VM_DIR_ENTRIES); 650 651 #if LU_DEBUG 652 printf("VM: pt_map_in_range: src = %d, dst = %d\n", 653 src_vmp->vm_endpoint, dst_vmp->vm_endpoint); 654 printf("VM: pt_map_in_range: transferring from 0x%08x (pde %d pte %d) to 0x%08x (pde %d pte %d)\n", 655 start, ARCH_VM_PDE(start), ARCH_VM_PTE(start), 656 end, ARCH_VM_PDE(end), ARCH_VM_PTE(end)); 657 #endif 658 659 /* Scan all page-table entries in the range. */ 660 for(viraddr = start; viraddr <= end; viraddr += VM_PAGE_SIZE) { 661 pde = ARCH_VM_PDE(viraddr); 662 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) { 663 if(viraddr == VM_DATATOP) break; 664 continue; 665 } 666 pte = ARCH_VM_PTE(viraddr); 667 if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) { 668 if(viraddr == VM_DATATOP) break; 669 continue; 670 } 671 672 /* Transfer the mapping. */ 673 dst_pt->pt_pt[pde][pte] = pt->pt_pt[pde][pte]; 674 assert(dst_pt->pt_pt[pde]); 675 676 if(viraddr == VM_DATATOP) break; 677 } 678 679 return OK; 680 } 681 682 /*===========================================================================* 683 * pt_ptmap * 684 *===========================================================================*/ 685 int pt_ptmap(struct vmproc *src_vmp, struct vmproc *dst_vmp) 686 { 687 /* Transfer mappings to page dir and page tables from source process and 688 * destination process. 689 */ 690 int pde, r; 691 phys_bytes physaddr; 692 vir_bytes viraddr; 693 pt_t *pt; 694 695 pt = &src_vmp->vm_pt; 696 697 #if LU_DEBUG 698 printf("VM: pt_ptmap: src = %d, dst = %d\n", 699 src_vmp->vm_endpoint, dst_vmp->vm_endpoint); 700 #endif 701 702 /* Transfer mapping to the page directory. */ 703 viraddr = (vir_bytes) pt->pt_dir; 704 physaddr = pt->pt_dir_phys & ARCH_VM_ADDR_MASK; 705 #if defined(__i386__) 706 if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, VM_PAGE_SIZE, 707 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW, 708 #elif defined(__arm__) 709 if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, ARCH_PAGEDIR_SIZE, 710 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | 711 ARM_VM_PTE_CACHED , 712 #endif 713 WMF_OVERWRITE)) != OK) { 714 return r; 715 } 716 #if LU_DEBUG 717 printf("VM: pt_ptmap: transferred mapping to page dir: 0x%08x (0x%08x)\n", 718 viraddr, physaddr); 719 #endif 720 721 /* Scan all non-reserved page-directory entries. */ 722 for(pde=0; pde < kern_start_pde; pde++) { 723 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) { 724 continue; 725 } 726 727 if(!pt->pt_pt[pde]) { panic("pde %d empty\n", pde); } 728 729 /* Transfer mapping to the page table. */ 730 viraddr = (vir_bytes) pt->pt_pt[pde]; 731 #if defined(__i386__) 732 physaddr = pt->pt_dir[pde] & ARCH_VM_ADDR_MASK; 733 #elif defined(__arm__) 734 physaddr = pt->pt_dir[pde] & ARCH_VM_PDE_MASK; 735 #endif 736 assert(viraddr); 737 if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, VM_PAGE_SIZE, 738 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW 739 #ifdef __arm__ 740 | ARM_VM_PTE_CACHED 741 #endif 742 , 743 WMF_OVERWRITE)) != OK) { 744 return r; 745 } 746 } 747 748 return OK; 749 } 750 751 void pt_clearmapcache(void) 752 { 753 /* Make sure kernel will invalidate tlb when using current 754 * pagetable (i.e. vm's) to make new mappings before new cr3 755 * is loaded. 756 */ 757 if(sys_vmctl(SELF, VMCTL_CLEARMAPCACHE, 0) != OK) 758 panic("VMCTL_CLEARMAPCACHE failed"); 759 } 760 761 int pt_writable(struct vmproc *vmp, vir_bytes v) 762 { 763 u32_t entry; 764 pt_t *pt = &vmp->vm_pt; 765 assert(!(v % VM_PAGE_SIZE)); 766 int pde = ARCH_VM_PDE(v); 767 int pte = ARCH_VM_PTE(v); 768 769 assert(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT); 770 assert(pt->pt_pt[pde]); 771 772 entry = pt->pt_pt[pde][pte]; 773 774 #if defined(__i386__) 775 return((entry & PTF_WRITE) ? 1 : 0); 776 #elif defined(__arm__) 777 return((entry & ARCH_VM_PTE_RO) ? 0 : 1); 778 #endif 779 } 780 781 /*===========================================================================* 782 * pt_writemap * 783 *===========================================================================*/ 784 int pt_writemap(struct vmproc * vmp, 785 pt_t *pt, 786 vir_bytes v, 787 phys_bytes physaddr, 788 size_t bytes, 789 u32_t flags, 790 u32_t writemapflags) 791 { 792 /* Write mapping into page table. Allocate a new page table if necessary. */ 793 /* Page directory and table entries for this virtual address. */ 794 int p, pages; 795 int verify = 0; 796 int ret = OK; 797 798 #ifdef CONFIG_SMP 799 int vminhibit_clear = 0; 800 /* FIXME 801 * don't do it everytime, stop the process only on the first change and 802 * resume the execution on the last change. Do in a wrapper of this 803 * function 804 */ 805 if (vmp && vmp->vm_endpoint != NONE && vmp->vm_endpoint != VM_PROC_NR && 806 !(vmp->vm_flags & VMF_EXITING)) { 807 sys_vmctl(vmp->vm_endpoint, VMCTL_VMINHIBIT_SET, 0); 808 vminhibit_clear = 1; 809 } 810 #endif 811 812 if(writemapflags & WMF_VERIFY) 813 verify = 1; 814 815 assert(!(bytes % VM_PAGE_SIZE)); 816 assert(!(flags & ~(PTF_ALLFLAGS))); 817 818 pages = bytes / VM_PAGE_SIZE; 819 820 /* MAP_NONE means to clear the mapping. It doesn't matter 821 * what's actually written into the PTE if PRESENT 822 * isn't on, so we can just write MAP_NONE into it. 823 */ 824 assert(physaddr == MAP_NONE || (flags & ARCH_VM_PTE_PRESENT)); 825 assert(physaddr != MAP_NONE || !flags); 826 827 /* First make sure all the necessary page tables are allocated, 828 * before we start writing in any of them, because it's a pain 829 * to undo our work properly. 830 */ 831 ret = pt_ptalloc_in_range(pt, v, v + VM_PAGE_SIZE*pages, flags, verify); 832 if(ret != OK) { 833 printf("VM: writemap: pt_ptalloc_in_range failed\n"); 834 goto resume_exit; 835 } 836 837 /* Now write in them. */ 838 for(p = 0; p < pages; p++) { 839 u32_t entry; 840 int pde = ARCH_VM_PDE(v); 841 int pte = ARCH_VM_PTE(v); 842 843 assert(!(v % VM_PAGE_SIZE)); 844 assert(pte >= 0 && pte < ARCH_VM_PT_ENTRIES); 845 assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES); 846 847 /* Page table has to be there. */ 848 assert(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT); 849 850 /* We do not expect it to be a bigpage. */ 851 assert(!(pt->pt_dir[pde] & ARCH_VM_BIGPAGE)); 852 853 /* Make sure page directory entry for this page table 854 * is marked present and page table entry is available. 855 */ 856 assert(pt->pt_pt[pde]); 857 858 if(writemapflags & (WMF_WRITEFLAGSONLY|WMF_FREE)) { 859 #if defined(__i386__) 860 physaddr = pt->pt_pt[pde][pte] & ARCH_VM_ADDR_MASK; 861 #elif defined(__arm__) 862 physaddr = pt->pt_pt[pde][pte] & ARM_VM_PTE_MASK; 863 #endif 864 } 865 866 if(writemapflags & WMF_FREE) { 867 free_mem(ABS2CLICK(physaddr), 1); 868 } 869 870 /* Entry we will write. */ 871 #if defined(__i386__) 872 entry = (physaddr & ARCH_VM_ADDR_MASK) | flags; 873 #elif defined(__arm__) 874 entry = (physaddr & ARM_VM_PTE_MASK) | flags; 875 #endif 876 877 if(verify) { 878 u32_t maskedentry; 879 maskedentry = pt->pt_pt[pde][pte]; 880 #if defined(__i386__) 881 maskedentry &= ~(I386_VM_ACC|I386_VM_DIRTY); 882 #endif 883 /* Verify pagetable entry. */ 884 #if defined(__i386__) 885 if(entry & ARCH_VM_PTE_RW) { 886 /* If we expect a writable page, allow a readonly page. */ 887 maskedentry |= ARCH_VM_PTE_RW; 888 } 889 #elif defined(__arm__) 890 if(!(entry & ARCH_VM_PTE_RO)) { 891 /* If we expect a writable page, allow a readonly page. */ 892 maskedentry &= ~ARCH_VM_PTE_RO; 893 } 894 maskedentry &= ~(ARM_VM_PTE_WB|ARM_VM_PTE_WT); 895 #endif 896 if(maskedentry != entry) { 897 printf("pt_writemap: mismatch: "); 898 #if defined(__i386__) 899 if((entry & ARCH_VM_ADDR_MASK) != 900 (maskedentry & ARCH_VM_ADDR_MASK)) { 901 #elif defined(__arm__) 902 if((entry & ARM_VM_PTE_MASK) != 903 (maskedentry & ARM_VM_PTE_MASK)) { 904 #endif 905 printf("pt_writemap: physaddr mismatch (0x%lx, 0x%lx); ", 906 (long)entry, (long)maskedentry); 907 } else printf("phys ok; "); 908 printf(" flags: found %s; ", 909 ptestr(pt->pt_pt[pde][pte])); 910 printf(" masked %s; ", 911 ptestr(maskedentry)); 912 printf(" expected %s\n", ptestr(entry)); 913 printf("found 0x%x, wanted 0x%x\n", 914 pt->pt_pt[pde][pte], entry); 915 ret = EFAULT; 916 goto resume_exit; 917 } 918 } else { 919 /* Write pagetable entry. */ 920 pt->pt_pt[pde][pte] = entry; 921 } 922 923 physaddr += VM_PAGE_SIZE; 924 v += VM_PAGE_SIZE; 925 } 926 927 resume_exit: 928 929 #ifdef CONFIG_SMP 930 if (vminhibit_clear) { 931 assert(vmp && vmp->vm_endpoint != NONE && vmp->vm_endpoint != VM_PROC_NR && 932 !(vmp->vm_flags & VMF_EXITING)); 933 sys_vmctl(vmp->vm_endpoint, VMCTL_VMINHIBIT_CLEAR, 0); 934 } 935 #endif 936 937 return ret; 938 } 939 940 /*===========================================================================* 941 * pt_checkrange * 942 *===========================================================================*/ 943 int pt_checkrange(pt_t *pt, vir_bytes v, size_t bytes, 944 int write) 945 { 946 int p, pages; 947 948 assert(!(bytes % VM_PAGE_SIZE)); 949 950 pages = bytes / VM_PAGE_SIZE; 951 952 for(p = 0; p < pages; p++) { 953 int pde = ARCH_VM_PDE(v); 954 int pte = ARCH_VM_PTE(v); 955 956 assert(!(v % VM_PAGE_SIZE)); 957 assert(pte >= 0 && pte < ARCH_VM_PT_ENTRIES); 958 assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES); 959 960 /* Page table has to be there. */ 961 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) 962 return EFAULT; 963 964 /* Make sure page directory entry for this page table 965 * is marked present and page table entry is available. 966 */ 967 assert((pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT) && pt->pt_pt[pde]); 968 969 if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) { 970 return EFAULT; 971 } 972 973 #if defined(__i386__) 974 if(write && !(pt->pt_pt[pde][pte] & ARCH_VM_PTE_RW)) { 975 #elif defined(__arm__) 976 if(write && (pt->pt_pt[pde][pte] & ARCH_VM_PTE_RO)) { 977 #endif 978 return EFAULT; 979 } 980 981 v += VM_PAGE_SIZE; 982 } 983 984 return OK; 985 } 986 987 /*===========================================================================* 988 * pt_new * 989 *===========================================================================*/ 990 int pt_new(pt_t *pt) 991 { 992 /* Allocate a pagetable root. Allocate a page-aligned page directory 993 * and set them to 0 (indicating no page tables are allocated). Lookup 994 * its physical address as we'll need that in the future. Verify it's 995 * page-aligned. 996 */ 997 int i, r; 998 999 /* Don't ever re-allocate/re-move a certain process slot's 1000 * page directory once it's been created. This is a fraction 1001 * faster, but also avoids having to invalidate the page 1002 * mappings from in-kernel page tables pointing to 1003 * the page directories (the page_directories data). 1004 */ 1005 if(!pt->pt_dir && 1006 !(pt->pt_dir = vm_allocpages((phys_bytes *)&pt->pt_dir_phys, 1007 VMP_PAGEDIR, ARCH_PAGEDIR_SIZE/VM_PAGE_SIZE))) { 1008 return ENOMEM; 1009 } 1010 1011 assert(!((u32_t)pt->pt_dir_phys % ARCH_PAGEDIR_SIZE)); 1012 1013 for(i = 0; i < ARCH_VM_DIR_ENTRIES; i++) { 1014 pt->pt_dir[i] = 0; /* invalid entry (PRESENT bit = 0) */ 1015 pt->pt_pt[i] = NULL; 1016 } 1017 1018 /* Where to start looking for free virtual address space? */ 1019 pt->pt_virtop = 0; 1020 1021 /* Map in kernel. */ 1022 if((r=pt_mapkernel(pt)) != OK) 1023 return r; 1024 1025 return OK; 1026 } 1027 1028 static int freepde(void) 1029 { 1030 int p = kernel_boot_info.freepde_start++; 1031 assert(kernel_boot_info.freepde_start < ARCH_VM_DIR_ENTRIES); 1032 return p; 1033 } 1034 1035 void pt_allocate_kernel_mapped_pagetables(void) 1036 { 1037 /* Reserve PDEs available for mapping in the page directories. */ 1038 int pd; 1039 for(pd = 0; pd < MAX_PAGEDIR_PDES; pd++) { 1040 struct pdm *pdm = &pagedir_mappings[pd]; 1041 if(!pdm->pdeno) { 1042 pdm->pdeno = freepde(); 1043 assert(pdm->pdeno); 1044 } 1045 phys_bytes ph; 1046 1047 /* Allocate us a page table in which to 1048 * remember page directory pointers. 1049 */ 1050 if(!(pdm->page_directories = 1051 vm_allocpage(&ph, VMP_PAGETABLE))) { 1052 panic("no virt addr for vm mappings"); 1053 } 1054 memset(pdm->page_directories, 0, VM_PAGE_SIZE); 1055 pdm->phys = ph; 1056 1057 #if defined(__i386__) 1058 pdm->val = (ph & ARCH_VM_ADDR_MASK) | 1059 ARCH_VM_PDE_PRESENT | ARCH_VM_PTE_RW; 1060 #elif defined(__arm__) 1061 pdm->val = (ph & ARCH_VM_PDE_MASK) 1062 | ARCH_VM_PDE_PRESENT 1063 | ARM_VM_PTE_CACHED 1064 | ARM_VM_PDE_DOMAIN; //LSC FIXME 1065 #endif 1066 } 1067 } 1068 1069 static void pt_copy(pt_t *dst, pt_t *src) 1070 { 1071 int pde; 1072 for(pde=0; pde < kern_start_pde; pde++) { 1073 if(!(src->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) { 1074 continue; 1075 } 1076 assert(!(src->pt_dir[pde] & ARCH_VM_BIGPAGE)); 1077 if(!src->pt_pt[pde]) { panic("pde %d empty\n", pde); } 1078 if(pt_ptalloc(dst, pde, 0) != OK) 1079 panic("pt_ptalloc failed"); 1080 memcpy(dst->pt_pt[pde], src->pt_pt[pde], 1081 ARCH_VM_PT_ENTRIES * sizeof(*dst->pt_pt[pde])); 1082 } 1083 } 1084 1085 /*===========================================================================* 1086 * pt_init * 1087 *===========================================================================*/ 1088 void pt_init(void) 1089 { 1090 pt_t *newpt, newpt_dyn; 1091 int s, r, p; 1092 phys_bytes phys; 1093 vir_bytes sparepages_mem; 1094 #if defined(__arm__) 1095 vir_bytes sparepagedirs_mem; 1096 #endif 1097 static u32_t currentpagedir[ARCH_VM_DIR_ENTRIES]; 1098 int m = kernel_boot_info.kern_mod; 1099 #if defined(__i386__) 1100 int global_bit_ok = 0; 1101 u32_t mypdbr; /* Page Directory Base Register (cr3) value */ 1102 #elif defined(__arm__) 1103 u32_t myttbr; 1104 #endif 1105 1106 /* Find what the physical location of the kernel is. */ 1107 assert(m >= 0); 1108 assert(m < kernel_boot_info.mods_with_kernel); 1109 assert(kernel_boot_info.mods_with_kernel < MULTIBOOT_MAX_MODS); 1110 kern_mb_mod = &kernel_boot_info.module_list[m]; 1111 kern_size = kern_mb_mod->mod_end - kern_mb_mod->mod_start; 1112 assert(!(kern_mb_mod->mod_start % ARCH_BIG_PAGE_SIZE)); 1113 assert(!(kernel_boot_info.vir_kern_start % ARCH_BIG_PAGE_SIZE)); 1114 kern_start_pde = kernel_boot_info.vir_kern_start / ARCH_BIG_PAGE_SIZE; 1115 1116 /* Get ourselves spare pages. */ 1117 sparepages_mem = (vir_bytes) static_sparepages; 1118 assert(!(sparepages_mem % VM_PAGE_SIZE)); 1119 1120 #if defined(__arm__) 1121 /* Get ourselves spare pagedirs. */ 1122 sparepagedirs_mem = (vir_bytes) static_sparepagedirs; 1123 assert(!(sparepagedirs_mem % ARCH_PAGEDIR_SIZE)); 1124 #endif 1125 1126 /* Spare pages are used to allocate memory before VM has its own page 1127 * table that things (i.e. arbitrary physical memory) can be mapped into. 1128 * We get it by pre-allocating it in our bss (allocated and mapped in by 1129 * the kernel) in static_sparepages. We also need the physical addresses 1130 * though; we look them up now so they are ready for use. 1131 */ 1132 #if defined(__arm__) 1133 missing_sparedirs = 0; 1134 assert(STATIC_SPAREPAGEDIRS <= SPAREPAGEDIRS); 1135 for(s = 0; s < SPAREPAGEDIRS; s++) { 1136 vir_bytes v = (sparepagedirs_mem + s*ARCH_PAGEDIR_SIZE);; 1137 phys_bytes ph; 1138 if((r=sys_umap(SELF, VM_D, (vir_bytes) v, 1139 ARCH_PAGEDIR_SIZE, &ph)) != OK) 1140 panic("pt_init: sys_umap failed: %d", r); 1141 if(s >= STATIC_SPAREPAGEDIRS) { 1142 sparepagedirs[s].pagedir = NULL; 1143 missing_sparedirs++; 1144 continue; 1145 } 1146 sparepagedirs[s].pagedir = (void *) v; 1147 sparepagedirs[s].phys = ph; 1148 } 1149 #endif 1150 1151 if(!(spare_pagequeue = reservedqueue_new(SPAREPAGES, 1, 1, 0))) 1152 panic("reservedqueue_new for single pages failed"); 1153 1154 assert(STATIC_SPAREPAGES < SPAREPAGES); 1155 for(s = 0; s < STATIC_SPAREPAGES; s++) { 1156 void *v = (void *) (sparepages_mem + s*VM_PAGE_SIZE); 1157 phys_bytes ph; 1158 if((r=sys_umap(SELF, VM_D, (vir_bytes) v, 1159 VM_PAGE_SIZE*SPAREPAGES, &ph)) != OK) 1160 panic("pt_init: sys_umap failed: %d", r); 1161 reservedqueue_add(spare_pagequeue, v, ph); 1162 } 1163 1164 #if defined(__i386__) 1165 /* global bit and 4MB pages available? */ 1166 global_bit_ok = _cpufeature(_CPUF_I386_PGE); 1167 bigpage_ok = _cpufeature(_CPUF_I386_PSE); 1168 1169 /* Set bit for PTE's and PDE's if available. */ 1170 if(global_bit_ok) 1171 global_bit = I386_VM_GLOBAL; 1172 #endif 1173 1174 /* Now reserve another pde for kernel's own mappings. */ 1175 { 1176 int kernmap_pde; 1177 phys_bytes addr, len; 1178 int flags, pindex = 0; 1179 u32_t offset = 0; 1180 1181 kernmap_pde = freepde(); 1182 offset = kernmap_pde * ARCH_BIG_PAGE_SIZE; 1183 1184 while(sys_vmctl_get_mapping(pindex, &addr, &len, 1185 &flags) == OK) { 1186 int usedpde; 1187 vir_bytes vir; 1188 if(pindex >= MAX_KERNMAPPINGS) 1189 panic("VM: too many kernel mappings: %d", pindex); 1190 kern_mappings[pindex].phys_addr = addr; 1191 kern_mappings[pindex].len = len; 1192 kern_mappings[pindex].flags = flags; 1193 kern_mappings[pindex].vir_addr = offset; 1194 kern_mappings[pindex].flags = 1195 ARCH_VM_PTE_PRESENT; 1196 if(flags & VMMF_UNCACHED) 1197 #if defined(__i386__) 1198 kern_mappings[pindex].flags |= PTF_NOCACHE; 1199 #elif defined(__arm__) 1200 kern_mappings[pindex].flags |= ARM_VM_PTE_DEVICE; 1201 else { 1202 kern_mappings[pindex].flags |= ARM_VM_PTE_CACHED; 1203 } 1204 #endif 1205 if(flags & VMMF_USER) 1206 kern_mappings[pindex].flags |= ARCH_VM_PTE_USER; 1207 #if defined(__arm__) 1208 else 1209 kern_mappings[pindex].flags |= ARM_VM_PTE_SUPER; 1210 #endif 1211 if(flags & VMMF_WRITE) 1212 kern_mappings[pindex].flags |= ARCH_VM_PTE_RW; 1213 #if defined(__arm__) 1214 else 1215 kern_mappings[pindex].flags |= ARCH_VM_PTE_RO; 1216 #endif 1217 1218 #if defined(__i386__) 1219 if(flags & VMMF_GLO) 1220 kern_mappings[pindex].flags |= I386_VM_GLOBAL; 1221 #endif 1222 1223 if(addr % VM_PAGE_SIZE) 1224 panic("VM: addr unaligned: %lu", addr); 1225 if(len % VM_PAGE_SIZE) 1226 panic("VM: len unaligned: %lu", len); 1227 vir = offset; 1228 if(sys_vmctl_reply_mapping(pindex, vir) != OK) 1229 panic("VM: reply failed"); 1230 offset += len; 1231 pindex++; 1232 kernmappings++; 1233 1234 usedpde = ARCH_VM_PDE(offset); 1235 while(usedpde > kernmap_pde) { 1236 int newpde = freepde(); 1237 assert(newpde == kernmap_pde+1); 1238 kernmap_pde = newpde; 1239 } 1240 } 1241 } 1242 1243 pt_allocate_kernel_mapped_pagetables(); 1244 1245 /* Allright. Now. We have to make our own page directory and page tables, 1246 * that the kernel has already set up, accessible to us. It's easier to 1247 * understand if we just copy all the required pages (i.e. page directory 1248 * and page tables), and set up the pointers as if VM had done it itself. 1249 * 1250 * This allocation will happen without using any page table, and just 1251 * uses spare pages. 1252 */ 1253 newpt = &vmprocess->vm_pt; 1254 if(pt_new(newpt) != OK) 1255 panic("vm pt_new failed"); 1256 1257 /* Get our current pagedir so we can see it. */ 1258 #if defined(__i386__) 1259 if(sys_vmctl_get_pdbr(SELF, &mypdbr) != OK) 1260 #elif defined(__arm__) 1261 if(sys_vmctl_get_pdbr(SELF, &myttbr) != OK) 1262 #endif 1263 1264 panic("VM: sys_vmctl_get_pdbr failed"); 1265 #if defined(__i386__) 1266 if(sys_vircopy(NONE, mypdbr, SELF, 1267 (vir_bytes) currentpagedir, VM_PAGE_SIZE, 0) != OK) 1268 #elif defined(__arm__) 1269 if(sys_vircopy(NONE, myttbr, SELF, 1270 (vir_bytes) currentpagedir, ARCH_PAGEDIR_SIZE, 0) != OK) 1271 #endif 1272 panic("VM: sys_vircopy failed"); 1273 1274 /* We have mapped in kernel ourselves; now copy mappings for VM 1275 * that kernel made, including allocations for BSS. Skip identity 1276 * mapping bits; just map in VM. 1277 */ 1278 for(p = 0; p < ARCH_VM_DIR_ENTRIES; p++) { 1279 u32_t entry = currentpagedir[p]; 1280 phys_bytes ptaddr_kern, ptaddr_us; 1281 1282 /* BIGPAGEs are kernel mapping (do ourselves) or boot 1283 * identity mapping (don't want). 1284 */ 1285 if(!(entry & ARCH_VM_PDE_PRESENT)) continue; 1286 if((entry & ARCH_VM_BIGPAGE)) continue; 1287 1288 if(pt_ptalloc(newpt, p, 0) != OK) 1289 panic("pt_ptalloc failed"); 1290 assert(newpt->pt_dir[p] & ARCH_VM_PDE_PRESENT); 1291 1292 #if defined(__i386__) 1293 ptaddr_kern = entry & ARCH_VM_ADDR_MASK; 1294 ptaddr_us = newpt->pt_dir[p] & ARCH_VM_ADDR_MASK; 1295 #elif defined(__arm__) 1296 ptaddr_kern = entry & ARCH_VM_PDE_MASK; 1297 ptaddr_us = newpt->pt_dir[p] & ARCH_VM_PDE_MASK; 1298 #endif 1299 1300 /* Copy kernel-initialized pagetable contents into our 1301 * normally accessible pagetable. 1302 */ 1303 if(sys_abscopy(ptaddr_kern, ptaddr_us, VM_PAGE_SIZE) != OK) 1304 panic("pt_init: abscopy failed"); 1305 } 1306 1307 /* Inform kernel vm has a newly built page table. */ 1308 assert(vmproc[VM_PROC_NR].vm_endpoint == VM_PROC_NR); 1309 pt_bind(newpt, &vmproc[VM_PROC_NR]); 1310 1311 pt_init_done = 1; 1312 1313 /* VM is now fully functional in that it can dynamically allocate memory 1314 * for itself. 1315 * 1316 * We don't want to keep using the bootstrap statically allocated spare 1317 * pages though, as the physical addresses will change on liveupdate. So we 1318 * re-do part of the initialization now with purely dynamically allocated 1319 * memory. First throw out the static pool. 1320 * 1321 * Then allocate the kernel-shared-pagetables and VM pagetables with dynamic 1322 * memory. 1323 */ 1324 1325 alloc_cycle(); /* Make sure allocating works */ 1326 while(vm_getsparepage(&phys)) ; /* Use up all static pages */ 1327 alloc_cycle(); /* Refill spares with dynamic */ 1328 pt_allocate_kernel_mapped_pagetables(); /* Reallocate in-kernel pages */ 1329 pt_bind(newpt, &vmproc[VM_PROC_NR]); /* Recalculate */ 1330 pt_mapkernel(newpt); /* Rewrite pagetable info */ 1331 1332 /* Flush TLB just in case any of those mappings have been touched */ 1333 if((sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) { 1334 panic("VMCTL_FLUSHTLB failed"); 1335 } 1336 1337 /* Recreate VM page table with dynamic-only allocations */ 1338 memset(&newpt_dyn, 0, sizeof(newpt_dyn)); 1339 pt_new(&newpt_dyn); 1340 pt_copy(&newpt_dyn, newpt); 1341 memcpy(newpt, &newpt_dyn, sizeof(*newpt)); 1342 1343 pt_bind(newpt, &vmproc[VM_PROC_NR]); /* Recalculate */ 1344 pt_mapkernel(newpt); /* Rewrite pagetable info */ 1345 1346 /* Flush TLB just in case any of those mappings have been touched */ 1347 if((sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) { 1348 panic("VMCTL_FLUSHTLB failed"); 1349 } 1350 1351 /* All OK. */ 1352 return; 1353 } 1354 1355 /*===========================================================================* 1356 * pt_bind * 1357 *===========================================================================*/ 1358 int pt_bind(pt_t *pt, struct vmproc *who) 1359 { 1360 int procslot, pdeslot; 1361 u32_t phys; 1362 void *pdes; 1363 int pagedir_pde; 1364 int slots_per_pde; 1365 int pages_per_pagedir = ARCH_PAGEDIR_SIZE/VM_PAGE_SIZE; 1366 struct pdm *pdm; 1367 1368 slots_per_pde = ARCH_VM_PT_ENTRIES / pages_per_pagedir; 1369 1370 /* Basic sanity checks. */ 1371 assert(who); 1372 assert(who->vm_flags & VMF_INUSE); 1373 assert(pt); 1374 1375 procslot = who->vm_slot; 1376 pdm = &pagedir_mappings[procslot/slots_per_pde]; 1377 pdeslot = procslot%slots_per_pde; 1378 pagedir_pde = pdm->pdeno; 1379 assert(pdeslot >= 0); 1380 assert(procslot < ELEMENTS(vmproc)); 1381 assert(pdeslot < ARCH_VM_PT_ENTRIES / pages_per_pagedir); 1382 assert(pagedir_pde >= 0); 1383 1384 #if defined(__i386__) 1385 phys = pt->pt_dir_phys & ARCH_VM_ADDR_MASK; 1386 #elif defined(__arm__) 1387 phys = pt->pt_dir_phys & ARM_VM_PTE_MASK; 1388 #endif 1389 assert(pt->pt_dir_phys == phys); 1390 assert(!(pt->pt_dir_phys % ARCH_PAGEDIR_SIZE)); 1391 1392 /* Update "page directory pagetable." */ 1393 #if defined(__i386__) 1394 pdm->page_directories[pdeslot] = 1395 phys | ARCH_VM_PDE_PRESENT|ARCH_VM_PTE_RW; 1396 #elif defined(__arm__) 1397 { 1398 int i; 1399 for (i = 0; i < pages_per_pagedir; i++) { 1400 pdm->page_directories[pdeslot*pages_per_pagedir+i] = 1401 (phys+i*VM_PAGE_SIZE) 1402 | ARCH_VM_PTE_PRESENT 1403 | ARCH_VM_PTE_RW 1404 | ARM_VM_PTE_CACHED 1405 | ARCH_VM_PTE_USER; //LSC FIXME 1406 } 1407 } 1408 #endif 1409 1410 /* This is where the PDE's will be visible to the kernel 1411 * in its address space. 1412 */ 1413 pdes = (void *) (pagedir_pde*ARCH_BIG_PAGE_SIZE + 1414 #if defined(__i386__) 1415 pdeslot * VM_PAGE_SIZE); 1416 #elif defined(__arm__) 1417 pdeslot * ARCH_PAGEDIR_SIZE); 1418 #endif 1419 1420 /* Tell kernel about new page table root. */ 1421 return sys_vmctl_set_addrspace(who->vm_endpoint, pt->pt_dir_phys , pdes); 1422 } 1423 1424 /*===========================================================================* 1425 * pt_free * 1426 *===========================================================================*/ 1427 void pt_free(pt_t *pt) 1428 { 1429 /* Free memory associated with this pagetable. */ 1430 int i; 1431 1432 for(i = 0; i < ARCH_VM_DIR_ENTRIES; i++) 1433 if(pt->pt_pt[i]) 1434 vm_freepages((vir_bytes) pt->pt_pt[i], 1); 1435 1436 return; 1437 } 1438 1439 /*===========================================================================* 1440 * pt_mapkernel * 1441 *===========================================================================*/ 1442 int pt_mapkernel(pt_t *pt) 1443 { 1444 int i; 1445 int kern_pde = kern_start_pde; 1446 phys_bytes addr, mapped = 0; 1447 1448 /* Any page table needs to map in the kernel address space. */ 1449 assert(bigpage_ok); 1450 assert(kern_pde >= 0); 1451 1452 /* pt_init() has made sure this is ok. */ 1453 addr = kern_mb_mod->mod_start; 1454 1455 /* Actually mapping in kernel */ 1456 while(mapped < kern_size) { 1457 #if defined(__i386__) 1458 pt->pt_dir[kern_pde] = addr | ARCH_VM_PDE_PRESENT | 1459 ARCH_VM_BIGPAGE | ARCH_VM_PTE_RW | global_bit; 1460 #elif defined(__arm__) 1461 pt->pt_dir[kern_pde] = (addr & ARM_VM_SECTION_MASK) 1462 | ARM_VM_SECTION 1463 | ARM_VM_SECTION_DOMAIN 1464 | ARM_VM_SECTION_CACHED 1465 | ARM_VM_SECTION_SUPER; 1466 #endif 1467 kern_pde++; 1468 mapped += ARCH_BIG_PAGE_SIZE; 1469 addr += ARCH_BIG_PAGE_SIZE; 1470 } 1471 1472 /* Kernel also wants to know about all page directories. */ 1473 { 1474 int pd; 1475 for(pd = 0; pd < MAX_PAGEDIR_PDES; pd++) { 1476 struct pdm *pdm = &pagedir_mappings[pd]; 1477 1478 assert(pdm->pdeno > 0); 1479 assert(pdm->pdeno > kern_pde); 1480 pt->pt_dir[pdm->pdeno] = pdm->val; 1481 } 1482 } 1483 1484 /* Kernel also wants various mappings of its own. */ 1485 for(i = 0; i < kernmappings; i++) { 1486 int r; 1487 if((r=pt_writemap(NULL, pt, 1488 kern_mappings[i].vir_addr, 1489 kern_mappings[i].phys_addr, 1490 kern_mappings[i].len, 1491 kern_mappings[i].flags, 0)) != OK) { 1492 return r; 1493 } 1494 1495 } 1496 1497 return OK; 1498 } 1499 1500 int get_vm_self_pages(void) { return vm_self_pages; } 1501