1 /* $NetBSD: pmap.c,v 1.27 2010/11/12 07:59:26 uebayasi Exp $ */ 2 3 4 /*- 5 * Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 10 * NASA Ames Research Center and by Chris G. Demetriou. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /*- 35 * Copyright (c) 1991 Regents of the University of California. 36 * All rights reserved. 37 * Copyright (c) 1994 John S. Dyson 38 * All rights reserved. 39 * Copyright (c) 1994 David Greenman 40 * All rights reserved. 41 * Copyright (c) 1998,2000 Doug Rabson 42 * All rights reserved. 43 * 44 * This code is derived from software contributed to Berkeley by 45 * the Systems Programming Group of the University of Utah Computer 46 * Science Department and William Jolitz of UUNET Technologies Inc. 47 * 48 * Redistribution and use in source and binary forms, with or without 49 * modification, are permitted provided that the following conditions 50 * are met: 51 * 1. Redistributions of source code must retain the above copyright 52 * notice, this list of conditions and the following disclaimer. 53 * 2. Redistributions in binary form must reproduce the above copyright 54 * notice, this list of conditions and the following disclaimer in the 55 * documentation and/or other materials provided with the distribution. 56 * 3. All advertising materials mentioning features or use of this software 57 * must display the following acknowledgement: 58 * This product includes software developed by the University of 59 * California, Berkeley and its contributors. 60 * 4. Neither the name of the University nor the names of its contributors 61 * may be used to endorse or promote products derived from this software 62 * without specific prior written permission. 63 * 64 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 65 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 66 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 67 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 68 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 69 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 70 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 71 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 72 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 73 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 74 * SUCH DAMAGE. 75 * 76 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 77 * from: i386 Id: pmap.c,v 1.193 1998/04/19 15:22:48 bde Exp 78 * with some ideas from NetBSD's alpha pmap 79 */ 80 81 /* __FBSDID("$FreeBSD: src/sys/ia64/ia64/pmap.c,v 1.172 2005/11/20 06:09:48 alc Exp $"); */ 82 83 84 /* XXX: This module is a mess. Need to clean up Locking, list traversal. etc....... */ 85 86 #include <sys/cdefs.h> 87 88 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.27 2010/11/12 07:59:26 uebayasi Exp $"); 89 90 #include <sys/param.h> 91 #include <sys/systm.h> 92 #include <sys/buf.h> 93 #include <sys/reboot.h> 94 #include <sys/lock.h> 95 96 #include <uvm/uvm.h> 97 98 #include <machine/pal.h> 99 #include <machine/atomic.h> 100 #include <machine/pte.h> 101 #include <sys/sched.h> 102 #include <machine/cpufunc.h> 103 #include <machine/md_var.h> 104 105 106 /* 107 * Kernel virtual memory management. 108 */ 109 static int nkpt; 110 struct ia64_lpte **ia64_kptdir; 111 #define KPTE_DIR_INDEX(va) \ 112 ((va >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1)) 113 #define KPTE_PTE_INDEX(va) \ 114 ((va >> PAGE_SHIFT) & ((1<<(PAGE_SHIFT-5))-1)) 115 #define NKPTEPG (PAGE_SIZE / sizeof(struct ia64_lpte)) 116 117 118 /* Values for ptc.e. XXX values for SKI. */ 119 static uint64_t pmap_ptc_e_base = 0x100000000; 120 static uint64_t pmap_ptc_e_count1 = 3; 121 static uint64_t pmap_ptc_e_count2 = 2; 122 static uint64_t pmap_ptc_e_stride1 = 0x2000; 123 static uint64_t pmap_ptc_e_stride2 = 0x100000000; 124 kmutex_t pmap_ptc_lock; /* Global PTC lock */ 125 126 /* VHPT Base */ 127 128 vaddr_t vhpt_base; 129 vaddr_t pmap_vhpt_log2size; 130 131 struct ia64_bucket *pmap_vhpt_bucket; 132 int pmap_vhpt_nbuckets; 133 kmutex_t pmap_vhptlock; /* VHPT collision chain lock */ 134 135 int pmap_vhpt_inserts; 136 int pmap_vhpt_resident; 137 int pmap_vhpt_collisions; 138 139 #ifdef DEBUG 140 static void dump_vhpt(void); 141 #endif 142 143 /* 144 * Data for the RID allocator 145 */ 146 static int pmap_ridcount; 147 static int pmap_rididx; 148 static int pmap_ridmapsz; 149 static int pmap_ridmax; 150 static uint64_t *pmap_ridmap; 151 kmutex_t pmap_rid_lock; /* RID allocator lock */ 152 153 154 bool pmap_initialized; /* Has pmap_init completed? */ 155 u_long pmap_pages_stolen; /* instrumentation */ 156 157 static struct pmap kernel_pmap_store; /* the kernel's pmap (proc0) */ 158 struct pmap *const kernel_pmap_ptr = &kernel_pmap_store; 159 160 static vaddr_t kernel_vm_end; /* VA of last avail page ( end of kernel Address Space ) */ 161 162 /* 163 * This variable contains the number of CPU IDs we need to allocate 164 * space for when allocating the pmap structure. It is used to 165 * size a per-CPU array of ASN and ASN Generation number. 166 */ 167 u_long pmap_ncpuids; 168 169 #ifndef PMAP_PV_LOWAT 170 #define PMAP_PV_LOWAT 16 171 #endif 172 int pmap_pv_lowat = PMAP_PV_LOWAT; 173 174 /* 175 * PV table management functions. 176 */ 177 void *pmap_pv_page_alloc(struct pool *, int); 178 void pmap_pv_page_free(struct pool *, void *); 179 180 struct pool_allocator pmap_pv_page_allocator = { 181 pmap_pv_page_alloc, pmap_pv_page_free, 0, 182 }; 183 184 bool pmap_poolpage_alloc(paddr_t *); 185 void pmap_poolpage_free(paddr_t); 186 187 /* 188 * List of all pmaps, used to update them when e.g. additional kernel 189 * page tables are allocated. This list is kept LRU-ordered by 190 * pmap_activate(). XXX: Check on this..... 191 */ 192 TAILQ_HEAD(, pmap) pmap_all_pmaps; 193 194 /* 195 * The pools from which pmap structures and sub-structures are allocated. 196 */ 197 struct pool pmap_pmap_pool; 198 struct pool pmap_ia64_lpte_pool; 199 struct pool pmap_pv_pool; 200 201 kmutex_t pmap_main_lock; 202 kmutex_t pmap_all_pmaps_slock; 203 204 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 205 /* XXX(kochi) need to use only spin lock? */ 206 #define PMAP_MAP_TO_HEAD_LOCK() \ 207 spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 208 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 209 spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 210 #define PMAP_HEAD_TO_MAP_LOCK() \ 211 spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 212 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 213 spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 214 #else 215 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 216 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 217 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 218 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 219 #endif /* MULTIPROCESSOR || LOCKDEBUG */ 220 221 222 #define pmap_accessed(lpte) ((lpte)->pte & PTE_ACCESSED) 223 #define pmap_dirty(lpte) ((lpte)->pte & PTE_DIRTY) 224 #define pmap_managed(lpte) ((lpte)->pte & PTE_MANAGED) 225 #define pmap_ppn(lpte) ((lpte)->pte & PTE_PPN_MASK) 226 #define pmap_present(lpte) ((lpte)->pte & PTE_PRESENT) 227 #define pmap_prot(lpte) (((lpte)->pte & PTE_PROT_MASK) >> 56) 228 #define pmap_wired(lpte) ((lpte)->pte & PTE_WIRED) 229 230 #define pmap_clear_accessed(lpte) (lpte)->pte &= ~PTE_ACCESSED 231 #define pmap_clear_dirty(lpte) (lpte)->pte &= ~PTE_DIRTY 232 #define pmap_clear_present(lpte) (lpte)->pte &= ~PTE_PRESENT 233 #define pmap_clear_wired(lpte) (lpte)->pte &= ~PTE_WIRED 234 235 #define pmap_set_wired(lpte) (lpte)->pte |= PTE_WIRED 236 237 238 /* 239 * The VHPT bucket head structure. 240 */ 241 struct ia64_bucket { 242 uint64_t chain; 243 kmutex_t lock; 244 u_int length; 245 }; 246 247 248 /* Local Helper functions */ 249 250 static void pmap_invalidate_all(pmap_t); 251 static void pmap_invalidate_page(pmap_t, vaddr_t); 252 253 static pmap_t pmap_switch(pmap_t pm); 254 static pmap_t pmap_install(pmap_t); 255 256 static struct ia64_lpte *pmap_find_kpte(vaddr_t); 257 258 static void 259 pmap_set_pte(struct ia64_lpte *, vaddr_t, vaddr_t, bool, bool); 260 static void 261 pmap_free_pte(struct ia64_lpte *pte, vaddr_t va); 262 263 static __inline void 264 pmap_pte_prot(pmap_t pm, struct ia64_lpte *pte, vm_prot_t prot); 265 static int 266 pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vaddr_t va, 267 pv_entry_t pv, int freepte); 268 269 static struct ia64_lpte * 270 pmap_find_pte(vaddr_t va); 271 static int 272 pmap_remove_entry(pmap_t pmap, struct vm_page * pg, vaddr_t va, pv_entry_t pv); 273 static void 274 pmap_insert_entry(pmap_t pmap, vaddr_t va, struct vm_page *pg); 275 276 static __inline int 277 pmap_track_modified(vaddr_t va); 278 279 static void 280 pmap_enter_vhpt(struct ia64_lpte *, vaddr_t); 281 static int pmap_remove_vhpt(vaddr_t); 282 static struct ia64_lpte * 283 pmap_find_vhpt(vaddr_t); 284 void 285 pmap_page_purge(struct vm_page * pg); 286 static void 287 pmap_remove_page(pmap_t pmap, vaddr_t va); 288 289 290 static uint32_t pmap_allocate_rid(void); 291 static void pmap_free_rid(uint32_t rid); 292 293 static vaddr_t 294 pmap_steal_vhpt_memory(vsize_t); 295 296 /* 297 * pmap_steal_memory: [ INTERFACE ] 298 * 299 * Bootstrap memory allocator (alternative to uvm_pageboot_alloc()). 300 * This function allows for early dynamic memory allocation until the 301 * virtual memory system has been bootstrapped. After that point, either 302 * kmem_alloc or malloc should be used. This function works by stealing 303 * pages from the (to be) managed page pool, then implicitly mapping the 304 * pages (by using their RR7 addresses) and zeroing them. 305 * 306 * It may be used once the physical memory segments have been pre-loaded 307 * into the vm_physmem[] array. Early memory allocation MUST use this 308 * interface! This cannot be used after uvm_page_init(), and will 309 * generate a panic if tried. 310 * 311 * Note that this memory will never be freed, and in essence it is wired 312 * down. 313 * 314 * We must adjust *vstartp and/or *vendp iff we use address space 315 * from the kernel virtual address range defined by pmap_virtual_space(). 316 * 317 * Note: no locking is necessary in this function. 318 */ 319 vaddr_t 320 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 321 { 322 int lcv, npgs, x; 323 vaddr_t va; 324 paddr_t pa; 325 326 size = round_page(size); 327 npgs = atop(size); 328 329 #if 0 330 printf("PSM: size 0x%lx (npgs 0x%x)\n", size, npgs); 331 #endif 332 333 for (lcv = 0; lcv < vm_nphysseg; lcv++) { 334 if (uvm.page_init_done == true) 335 panic("pmap_steal_memory: called _after_ bootstrap"); 336 337 #if 0 338 printf(" bank %d: avail_start 0x%lx, start 0x%lx, " 339 "avail_end 0x%lx\n", lcv, VM_PHYSMEM_PTR(lcv)->avail_start, 340 VM_PHYSMEM_PTR(lcv)->start, VM_PHYSMEM_PTR(lcv)->avail_end); 341 #endif 342 343 if (VM_PHYSMEM_PTR(lcv)->avail_start != VM_PHYSMEM_PTR(lcv)->start || 344 VM_PHYSMEM_PTR(lcv)->avail_start >= VM_PHYSMEM_PTR(lcv)->avail_end) 345 continue; 346 347 #if 0 348 printf(" avail_end - avail_start = 0x%lx\n", 349 VM_PHYSMEM_PTR(lcv)->avail_end - VM_PHYSMEM_PTR(lcv)->avail_start); 350 #endif 351 352 if ((VM_PHYSMEM_PTR(lcv)->avail_end - VM_PHYSMEM_PTR(lcv)->avail_start) 353 < npgs) 354 continue; 355 356 /* 357 * There are enough pages here; steal them! 358 */ 359 pa = ptoa(VM_PHYSMEM_PTR(lcv)->avail_start); 360 VM_PHYSMEM_PTR(lcv)->avail_start += npgs; 361 VM_PHYSMEM_PTR(lcv)->start += npgs; 362 363 364 /* 365 * Have we used up this segment? 366 */ 367 if (VM_PHYSMEM_PTR(lcv)->avail_start == VM_PHYSMEM_PTR(lcv)->end) { 368 if (vm_nphysseg == 1) 369 panic("pmap_steal_memory: out of memory!"); 370 371 /* Remove this segment from the list. */ 372 vm_nphysseg--; 373 for (x = lcv; x < vm_nphysseg; x++) { 374 /* structure copy */ 375 VM_PHYSMEM_PTR_SWAP(x, x + 1); 376 } 377 } 378 379 va = IA64_PHYS_TO_RR7(pa); 380 memset((void *)va, 0, size); 381 pmap_pages_stolen += npgs; 382 return va; 383 } 384 385 /* 386 * If we got here, this was no memory left. 387 */ 388 panic("pmap_steal_memory: no memory to steal"); 389 } 390 391 392 /* 393 * pmap_steal_vhpt_memory: Derived from alpha/pmap.c:pmap_steal_memory() 394 * Note: This function is not visible outside the pmap module. 395 * Based on pmap_steal_memory(); 396 * Assumptions: size is always a power of 2. 397 * Returns: Allocated memory at a naturally aligned address 398 */ 399 400 static vaddr_t 401 pmap_steal_vhpt_memory(vsize_t size) 402 { 403 int lcv, npgs, x; 404 vaddr_t va; 405 paddr_t pa; 406 407 paddr_t vhpt_start = 0, start1, start2, end1, end2; 408 409 size = round_page(size); 410 npgs = atop(size); 411 412 #if 1 413 printf("VHPTPSM: size 0x%lx (npgs 0x%x)\n", size, npgs); 414 #endif 415 416 for (lcv = 0; lcv < vm_nphysseg; lcv++) { 417 if (uvm.page_init_done == true) 418 panic("pmap_vhpt_steal_memory: called _after_ bootstrap"); 419 420 #if 1 421 printf(" lcv %d: avail_start 0x%lx, start 0x%lx, " 422 "avail_end 0x%lx\n", lcv, VM_PHYSMEM_PTR(lcv)->avail_start, 423 VM_PHYSMEM_PTR(lcv)->start, VM_PHYSMEM_PTR(lcv)->avail_end); 424 printf(" avail_end - avail_start = 0x%lx\n", 425 VM_PHYSMEM_PTR(lcv)->avail_end - VM_PHYSMEM_PTR(lcv)->avail_start); 426 #endif 427 428 if (VM_PHYSMEM_PTR(lcv)->avail_start != VM_PHYSMEM_PTR(lcv)->start || /* XXX: ??? */ 429 VM_PHYSMEM_PTR(lcv)->avail_start >= VM_PHYSMEM_PTR(lcv)->avail_end) 430 continue; 431 432 /* Break off a VHPT sized, aligned chunk off this segment. */ 433 434 start1 = VM_PHYSMEM_PTR(lcv)->avail_start; 435 436 /* Align requested start address on requested size boundary */ 437 end1 = vhpt_start = roundup(start1, npgs); 438 439 start2 = vhpt_start + npgs; 440 end2 = VM_PHYSMEM_PTR(lcv)->avail_end; 441 442 443 /* Case 1: Doesn't fit. skip this segment */ 444 445 if (start2 > end2) { 446 vhpt_start = 0; 447 continue; 448 } 449 450 /* For all cases of fit: 451 * - Remove segment. 452 * - Re-insert fragments via uvm_page_physload(); 453 */ 454 455 /* 456 * We _fail_ on a vhpt request which exhausts memory. 457 */ 458 if (start1 == end1 && 459 start2 == end2 && 460 vm_nphysseg == 1) { 461 #ifdef DEBUG 462 printf("pmap_vhpt_steal_memory: out of memory!"); 463 #endif 464 return -1; 465 } 466 467 /* Remove this segment from the list. */ 468 vm_nphysseg--; 469 // physmem -= end2 - start1; 470 for (x = lcv; x < vm_nphysseg; x++) { 471 /* structure copy */ 472 VM_PHYSMEM_PTR_SWAP(x, x + 1); 473 } 474 475 /* Case 2: Perfect fit - skip segment reload. */ 476 477 if (start1 == end1 && start2 == end2) break; 478 479 /* Case 3: Left unfit - reload it. 480 */ 481 482 if (start1 != end1) { 483 uvm_page_physload(start1, end1, start1, end1, 484 VM_FREELIST_DEFAULT); 485 } 486 487 /* Case 4: Right unfit - reload it. */ 488 489 if (start2 != end2) { 490 uvm_page_physload(start2, end2, start2, end2, 491 VM_FREELIST_DEFAULT); 492 } 493 494 /* Case 5: Both unfit - Redundant, isn't it ? */ 495 break; 496 } 497 498 /* 499 * If we got here, we couldn't find a fit. 500 */ 501 if (vhpt_start == 0) { 502 #ifdef DEBUG 503 printf("pmap_steal_vhpt_memory: no VHPT aligned fit found."); 504 #endif 505 return -1; 506 } 507 508 /* 509 * There are enough pages here; steal them! 510 */ 511 pa = ptoa(vhpt_start); 512 va = IA64_PHYS_TO_RR7(pa); 513 memset((void *)va, 0, size); 514 pmap_pages_stolen += npgs; 515 return va; 516 } 517 518 519 520 521 522 /* 523 * pmap_bootstrap: 524 * 525 * Bootstrap the system to run with virtual memory. 526 * 527 * Note: no locking is necessary in this function. 528 */ 529 void 530 pmap_bootstrap(void) 531 { 532 struct ia64_pal_result res; 533 vaddr_t base, limit; 534 size_t size; 535 vsize_t bufsz; 536 537 int i, ridbits; 538 539 /* 540 * Query the PAL Code to find the loop parameters for the 541 * ptc.e instruction. 542 */ 543 res = ia64_call_pal_static(PAL_PTCE_INFO, 0, 0, 0); 544 if (res.pal_status != 0) 545 panic("Can't configure ptc.e parameters"); 546 pmap_ptc_e_base = res.pal_result[0]; 547 pmap_ptc_e_count1 = res.pal_result[1] >> 32; 548 pmap_ptc_e_count2 = res.pal_result[1] & ((1L<<32) - 1); 549 pmap_ptc_e_stride1 = res.pal_result[2] >> 32; 550 pmap_ptc_e_stride2 = res.pal_result[2] & ((1L<<32) - 1); 551 if (bootverbose) 552 printf("ptc.e base=0x%lx, count1=%ld, count2=%ld, " 553 "stride1=0x%lx, stride2=0x%lx\n", 554 pmap_ptc_e_base, 555 pmap_ptc_e_count1, 556 pmap_ptc_e_count2, 557 pmap_ptc_e_stride1, 558 pmap_ptc_e_stride2); 559 mutex_init(&pmap_ptc_lock, MUTEX_DEFAULT, IPL_VM); 560 561 /* 562 * Setup RIDs. RIDs 0..7 are reserved for the kernel. 563 * 564 * We currently need at least 19 bits in the RID because PID_MAX 565 * can only be encoded in 17 bits and we need RIDs for 5 regions 566 * per process. With PID_MAX equalling 99999 this means that we 567 * need to be able to encode 499995 (=5*PID_MAX). 568 * The Itanium processor only has 18 bits and the architected 569 * minimum is exactly that. So, we cannot use a PID based scheme 570 * in those cases. Enter pmap_ridmap... 571 * We should avoid the map when running on a processor that has 572 * implemented enough bits. This means that we should pass the 573 * process/thread ID to pmap. This we currently don't do, so we 574 * use the map anyway. However, we don't want to allocate a map 575 * that is large enough to cover the range dictated by the number 576 * of bits in the RID, because that may result in a RID map of 577 * 2MB in size for a 24-bit RID. A 64KB map is enough. 578 * The bottomline: we create a 32KB map when the processor only 579 * implements 18 bits (or when we can't figure it out). Otherwise 580 * we create a 64KB map. 581 */ 582 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0); 583 if (res.pal_status != 0) { 584 if (bootverbose) 585 printf("Can't read VM Summary - assuming 18 Region ID bits\n"); 586 ridbits = 18; /* guaranteed minimum */ 587 } else { 588 ridbits = (res.pal_result[1] >> 8) & 0xff; 589 if (bootverbose) 590 printf("Processor supports %d Region ID bits\n", 591 ridbits); 592 } 593 if (ridbits > 19) 594 ridbits = 19; 595 596 pmap_ridmax = (1 << ridbits); 597 pmap_ridmapsz = pmap_ridmax / 64; 598 pmap_ridmap = (uint64_t *)uvm_pageboot_alloc(pmap_ridmax / 8); 599 pmap_ridmap[0] |= 0xff; 600 pmap_rididx = 0; 601 pmap_ridcount = 8; 602 603 /* XXX: The FreeBSD pmap.c defines initialises this like this: 604 * mtx_init(&pmap_ridmutex, "RID allocator lock", NULL, MTX_DEF); 605 * MTX_DEF can *sleep*. 606 */ 607 mutex_init(&pmap_rid_lock, MUTEX_DEFAULT, IPL_VM); 608 609 610 /* 611 * Compute the number of pages kmem_map will have. 612 */ 613 kmeminit_nkmempages(); 614 615 /* 616 * Figure out how many initial PTE's are necessary to map the 617 * kernel. We also reserve space for kmem_alloc_pageable() 618 * for vm_fork(). 619 */ 620 621 /* Get size of buffer cache and set an upper limit */ 622 bufsz = buf_memcalc(); 623 buf_setvalimit(bufsz); 624 625 nkpt = (((ubc_nwins << ubc_winshift) + uvm_emap_size + 626 bufsz + 16 * NCARGS + pager_map_size) / PAGE_SIZE + 627 USRIOSIZE + (maxproc * UPAGES) + nkmempages) / NKPTEPG; 628 629 /* 630 * Allocate some memory for initial kernel 'page tables'. 631 */ 632 ia64_kptdir = (void *)uvm_pageboot_alloc((nkpt + 1) * PAGE_SIZE); 633 for (i = 0; i < nkpt; i++) { 634 ia64_kptdir[i] = (void*)( (vaddr_t)ia64_kptdir + PAGE_SIZE * (i + 1)); 635 } 636 637 kernel_vm_end = nkpt * PAGE_SIZE * NKPTEPG + VM_MIN_KERNEL_ADDRESS - 638 VM_GATEWAY_SIZE; 639 640 /* 641 * Initialize the pmap pools and list. 642 */ 643 pmap_ncpuids = pmap_ridmax; 644 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 645 &pool_allocator_nointr, IPL_NONE); /* This may block. */ 646 647 /* XXX: Need to convert ia64_kptdir[][] to a pool. ????*/ 648 649 /* The default pool allocator uses uvm_km_alloc & friends. 650 * XXX: We should be using regular vm_alloced mem for regular, non-kernel ptesl 651 */ 652 653 pool_init(&pmap_ia64_lpte_pool, sizeof (struct ia64_lpte), 654 sizeof(void *), 0, 0, "ptpl", NULL, IPL_NONE); 655 656 pool_init(&pmap_pv_pool, sizeof (struct pv_entry), sizeof(void *), 657 0, 0, "pvpl", &pmap_pv_page_allocator, IPL_NONE); 658 659 TAILQ_INIT(&pmap_all_pmaps); 660 661 662 /* 663 * Figure out a useful size for the VHPT, based on the size of 664 * physical memory and try to locate a region which is large 665 * enough to contain the VHPT (which must be a power of two in 666 * size and aligned to a natural boundary). 667 * We silently bump up the VHPT size to the minimum size if the 668 * user has set the tunable too small. Likewise, the VHPT size 669 * is silently capped to the maximum allowed. 670 */ 671 672 pmap_vhpt_log2size = PMAP_VHPT_LOG2SIZE; 673 674 if (pmap_vhpt_log2size == 0) { 675 pmap_vhpt_log2size = 15; 676 size = 1UL << pmap_vhpt_log2size; 677 while (size < physmem * 32) { 678 pmap_vhpt_log2size++; 679 size <<= 1; 680 } 681 } 682 else 683 if (pmap_vhpt_log2size < 15) pmap_vhpt_log2size = 15; 684 685 if (pmap_vhpt_log2size > 61) pmap_vhpt_log2size = 61; 686 687 vhpt_base = 0; 688 base = limit = 0; 689 size = 1UL << pmap_vhpt_log2size; 690 while (vhpt_base == 0 && size) { 691 if (bootverbose) 692 printf("Trying VHPT size 0x%lx\n", size); 693 694 /* allocate size bytes aligned at size */ 695 /* #ifdef MULTIPROCESSOR, then (size * MAXCPU) bytes */ 696 base = pmap_steal_vhpt_memory(size); 697 698 if (!base) { 699 /* Can't fit, try next smaller size. */ 700 pmap_vhpt_log2size--; 701 size >>= 1; 702 } else 703 vhpt_base = IA64_PHYS_TO_RR7(base); 704 } 705 if (pmap_vhpt_log2size < 15) 706 panic("Can't find space for VHPT"); 707 708 if (bootverbose) 709 printf("Putting VHPT at 0x%lx\n", base); 710 711 mutex_init(&pmap_vhptlock, MUTEX_DEFAULT, IPL_VM); 712 713 __asm __volatile("mov cr.pta=%0;; srlz.i;;" :: 714 "r" (vhpt_base + (1<<8) + (pmap_vhpt_log2size<<2) + 1)); 715 716 #ifdef DEBUG 717 dump_vhpt(); 718 #endif 719 720 /* 721 * Initialise vhpt pte entries. 722 */ 723 724 pmap_vhpt_nbuckets = size / sizeof(struct ia64_lpte); 725 726 pmap_vhpt_bucket = (void *)uvm_pageboot_alloc(pmap_vhpt_nbuckets * 727 sizeof(struct ia64_bucket)); 728 729 struct ia64_lpte *pte; 730 731 pte = (struct ia64_lpte *)vhpt_base; 732 for (i = 0; i < pmap_vhpt_nbuckets; i++) { 733 pte[i].pte = 0; 734 pte[i].itir = 0; 735 pte[i].tag = 1UL << 63; /* Invalid tag */ 736 pte[i].chain = (uintptr_t)(pmap_vhpt_bucket + i); 737 /* Stolen memory is zeroed! */ 738 mutex_init(&pmap_vhpt_bucket[i].lock, MUTEX_DEFAULT, 739 IPL_VM); 740 } 741 742 /* 743 * Initialize the locks. 744 */ 745 mutex_init(&pmap_main_lock, MUTEX_DEFAULT, IPL_VM); 746 mutex_init(&pmap_all_pmaps_slock, MUTEX_DEFAULT, IPL_VM); 747 748 /* 749 * Initialize the kernel pmap (which is statically allocated). 750 */ 751 memset(pmap_kernel(), 0, sizeof(struct pmap)); 752 753 mutex_init(&pmap_kernel()->pm_slock, MUTEX_DEFAULT, IPL_VM); 754 for (i = 0; i < 5; i++) 755 pmap_kernel()->pm_rid[i] = 0; 756 pmap_kernel()->pm_active = 1; 757 TAILQ_INIT(&pmap_kernel()->pm_pvlist); 758 759 TAILQ_INSERT_TAIL(&pmap_all_pmaps, pmap_kernel(), pm_list); 760 761 /* 762 * Region 5 is mapped via the vhpt. 763 */ 764 ia64_set_rr(IA64_RR_BASE(5), 765 (5 << 8) | (PAGE_SHIFT << 2) | 1); 766 767 /* 768 * Region 6 is direct mapped UC and region 7 is direct mapped 769 * WC. The details of this is controlled by the Alt {I,D}TLB 770 * handlers. Here we just make sure that they have the largest 771 * possible page size to minimise TLB usage. 772 */ 773 ia64_set_rr(IA64_RR_BASE(6), (6 << 8) | (IA64_ID_PAGE_SHIFT << 2)); 774 ia64_set_rr(IA64_RR_BASE(7), (7 << 8) | (IA64_ID_PAGE_SHIFT << 2)); 775 ia64_srlz_d(); 776 777 /* 778 * Clear out any random TLB entries left over from booting. 779 */ 780 pmap_invalidate_all(pmap_kernel()); 781 782 map_gateway_page(); 783 } 784 785 786 /* 787 * pmap_init: [ INTERFACE ] 788 * 789 * Initialize the pmap module. Called by vm_init(), to initialize any 790 * structures that the pmap system needs to map virtual memory. 791 * 792 * Note: no locking is necessary in this function. 793 */ 794 void 795 pmap_init(void) 796 { 797 798 799 /* 800 * Set a low water mark on the pv_entry pool, so that we are 801 * more likely to have these around even in extreme memory 802 * starvation. 803 */ 804 pool_setlowat(&pmap_pv_pool, pmap_pv_lowat); 805 806 807 /* 808 * Now it is safe to enable pv entry recording. 809 */ 810 pmap_initialized = true; 811 812 } 813 814 815 /* 816 * vtophys: virtual address to physical address. For use by 817 * machine-dependent code only. 818 */ 819 820 paddr_t 821 vtophys(vaddr_t va) 822 { 823 paddr_t pa; 824 825 if (pmap_extract(pmap_kernel(), va, &pa) == true) 826 return pa; 827 return 0; 828 } 829 830 /* 831 * pmap_virtual_space: [ INTERFACE ] 832 * 833 * Define the initial bounds of the kernel virtual address space. 834 */ 835 void 836 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp) 837 { 838 *vstartp = VM_MIN_KERNEL_ADDRESS; 839 *vendp = VM_MAX_KERNEL_ADDRESS; 840 } 841 842 /* 843 * pmap_remove_all: [ INTERFACE ] 844 * 845 * This function is a hint to the pmap implementation that all 846 * entries in pmap will be removed before any more entries are 847 * entered. 848 */ 849 850 void 851 pmap_remove_all(pmap_t pmap) 852 { 853 /* Nothing Yet */ 854 } 855 856 /* 857 * pmap_remove: [ INTERFACE ] 858 * 859 * Remove the given range of addresses from the specified map. 860 * 861 * It is assumed that the start and end are properly 862 * rounded to the page size. 863 */ 864 void 865 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva) 866 { 867 pmap_t oldpmap; 868 vaddr_t va; 869 pv_entry_t pv; 870 struct ia64_lpte *pte; 871 872 if (pmap->pm_stats.resident_count == 0) 873 return; 874 875 PMAP_MAP_TO_HEAD_LOCK(); 876 PMAP_LOCK(pmap); 877 oldpmap = pmap_install(pmap); 878 879 /* 880 * special handling of removing one page. a very 881 * common operation and easy to short circuit some 882 * code. 883 */ 884 if (sva + PAGE_SIZE == eva) { 885 pmap_remove_page(pmap, sva); 886 goto out; 887 } 888 889 if (pmap->pm_stats.resident_count < ((eva - sva) >> PAGE_SHIFT)) { 890 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) { 891 va = pv->pv_va; 892 if (va >= sva && va < eva) { 893 pte = pmap_find_vhpt(va); 894 KASSERT(pte != NULL); 895 pmap_remove_pte(pmap, pte, va, pv, 1); 896 pmap_invalidate_page(pmap, va); 897 } 898 } 899 900 } else { 901 for (va = sva; va < eva; va += PAGE_SIZE) { 902 pte = pmap_find_vhpt(va); 903 if (pte != NULL) { 904 pmap_remove_pte(pmap, pte, va, 0, 1); 905 pmap_invalidate_page(pmap, va); 906 } 907 } 908 } 909 910 out: 911 pmap_install(oldpmap); 912 PMAP_UNLOCK(pmap); 913 PMAP_MAP_TO_HEAD_UNLOCK(); 914 915 } 916 917 918 919 920 /* 921 * pmap_zero_page: [ INTERFACE ] 922 * 923 * Zero the specified (machine independent) page by mapping the page 924 * into virtual memory and clear its contents, one machine dependent 925 * page at a time. 926 * 927 * Note: no locking is necessary in this function. 928 */ 929 void 930 pmap_zero_page(paddr_t phys) 931 { 932 vaddr_t va = IA64_PHYS_TO_RR7(phys); 933 memset((void *) va, 0, PAGE_SIZE); 934 } 935 936 /* 937 * pmap_copy_page: [ INTERFACE ] 938 * 939 * Copy the specified (machine independent) page by mapping the page 940 * into virtual memory and using memcpy to copy the page, one machine 941 * dependent page at a time. 942 * 943 * Note: no locking is necessary in this function. 944 */ 945 void 946 pmap_copy_page(paddr_t psrc, paddr_t pdst) 947 { 948 vaddr_t vsrc = IA64_PHYS_TO_RR7(psrc); 949 vaddr_t vdst = IA64_PHYS_TO_RR7(pdst); 950 memcpy((void *) vdst, (void *) vsrc, PAGE_SIZE); 951 } 952 953 /* 954 * pmap_unwire: [ INTERFACE ] 955 * 956 * Clear the wired attribute for a map/virtual-address pair. 957 * 958 * The mapping must already exist in the pmap. 959 */ 960 void 961 pmap_unwire(pmap_t pmap, vaddr_t va) 962 { 963 pmap_t oldpmap; 964 struct ia64_lpte *pte; 965 966 if (pmap == NULL) 967 return; 968 969 PMAP_LOCK(pmap); 970 oldpmap = pmap_install(pmap); 971 972 pte = pmap_find_vhpt(va); 973 974 KASSERT(pte != NULL); 975 976 /* 977 * If wiring actually changed (always?) clear the wire bit and 978 * update the wire count. Note that wiring is not a hardware 979 * characteristic so there is no need to invalidate the TLB. 980 */ 981 982 if (pmap_wired(pte)) { 983 pmap->pm_stats.wired_count--; 984 pmap_clear_wired(pte); 985 } 986 #ifdef DIAGNOSTIC 987 else { 988 printf("pmap_unwire: wiring for pmap %p va 0x%lx " 989 "didn't change!\n", pmap, va); 990 } 991 #endif 992 pmap_install(oldpmap); 993 PMAP_UNLOCK(pmap); 994 } 995 996 997 /* 998 * pmap_kenter_pa: [ INTERFACE ] 999 * 1000 * Enter a va -> pa mapping into the kernel pmap without any 1001 * physical->virtual tracking. 1002 * 1003 * Note: no locking is necessary in this function. 1004 */ 1005 void 1006 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1007 { 1008 struct ia64_lpte *pte; 1009 1010 pte = pmap_find_kpte(va); 1011 if (pmap_present(pte)) 1012 pmap_invalidate_page(pmap_kernel(), va); 1013 else 1014 pmap_enter_vhpt(pte, va); 1015 pmap_pte_prot(pmap_kernel(), pte, prot); 1016 pmap_set_pte(pte, va, pa, false, false); 1017 1018 } 1019 1020 1021 /* 1022 * pmap_kremove: [ INTERFACE ] 1023 * 1024 * Remove a mapping entered with pmap_kenter_pa() starting at va, 1025 * for size bytes (assumed to be page rounded). 1026 */ 1027 void 1028 pmap_kremove(vaddr_t va, vsize_t size) 1029 { 1030 struct ia64_lpte *pte; 1031 1032 pte = pmap_find_kpte(va); 1033 if (pmap_present(pte)) { 1034 pmap_remove_vhpt(va); 1035 pmap_invalidate_page(pmap_kernel(), va); 1036 pmap_clear_present(pte); 1037 } 1038 } 1039 1040 1041 1042 /* 1043 * pmap_create: [ INTERFACE ] 1044 * 1045 * Create and return a physical map. 1046 * 1047 * Note: no locking is necessary in this function. 1048 */ 1049 pmap_t 1050 pmap_create(void) 1051 { 1052 pmap_t pmap; 1053 int i; 1054 1055 #ifdef DEBUG 1056 printf("pmap_create()\n"); 1057 #endif 1058 1059 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1060 memset(pmap, 0, sizeof(*pmap)); 1061 1062 for (i = 0; i < 5; i++) 1063 pmap->pm_rid[i] = pmap_allocate_rid(); 1064 pmap->pm_active = 0; 1065 TAILQ_INIT(&pmap->pm_pvlist); 1066 memset(&pmap->pm_stats, 0, sizeof (pmap->pm_stats) ); 1067 1068 mutex_init(&pmap->pm_slock, MUTEX_DEFAULT, IPL_VM); 1069 1070 mutex_enter(&pmap_all_pmaps_slock); 1071 TAILQ_INSERT_TAIL(&pmap_all_pmaps, pmap, pm_list); 1072 mutex_exit(&pmap_all_pmaps_slock); 1073 1074 return pmap; 1075 } 1076 1077 /* 1078 * pmap_destroy: [ INTERFACE ] 1079 * 1080 * Drop the reference count on the specified pmap, releasing 1081 * all resources if the reference count drops to zero. 1082 */ 1083 void 1084 pmap_destroy(pmap_t pmap) 1085 { 1086 int i; 1087 1088 #ifdef DEBUG 1089 printf("pmap_destroy(%p)\n", pmap); 1090 #endif 1091 1092 for (i = 0; i < 5; i++) 1093 if (pmap->pm_rid[i]) 1094 pmap_free_rid(pmap->pm_rid[i]); 1095 /* 1096 * Remove it from the global list of all pmaps. 1097 */ 1098 mutex_enter(&pmap_all_pmaps_slock); 1099 TAILQ_REMOVE(&pmap_all_pmaps, pmap, pm_list); 1100 mutex_exit(&pmap_all_pmaps_slock); 1101 1102 pool_put(&pmap_pmap_pool, pmap); 1103 1104 } 1105 1106 1107 /* 1108 * pmap_activate: [ INTERFACE ] 1109 * 1110 * Activate the pmap used by the specified process. This includes 1111 * reloading the MMU context if the current process, and marking 1112 * the pmap in use by the processor. 1113 * 1114 * Note: We may use only spin locks here, since we are called 1115 * by a critical section in cpu_switch()! 1116 */ 1117 void 1118 pmap_activate(struct lwp *l) 1119 { 1120 pmap_install(vm_map_pmap(&l->l_proc->p_vmspace->vm_map)); 1121 } 1122 1123 /* 1124 * pmap_deactivate: [ INTERFACE ] 1125 * 1126 * Mark that the pmap used by the specified process is no longer 1127 * in use by the processor. 1128 * 1129 */ 1130 1131 void 1132 pmap_deactivate(struct lwp *l) 1133 { 1134 } 1135 1136 /* 1137 * pmap_protect: [ INTERFACE ] 1138 * 1139 * Set the physical protection on the specified range of this map 1140 * as requested. 1141 */ 1142 /* 1143 * Set the physical protection on the 1144 * specified range of this map as requested. 1145 */ 1146 void 1147 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1148 { 1149 pmap_t oldpmap; 1150 struct ia64_lpte *pte; 1151 vaddr_t pa; 1152 struct vm_page *pg; 1153 1154 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1155 pmap_remove(pmap, sva, eva); 1156 return; 1157 } 1158 1159 if (prot & VM_PROT_WRITE) 1160 return; 1161 1162 if ((sva & PAGE_MASK) || (eva & PAGE_MASK)) 1163 panic("pmap_protect: unaligned addresses"); 1164 1165 //uvm_lock_pageq(); 1166 PMAP_LOCK(pmap); 1167 oldpmap = pmap_install(pmap); 1168 while (sva < eva) { 1169 /* 1170 * If page is invalid, skip this page 1171 */ 1172 pte = pmap_find_vhpt(sva); 1173 if (pte == NULL) { 1174 sva += PAGE_SIZE; 1175 continue; 1176 } 1177 1178 if (pmap_prot(pte) != prot) { 1179 if (pmap_managed(pte)) { 1180 pa = pmap_ppn(pte); 1181 pg = PHYS_TO_VM_PAGE(pa); 1182 if (pmap_dirty(pte)) pmap_clear_dirty(pte); 1183 if (pmap_accessed(pte)) { 1184 pmap_clear_accessed(pte); 1185 } 1186 } 1187 pmap_pte_prot(pmap, pte, prot); 1188 pmap_invalidate_page(pmap, sva); 1189 } 1190 1191 sva += PAGE_SIZE; 1192 } 1193 //uvm_unlock_pageq(); 1194 pmap_install(oldpmap); 1195 PMAP_UNLOCK(pmap); 1196 } 1197 1198 1199 /* 1200 * pmap_extract: [ INTERFACE ] 1201 * 1202 * Extract the physical address associated with the given 1203 * pmap/virtual address pair. 1204 */ 1205 bool 1206 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap) 1207 { 1208 struct ia64_lpte *pte; 1209 pmap_t oldpmap; 1210 paddr_t pa; 1211 1212 pa = 0; 1213 mutex_enter(&pmap->pm_slock); 1214 oldpmap = pmap_install(pmap); /* XXX: isn't this a little inefficient ? */ 1215 pte = pmap_find_vhpt(va); 1216 if (pte != NULL && pmap_present(pte)) 1217 pap = (paddr_t *) pmap_ppn(pte); 1218 else { 1219 mutex_exit(&pmap->pm_slock); 1220 return false; 1221 } 1222 pmap_install(oldpmap); 1223 mutex_exit(&pmap->pm_slock); 1224 return true; 1225 1226 } 1227 1228 /* 1229 * pmap_clear_modify: [ INTERFACE ] 1230 * 1231 * Clear the modify bits on the specified physical page. 1232 */ 1233 bool 1234 pmap_clear_modify(struct vm_page *pg) 1235 { 1236 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 1237 bool rv = false; 1238 struct ia64_lpte *pte; 1239 pmap_t oldpmap; 1240 pv_entry_t pv; 1241 1242 if (pg->flags & PG_FAKE) 1243 return rv; 1244 1245 TAILQ_FOREACH(pv, &md->pv_list, pv_list) { 1246 PMAP_LOCK(pv->pv_pmap); 1247 oldpmap = pmap_install(pv->pv_pmap); 1248 pte = pmap_find_vhpt(pv->pv_va); 1249 KASSERT(pte != NULL); 1250 if (pmap_dirty(pte)) { 1251 rv = true; 1252 pmap_clear_dirty(pte); 1253 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 1254 } 1255 pmap_install(oldpmap); 1256 PMAP_UNLOCK(pv->pv_pmap); 1257 } 1258 return rv; 1259 } 1260 1261 /* 1262 * pmap_page_protect: [ INTERFACE ] 1263 * 1264 * Lower the permission for all mappings to a given page to 1265 * the permissions specified. 1266 */ 1267 void 1268 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1269 { 1270 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 1271 struct ia64_lpte *pte; 1272 pmap_t oldpmap, pmap; 1273 pv_entry_t pv; 1274 1275 if ((prot & VM_PROT_WRITE) != 0) 1276 return; 1277 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) { 1278 if (pg->flags & PG_RDONLY) 1279 return; 1280 TAILQ_FOREACH(pv, &md->pv_list, pv_list) { 1281 pmap = pv->pv_pmap; 1282 PMAP_LOCK(pmap); 1283 oldpmap = pmap_install(pmap); 1284 pte = pmap_find_vhpt(pv->pv_va); 1285 KASSERT(pte != NULL); 1286 pmap_pte_prot(pmap, pte, prot); 1287 pmap_invalidate_page(pmap, pv->pv_va); 1288 pmap_install(oldpmap); 1289 PMAP_UNLOCK(pmap); 1290 } 1291 1292 //UVM_LOCK_ASSERT_PAGEQ(); 1293 1294 pg->flags |= PG_RDONLY; 1295 } else { 1296 pmap_page_purge(pg); 1297 } 1298 } 1299 1300 /* 1301 * pmap_reference: [ INTERFACE ] 1302 * 1303 * Add a reference to the specified pmap. 1304 */ 1305 void 1306 pmap_reference(pmap_t pmap) 1307 { 1308 1309 #ifdef DEBUG 1310 printf("pmap_reference(%p)\n", pmap); 1311 #endif 1312 1313 PMAP_LOCK(pmap); 1314 pmap->pm_count++; 1315 PMAP_UNLOCK(pmap); 1316 } 1317 1318 /* 1319 * pmap_clear_reference: [ INTERFACE ] 1320 * 1321 * Clear the reference bit on the specified physical page. 1322 */ 1323 bool 1324 pmap_clear_reference(struct vm_page *pg) 1325 { 1326 return false; 1327 } 1328 1329 /* 1330 * pmap_phys_address: [ INTERFACE ] 1331 * 1332 * Return the physical address corresponding to the specified 1333 * cookie. Used by the device pager to decode a device driver's 1334 * mmap entry point return value. 1335 * 1336 * Note: no locking is necessary in this function. 1337 */ 1338 paddr_t 1339 pmap_phys_address(paddr_t ppn) 1340 { 1341 1342 return ia64_ptob(ppn); 1343 } 1344 1345 1346 /* 1347 * pmap_enter: [ INTERFACE ] 1348 * 1349 * Insert the given physical page (p) at 1350 * the specified virtual address (v) in the 1351 * target physical map with the protection requested. 1352 * 1353 * If specified, the page will be wired down, meaning 1354 * that the related pte can not be reclaimed. 1355 * 1356 * Note: This is the only routine which MAY NOT lazy-evaluate 1357 * or lose information. That is, this routine must actually 1358 * insert this page into the given map NOW. 1359 */ 1360 int 1361 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1362 { 1363 pmap_t oldpmap; 1364 vaddr_t opa; 1365 struct ia64_lpte origpte; 1366 struct ia64_lpte *pte; 1367 bool managed, wired; 1368 struct vm_page *pg; 1369 int error = 0; 1370 1371 PMAP_MAP_TO_HEAD_LOCK(); 1372 PMAP_LOCK(pmap); 1373 oldpmap = pmap_install(pmap); 1374 1375 va &= ~PAGE_MASK; 1376 1377 managed = false; 1378 1379 wired = (flags & PMAP_WIRED) !=0; 1380 1381 pg = PHYS_TO_VM_PAGE(pa); 1382 1383 1384 1385 #ifdef DIAGNOSTIC 1386 if (va > VM_MAX_KERNEL_ADDRESS) 1387 panic("pmap_enter: toobig"); 1388 #endif 1389 1390 /* 1391 * Find (or create) a pte for the given mapping. 1392 */ 1393 while ((pte = pmap_find_pte(va)) == NULL) { 1394 pmap_install(oldpmap); 1395 PMAP_UNLOCK(pmap); 1396 PMAP_MAP_TO_HEAD_UNLOCK(); 1397 uvm_kick_pdaemon(); 1398 PMAP_MAP_TO_HEAD_LOCK(); 1399 PMAP_LOCK(pmap); 1400 oldpmap = pmap_install(pmap); 1401 } 1402 origpte = *pte; 1403 if (!pmap_present(pte)) { 1404 opa = ~0UL; 1405 pmap_enter_vhpt(pte, va); 1406 } else 1407 opa = pmap_ppn(pte); 1408 1409 /* 1410 * Mapping has not changed, must be protection or wiring change. 1411 */ 1412 if (opa == pa) { 1413 /* 1414 * Wiring change, just update stats. We don't worry about 1415 * wiring PT pages as they remain resident as long as there 1416 * are valid mappings in them. Hence, if a user page is wired, 1417 * the PT page will be also. 1418 */ 1419 if (wired && !pmap_wired(&origpte)) 1420 pmap->pm_stats.wired_count++; 1421 else if (!wired && pmap_wired(&origpte)) 1422 pmap->pm_stats.wired_count--; 1423 1424 managed = (pmap_managed(&origpte)) ? true : false; 1425 1426 1427 /* 1428 * We might be turning off write access to the page, 1429 * so we go ahead and sense modify status. 1430 */ 1431 if (managed && pmap_dirty(&origpte) && pmap_track_modified(va)) 1432 pg->flags &= ~PG_CLEAN; 1433 1434 pmap_invalidate_page(pmap, va); 1435 goto validate; 1436 } 1437 1438 /* 1439 * Mapping has changed, invalidate old range and fall 1440 * through to handle validating new mapping. 1441 */ 1442 if (opa != ~0UL) { 1443 pmap_remove_pte(pmap, pte, va, 0, 0); 1444 pmap_enter_vhpt(pte, va); 1445 } 1446 1447 /* 1448 * Enter on the PV list if part of our managed memory. 1449 */ 1450 1451 if ((flags & (PG_FAKE)) == 0) { 1452 pmap_insert_entry(pmap, va, pg); 1453 managed = true; 1454 } 1455 1456 /* 1457 * Increment counters 1458 */ 1459 pmap->pm_stats.resident_count++; 1460 if (wired) 1461 pmap->pm_stats.wired_count++; 1462 1463 validate: 1464 1465 /* 1466 * Now validate mapping with desired protection/wiring. This 1467 * adds the pte to the VHPT if necessary. 1468 */ 1469 pmap_pte_prot(pmap, pte, prot); 1470 pmap_set_pte(pte, va, pa, wired, managed); 1471 1472 PMAP_MAP_TO_HEAD_UNLOCK(); 1473 pmap_install(oldpmap); 1474 PMAP_UNLOCK(pmap); 1475 1476 return error; /* XXX: Look into this. */ 1477 } 1478 1479 1480 /* 1481 * Routine: pmap_page_purge: => was: pmap_remove_all 1482 * Function: 1483 * Removes this physical page from 1484 * all physical maps in which it resides. 1485 * Reflects back modify bits to the pager. 1486 * 1487 * Notes: 1488 * Original versions of this routine were very 1489 * inefficient because they iteratively called 1490 * pmap_remove (slow...) 1491 */ 1492 1493 void 1494 pmap_page_purge(struct vm_page *pg) 1495 { 1496 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 1497 pmap_t oldpmap; 1498 pv_entry_t pv; 1499 1500 #if defined(DIAGNOSTIC) 1501 /* 1502 * XXX this makes pmap_page_protect(NONE) illegal for non-managed 1503 * pages! 1504 */ 1505 if (pg->flags & PG_FAKE) { 1506 panic("pmap_page_protect: illegal for unmanaged page, va: 0x%lx", VM_PAGE_TO_PHYS(pg)); 1507 } 1508 #endif 1509 //UVM_LOCK_ASSERT_PAGEQ(); 1510 1511 while ((pv = TAILQ_FIRST(&md->pv_list)) != NULL) { 1512 struct ia64_lpte *pte; 1513 pmap_t pmap = pv->pv_pmap; 1514 vaddr_t va = pv->pv_va; 1515 1516 PMAP_LOCK(pmap); 1517 oldpmap = pmap_install(pmap); 1518 pte = pmap_find_vhpt(va); 1519 KASSERT(pte != NULL); 1520 if (pmap_ppn(pte) != VM_PAGE_TO_PHYS(pg)) 1521 panic("pmap_remove_all: pv_table for %lx is inconsistent", VM_PAGE_TO_PHYS(pg)); 1522 pmap_remove_pte(pmap, pte, va, pv, 1); 1523 pmap_install(oldpmap); 1524 PMAP_UNLOCK(pmap); 1525 } 1526 1527 //UVM_LOCK_ASSERT_PAGEQ(); 1528 pg->flags |= PG_RDONLY; 1529 1530 } 1531 1532 1533 pmap_t 1534 pmap_switch(pmap_t pm) 1535 { 1536 pmap_t prevpm; 1537 int i; 1538 1539 //LOCK_ASSERT(simple_lock_held(&sched_lock)); 1540 1541 prevpm = curcpu()->ci_pmap; 1542 if (prevpm == pm) 1543 return prevpm; 1544 // if (prevpm != NULL) 1545 // atomic_clear_32(&prevpm->pm_active, PCPU_GET(cpumask)); 1546 if (pm == NULL) { 1547 for (i = 0; i < 5; i++) { 1548 ia64_set_rr(IA64_RR_BASE(i), 1549 (i << 8)|(PAGE_SHIFT << 2)|1); 1550 } 1551 } else { 1552 for (i = 0; i < 5; i++) { 1553 ia64_set_rr(IA64_RR_BASE(i), 1554 (pm->pm_rid[i] << 8)|(PAGE_SHIFT << 2)|1); 1555 } 1556 // atomic_set_32(&pm->pm_active, PCPU_GET(cpumask)); 1557 } 1558 curcpu()->ci_pmap = pm; 1559 ia64_srlz_d(); 1560 return prevpm; 1561 } 1562 1563 static pmap_t 1564 pmap_install(pmap_t pm) 1565 { 1566 pmap_t prevpm; 1567 1568 int splsched; 1569 1570 splsched = splsched(); 1571 prevpm = pmap_switch(pm); 1572 splx(splsched); 1573 return prevpm; 1574 } 1575 1576 static uint32_t 1577 pmap_allocate_rid(void) 1578 { 1579 uint64_t bit, bits; 1580 int rid; 1581 1582 mutex_enter(&pmap_rid_lock); 1583 if (pmap_ridcount == pmap_ridmax) 1584 panic("pmap_allocate_rid: All Region IDs used"); 1585 1586 /* Find an index with a free bit. */ 1587 while ((bits = pmap_ridmap[pmap_rididx]) == ~0UL) { 1588 pmap_rididx++; 1589 if (pmap_rididx == pmap_ridmapsz) 1590 pmap_rididx = 0; 1591 } 1592 rid = pmap_rididx * 64; 1593 1594 /* Find a free bit. */ 1595 bit = 1UL; 1596 while (bits & bit) { 1597 rid++; 1598 bit <<= 1; 1599 } 1600 1601 pmap_ridmap[pmap_rididx] |= bit; 1602 pmap_ridcount++; 1603 mutex_exit(&pmap_rid_lock); 1604 1605 return rid; 1606 } 1607 1608 static void 1609 pmap_free_rid(uint32_t rid) 1610 { 1611 uint64_t bit; 1612 int idx; 1613 1614 idx = rid / 64; 1615 bit = ~(1UL << (rid & 63)); 1616 1617 mutex_enter(&pmap_rid_lock); 1618 pmap_ridmap[idx] &= bit; 1619 pmap_ridcount--; 1620 mutex_exit(&pmap_rid_lock); 1621 } 1622 1623 /*************************************************** 1624 * Manipulate TLBs for a pmap 1625 ***************************************************/ 1626 1627 static void 1628 pmap_invalidate_page(pmap_t pmap, vaddr_t va) 1629 { 1630 KASSERT((pmap == pmap_kernel() || pmap == curcpu()->ci_pmap)); 1631 ia64_ptc_g(va, PAGE_SHIFT << 2); 1632 } 1633 1634 static void 1635 pmap_invalidate_all_1(void *arg) 1636 { 1637 uint64_t addr; 1638 int i, j; 1639 register_t psr; 1640 1641 psr = intr_disable(); 1642 addr = pmap_ptc_e_base; 1643 for (i = 0; i < pmap_ptc_e_count1; i++) { 1644 for (j = 0; j < pmap_ptc_e_count2; j++) { 1645 ia64_ptc_e(addr); 1646 addr += pmap_ptc_e_stride2; 1647 } 1648 addr += pmap_ptc_e_stride1; 1649 } 1650 intr_restore(psr); 1651 } 1652 1653 static void 1654 pmap_invalidate_all(pmap_t pmap) 1655 { 1656 KASSERT(pmap == pmap_kernel() || pmap == curcpu()->ci_pmap); 1657 1658 1659 #ifdef MULTIPROCESSOR 1660 smp_rendezvous(0, pmap_invalidate_all_1, 0, 0); 1661 #else 1662 pmap_invalidate_all_1(0); 1663 #endif 1664 } 1665 1666 /*************************************************** 1667 * Low level mapping routines..... 1668 ***************************************************/ 1669 1670 /* 1671 * Find the kernel lpte for mapping the given virtual address, which 1672 * must be in the part of region 5 which we can cover with our kernel 1673 * 'page tables'. 1674 */ 1675 static struct ia64_lpte * 1676 pmap_find_kpte(vaddr_t va) 1677 { 1678 KASSERT((va >> 61) == 5); 1679 KASSERT(IA64_RR_MASK(va) < (nkpt * PAGE_SIZE * NKPTEPG)); 1680 return &ia64_kptdir[KPTE_DIR_INDEX(va)][KPTE_PTE_INDEX(va)]; 1681 } 1682 1683 1684 /*************************************************** 1685 * Low level helper routines..... 1686 ***************************************************/ 1687 1688 /* 1689 * Find a pte suitable for mapping a user-space address. If one exists 1690 * in the VHPT, that one will be returned, otherwise a new pte is 1691 * allocated. 1692 */ 1693 static struct ia64_lpte * 1694 pmap_find_pte(vaddr_t va) 1695 { 1696 struct ia64_lpte *pte; 1697 1698 if (va >= VM_MAXUSER_ADDRESS) 1699 return pmap_find_kpte(va); 1700 1701 pte = pmap_find_vhpt(va); 1702 if (pte == NULL) { 1703 pte = pool_get(&pmap_ia64_lpte_pool, PR_NOWAIT); 1704 pte->tag = 1UL << 63; 1705 } 1706 1707 return pte; 1708 } 1709 1710 static __inline void 1711 pmap_pte_prot(pmap_t pm, struct ia64_lpte *pte, vm_prot_t prot) 1712 { 1713 static int prot2ar[4] = { 1714 PTE_AR_R, /* VM_PROT_NONE */ 1715 PTE_AR_RW, /* VM_PROT_WRITE */ 1716 PTE_AR_RX, /* VM_PROT_EXECUTE */ 1717 PTE_AR_RWX /* VM_PROT_WRITE|VM_PROT_EXECUTE */ 1718 }; 1719 1720 pte->pte &= ~(PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK); 1721 pte->pte |= (uint64_t)(prot & VM_PROT_ALL) << 56; 1722 pte->pte |= (prot == VM_PROT_NONE || pm == pmap_kernel()) 1723 ? PTE_PL_KERN : PTE_PL_USER; 1724 pte->pte |= prot2ar[(prot & VM_PROT_ALL) >> 1]; 1725 } 1726 1727 1728 1729 /* 1730 * Set a pte to contain a valid mapping and enter it in the VHPT. If 1731 * the pte was orginally valid, then its assumed to already be in the 1732 * VHPT. 1733 * This functions does not set the protection bits. It's expected 1734 * that those have been set correctly prior to calling this function. 1735 */ 1736 static void 1737 pmap_set_pte(struct ia64_lpte *pte, vaddr_t va, vaddr_t pa, 1738 bool wired, bool managed) 1739 { 1740 1741 pte->pte &= PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK; 1742 pte->pte |= PTE_PRESENT | PTE_MA_WB; 1743 pte->pte |= (managed) ? PTE_MANAGED : (PTE_DIRTY | PTE_ACCESSED); 1744 pte->pte |= (wired) ? PTE_WIRED : 0; 1745 pte->pte |= pa & PTE_PPN_MASK; 1746 1747 pte->itir = PAGE_SHIFT << 2; 1748 1749 pte->tag = ia64_ttag(va); 1750 } 1751 1752 /* 1753 * Remove the (possibly managed) mapping represented by pte from the 1754 * given pmap. 1755 */ 1756 static int 1757 pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vaddr_t va, 1758 pv_entry_t pv, int freepte) 1759 { 1760 int error; 1761 struct vm_page *pg; 1762 1763 KASSERT(pmap == pmap_kernel() || pmap == curcpu()->ci_pmap); 1764 1765 /* 1766 * First remove from the VHPT. 1767 */ 1768 error = pmap_remove_vhpt(va); 1769 if (error) 1770 return error; 1771 1772 pmap_invalidate_page(pmap, va); 1773 1774 if (pmap_wired(pte)) 1775 pmap->pm_stats.wired_count -= 1; 1776 1777 pmap->pm_stats.resident_count -= 1; 1778 if (pmap_managed(pte)) { 1779 pg = PHYS_TO_VM_PAGE(pmap_ppn(pte)); 1780 if (pmap_dirty(pte)) 1781 if (pmap_track_modified(va)) 1782 pg->flags &= ~(PG_CLEAN); 1783 if (pmap_accessed(pte)) 1784 pg->flags &= ~PG_CLEAN; /* XXX: Do we need this ? */ 1785 1786 1787 if (freepte) 1788 pmap_free_pte(pte, va); 1789 1790 error = pmap_remove_entry(pmap, pg, va, pv); 1791 1792 } 1793 if (freepte) 1794 pmap_free_pte(pte, va); 1795 return 0; 1796 } 1797 1798 1799 1800 /* 1801 * Free a pte which is now unused. This simply returns it to the zone 1802 * allocator if it is a user mapping. For kernel mappings, clear the 1803 * valid bit to make it clear that the mapping is not currently used. 1804 */ 1805 static void 1806 pmap_free_pte(struct ia64_lpte *pte, vaddr_t va) 1807 { 1808 if (va < VM_MAXUSER_ADDRESS) 1809 while (0); 1810 // pool_put(pool_ia64_lpte_pool, pte); XXX: Fixme for userspace 1811 else 1812 pmap_clear_present(pte); 1813 } 1814 1815 1816 /* 1817 * this routine defines the region(s) of memory that should 1818 * not be tested for the modified bit. 1819 */ 1820 static __inline int 1821 pmap_track_modified(vaddr_t va) 1822 { 1823 extern char *kmembase, kmemlimit; 1824 if ((va < (vaddr_t) kmembase) || (va >= (vaddr_t) kmemlimit)) 1825 return 1; 1826 else 1827 return 0; 1828 } 1829 1830 1831 /*************************************************** 1832 * page management routines. 1833 ***************************************************/ 1834 1835 1836 /* 1837 * get a new pv_entry, allocating a block from the system 1838 * when needed. 1839 * the memory allocation is performed bypassing the malloc code 1840 * because of the possibility of allocations at interrupt time. 1841 */ 1842 /* 1843 * get a new pv_entry, allocating a block from the system 1844 * when needed. 1845 */ 1846 static pv_entry_t 1847 get_pv_entry(pmap_t locked_pmap) 1848 { 1849 pv_entry_t allocated_pv; 1850 1851 //LOCK_ASSERT(simple_lock_held(locked_pmap->slock)); 1852 //UVM_LOCK_ASSERT_PAGEQ(); 1853 allocated_pv = pool_get(&pmap_pv_pool, PR_NOWAIT); 1854 return allocated_pv; 1855 1856 1857 /* XXX: Nice to have all this stuff later: 1858 * Reclaim pv entries: At first, destroy mappings to inactive 1859 * pages. After that, if a pv entry is still needed, destroy 1860 * mappings to active pages. 1861 */ 1862 } 1863 1864 /* 1865 * free the pv_entry back to the free list 1866 */ 1867 static __inline void 1868 free_pv_entry(pv_entry_t pv) 1869 { 1870 pool_put(&pmap_pv_pool, pv); 1871 } 1872 1873 1874 /* 1875 * Add an ia64_lpte to the VHPT. 1876 */ 1877 static void 1878 pmap_enter_vhpt(struct ia64_lpte *pte, vaddr_t va) 1879 { 1880 struct ia64_bucket *bckt; 1881 struct ia64_lpte *vhpte; 1882 uint64_t pte_pa; 1883 1884 /* Can fault, so get it out of the way. */ 1885 pte_pa = ia64_tpa((vaddr_t)pte); 1886 1887 vhpte = (struct ia64_lpte *)ia64_thash(va); 1888 bckt = (struct ia64_bucket *)vhpte->chain; 1889 /* XXX: fixme */ 1890 mutex_enter(&bckt->lock); 1891 pte->chain = bckt->chain; 1892 ia64_mf(); 1893 bckt->chain = pte_pa; 1894 1895 pmap_vhpt_inserts++; 1896 bckt->length++; 1897 /*XXX : fixme */ 1898 mutex_exit(&bckt->lock); 1899 1900 } 1901 1902 /* 1903 * Remove the ia64_lpte matching va from the VHPT. Return zero if it 1904 * worked or an appropriate error code otherwise. 1905 */ 1906 static int 1907 pmap_remove_vhpt(vaddr_t va) 1908 { 1909 struct ia64_bucket *bckt; 1910 struct ia64_lpte *pte; 1911 struct ia64_lpte *lpte; 1912 struct ia64_lpte *vhpte; 1913 uint64_t chain, tag; 1914 1915 tag = ia64_ttag(va); 1916 vhpte = (struct ia64_lpte *)ia64_thash(va); 1917 bckt = (struct ia64_bucket *)vhpte->chain; 1918 1919 lpte = NULL; 1920 mutex_enter(&bckt->lock); 1921 1922 1923 chain = bckt->chain; 1924 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain); 1925 while (chain != 0 && pte->tag != tag) { 1926 lpte = pte; 1927 chain = pte->chain; 1928 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain); 1929 } 1930 if (chain == 0) { 1931 mutex_exit(&bckt->lock); 1932 return ENOENT; 1933 } 1934 1935 /* Snip this pv_entry out of the collision chain. */ 1936 if (lpte == NULL) 1937 bckt->chain = pte->chain; 1938 else 1939 lpte->chain = pte->chain; 1940 ia64_mf(); 1941 1942 bckt->length--; 1943 mutex_exit(&bckt->lock); 1944 return 0; 1945 } 1946 1947 1948 /* 1949 * Find the ia64_lpte for the given va, if any. 1950 */ 1951 static struct ia64_lpte * 1952 pmap_find_vhpt(vaddr_t va) 1953 { 1954 struct ia64_bucket *bckt; 1955 struct ia64_lpte *pte; 1956 uint64_t chain, tag; 1957 1958 tag = ia64_ttag(va); 1959 pte = (struct ia64_lpte *)ia64_thash(va); 1960 bckt = (struct ia64_bucket *)pte->chain; 1961 1962 mutex_enter(&bckt->lock); 1963 chain = bckt->chain; 1964 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain); 1965 while (chain != 0 && pte->tag != tag) { 1966 chain = pte->chain; 1967 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain); 1968 } 1969 mutex_exit(&bckt->lock); 1970 return (chain != 0) ? pte : NULL; 1971 } 1972 1973 1974 /* 1975 * Remove an entry from the list of managed mappings. 1976 */ 1977 static int 1978 pmap_remove_entry(pmap_t pmap, struct vm_page *pg, vaddr_t va, pv_entry_t pv) 1979 { 1980 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 1981 1982 if (!pv) { 1983 if (md->pv_list_count < pmap->pm_stats.resident_count) { 1984 TAILQ_FOREACH(pv, &md->pv_list, pv_list) { 1985 if (pmap == pv->pv_pmap && va == pv->pv_va) 1986 break; 1987 } 1988 } else { 1989 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) { 1990 if (va == pv->pv_va) 1991 break; 1992 } 1993 } 1994 } 1995 1996 if (pv) { 1997 TAILQ_REMOVE(&md->pv_list, pv, pv_list); 1998 md->pv_list_count--; 1999 if (TAILQ_FIRST(&md->pv_list) == NULL) { 2000 //UVM_LOCK_ASSERT_PAGEQ(); 2001 pg->flags |= PG_RDONLY; 2002 } 2003 2004 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); 2005 free_pv_entry(pv); 2006 return 0; 2007 } else { 2008 return ENOENT; 2009 } 2010 } 2011 2012 2013 /* 2014 * Create a pv entry for page at pa for 2015 * (pmap, va). 2016 */ 2017 static void 2018 pmap_insert_entry(pmap_t pmap, vaddr_t va, struct vm_page *pg) 2019 { 2020 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 2021 pv_entry_t pv; 2022 2023 pv = get_pv_entry(pmap); 2024 pv->pv_pmap = pmap; 2025 pv->pv_va = va; 2026 2027 //LOCK_ASSERT(simple_lock_held(pmap->slock)); 2028 //UVM_LOCK_ASSERT_PAGEQ(); 2029 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); 2030 TAILQ_INSERT_TAIL(&md->pv_list, pv, pv_list); 2031 md->pv_list_count++; 2032 } 2033 2034 2035 /* 2036 * Remove a single page from a process address space 2037 */ 2038 static void 2039 pmap_remove_page(pmap_t pmap, vaddr_t va) 2040 { 2041 struct ia64_lpte *pte; 2042 2043 KASSERT(pmap == pmap_kernel() || pmap == curcpu()->ci_pmap); 2044 2045 pte = pmap_find_vhpt(va); 2046 if (pte) { 2047 pmap_remove_pte(pmap, pte, va, 0, 1); 2048 pmap_invalidate_page(pmap, va); 2049 } 2050 return; 2051 } 2052 2053 2054 2055 /* 2056 * pmap_pv_page_alloc: 2057 * 2058 * Allocate a page for the pv_entry pool. 2059 */ 2060 void * 2061 pmap_pv_page_alloc(struct pool *pp, int flags) 2062 { 2063 paddr_t pg; 2064 2065 if (pmap_poolpage_alloc(&pg)) 2066 return (void *)IA64_PHYS_TO_RR7(pg); 2067 return NULL; 2068 } 2069 2070 /* 2071 * pmap_pv_page_free: 2072 * 2073 * Free a pv_entry pool page. 2074 */ 2075 void 2076 pmap_pv_page_free(struct pool *pp, void *v) 2077 { 2078 2079 pmap_poolpage_free(IA64_RR_MASK((vaddr_t)v)); 2080 } 2081 2082 /******************** misc. functions ********************/ 2083 2084 /* 2085 * pmap_poolpage_alloc: based on alpha/pmap_physpage_alloc 2086 * 2087 * Allocate a single page from the VM system and return the 2088 * physical address for that page. 2089 */ 2090 bool 2091 pmap_poolpage_alloc(paddr_t *pap) 2092 { 2093 struct vm_page *pg; 2094 paddr_t pa; 2095 2096 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE|UVM_PGA_ZERO); 2097 if (pg != NULL) { 2098 pa = VM_PAGE_TO_PHYS(pg); 2099 2100 #ifdef DEBUG 2101 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 2102 mutex_enter(&md->pv_mutex); 2103 if (pg->wire_count != 0) { 2104 printf("pmap_physpage_alloc: page 0x%lx has " 2105 "%d references\n", pa, pg->wire_count); 2106 panic("pmap_physpage_alloc"); 2107 } 2108 mutex_exit(&md->pv_mutex); 2109 #endif 2110 *pap = pa; 2111 return true; 2112 } 2113 return false; 2114 } 2115 2116 /* 2117 * pmap_poolpage_free: based on alpha/pmap_physpage_free: 2118 * 2119 * Free the single page table page at the specified physical address. 2120 */ 2121 void 2122 pmap_poolpage_free(paddr_t pa) 2123 { 2124 struct vm_page *pg; 2125 2126 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 2127 panic("pmap_physpage_free: bogus physical page address"); 2128 2129 #ifdef DEBUG 2130 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 2131 mutex_enter(&md->pv_mutex); 2132 if (pg->wire_count != 0) 2133 panic("pmap_physpage_free: page still has references"); 2134 mutex_exit(&md->pv_mutex); 2135 #endif 2136 2137 uvm_pagefree(pg); 2138 } 2139 2140 #ifdef DEBUG 2141 2142 static void dump_vhpt(void) 2143 { 2144 2145 vaddr_t base; 2146 vsize_t size, i; 2147 struct ia64_lpte *pte; 2148 2149 __asm __volatile("mov %0=cr.pta;; srlz.i;;" : 2150 "=r" (base)); 2151 2152 #define VHPTBASE(x) ( (x) & (~0x7fffUL) ) 2153 #define VHPTSIZE(x) ( (vsize_t) (1 << (((x) & 0x7cUL) >> 2))) 2154 2155 size = VHPTSIZE(base); 2156 base = VHPTBASE(base); 2157 2158 pte = (void *) base; 2159 2160 printf("vhpt base = %lx \n", base); 2161 printf("vhpt size = %lx \n", size); 2162 2163 for(i = 0; i < size/sizeof(struct ia64_lpte);i++ ) { 2164 if(pte[i].pte & PTE_PRESENT) { 2165 printf("PTE_PRESENT "); 2166 2167 if(pte[i].pte & PTE_MA_MASK) printf("MA: "); 2168 if(pte[i].pte & PTE_MA_WB) printf("WB "); 2169 if(pte[i].pte & PTE_MA_UC) printf("UC "); 2170 if(pte[i].pte & PTE_MA_UCE) printf("UCE "); 2171 if(pte[i].pte & PTE_MA_WC) printf("WC "); 2172 if(pte[i].pte & PTE_MA_NATPAGE) printf("NATPAGE "); 2173 2174 if(pte[i].pte & PTE_ACCESSED) printf("PTE_ACCESSED "); 2175 if(pte[i].pte & PTE_DIRTY) printf("PTE_DIRTY "); 2176 2177 if(pte[i].pte & PTE_PL_MASK) printf("PL: "); 2178 if(pte[i].pte & PTE_PL_KERN) printf("KERN"); 2179 if(pte[i].pte & PTE_PL_USER) printf("USER"); 2180 2181 if(pte[i].pte & PTE_AR_MASK) printf("AR: "); 2182 if(pte[i].pte & PTE_AR_R) printf("R "); 2183 if(pte[i].pte & PTE_AR_RX) printf("RX "); 2184 if(pte[i].pte & PTE_AR_RWX) printf("RWX "); 2185 if(pte[i].pte & PTE_AR_R_RW) printf("R RW "); 2186 if(pte[i].pte & PTE_AR_RX_RWX) printf("RX RWX "); 2187 2188 printf("ppn = %lx", (pte[i].pte & PTE_PPN_MASK) >> 12); 2189 2190 if(pte[i].pte & PTE_ED) printf("ED "); 2191 2192 if(pte[i].pte & PTE_IG_MASK) printf("OS: "); 2193 if(pte[i].pte & PTE_WIRED) printf("WIRED "); 2194 if(pte[i].pte & PTE_MANAGED) printf("MANAGED "); 2195 printf("\n"); 2196 } 2197 2198 } 2199 2200 2201 2202 } 2203 #endif 2204