1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Re-map IO memory to kernel address space so that we can access it.
4 * This is needed for high PCI addresses that aren't mapped in the
5 * 640k-1MB IO memory area on PC's
6 *
7 * (C) Copyright 1995 1996 Linus Torvalds
8 */
9
10 #include <linux/memblock.h>
11 #include <linux/init.h>
12 #include <linux/io.h>
13 #include <linux/ioport.h>
14 #include <linux/slab.h>
15 #include <linux/vmalloc.h>
16 #include <linux/mmiotrace.h>
17 #include <linux/mem_encrypt.h>
18 #include <linux/efi.h>
19 #include <linux/pgtable.h>
20
21 #include <asm/set_memory.h>
22 #include <asm/e820/api.h>
23 #include <asm/efi.h>
24 #include <asm/fixmap.h>
25 #include <asm/tlbflush.h>
26 #include <asm/pgalloc.h>
27 #include <asm/memtype.h>
28 #include <asm/setup.h>
29
30 #include "physaddr.h"
31
32 /*
33 * Descriptor controlling ioremap() behavior.
34 */
35 struct ioremap_desc {
36 unsigned int flags;
37 };
38
39 /*
40 * Fix up the linear direct mapping of the kernel to avoid cache attribute
41 * conflicts.
42 */
ioremap_change_attr(unsigned long vaddr,unsigned long size,enum page_cache_mode pcm)43 int ioremap_change_attr(unsigned long vaddr, unsigned long size,
44 enum page_cache_mode pcm)
45 {
46 unsigned long nrpages = size >> PAGE_SHIFT;
47 int err;
48
49 switch (pcm) {
50 case _PAGE_CACHE_MODE_UC:
51 default:
52 err = _set_memory_uc(vaddr, nrpages);
53 break;
54 case _PAGE_CACHE_MODE_WC:
55 err = _set_memory_wc(vaddr, nrpages);
56 break;
57 case _PAGE_CACHE_MODE_WT:
58 err = _set_memory_wt(vaddr, nrpages);
59 break;
60 case _PAGE_CACHE_MODE_WB:
61 err = _set_memory_wb(vaddr, nrpages);
62 break;
63 }
64
65 return err;
66 }
67
68 /* Does the range (or a subset of) contain normal RAM? */
__ioremap_check_ram(struct resource * res)69 static unsigned int __ioremap_check_ram(struct resource *res)
70 {
71 unsigned long start_pfn, stop_pfn;
72 unsigned long i;
73
74 if ((res->flags & IORESOURCE_SYSTEM_RAM) != IORESOURCE_SYSTEM_RAM)
75 return 0;
76
77 start_pfn = (res->start + PAGE_SIZE - 1) >> PAGE_SHIFT;
78 stop_pfn = (res->end + 1) >> PAGE_SHIFT;
79 if (stop_pfn > start_pfn) {
80 for (i = 0; i < (stop_pfn - start_pfn); ++i)
81 if (pfn_valid(start_pfn + i) &&
82 !PageReserved(pfn_to_page(start_pfn + i)))
83 return IORES_MAP_SYSTEM_RAM;
84 }
85
86 return 0;
87 }
88
89 /*
90 * In a SEV guest, NONE and RESERVED should not be mapped encrypted because
91 * there the whole memory is already encrypted.
92 */
__ioremap_check_encrypted(struct resource * res)93 static unsigned int __ioremap_check_encrypted(struct resource *res)
94 {
95 if (!sev_active())
96 return 0;
97
98 switch (res->desc) {
99 case IORES_DESC_NONE:
100 case IORES_DESC_RESERVED:
101 break;
102 default:
103 return IORES_MAP_ENCRYPTED;
104 }
105
106 return 0;
107 }
108
109 /*
110 * The EFI runtime services data area is not covered by walk_mem_res(), but must
111 * be mapped encrypted when SEV is active.
112 */
__ioremap_check_other(resource_size_t addr,struct ioremap_desc * desc)113 static void __ioremap_check_other(resource_size_t addr, struct ioremap_desc *desc)
114 {
115 if (!sev_active())
116 return;
117
118 if (!IS_ENABLED(CONFIG_EFI))
119 return;
120
121 if (efi_mem_type(addr) == EFI_RUNTIME_SERVICES_DATA)
122 desc->flags |= IORES_MAP_ENCRYPTED;
123 }
124
__ioremap_collect_map_flags(struct resource * res,void * arg)125 static int __ioremap_collect_map_flags(struct resource *res, void *arg)
126 {
127 struct ioremap_desc *desc = arg;
128
129 if (!(desc->flags & IORES_MAP_SYSTEM_RAM))
130 desc->flags |= __ioremap_check_ram(res);
131
132 if (!(desc->flags & IORES_MAP_ENCRYPTED))
133 desc->flags |= __ioremap_check_encrypted(res);
134
135 return ((desc->flags & (IORES_MAP_SYSTEM_RAM | IORES_MAP_ENCRYPTED)) ==
136 (IORES_MAP_SYSTEM_RAM | IORES_MAP_ENCRYPTED));
137 }
138
139 /*
140 * To avoid multiple resource walks, this function walks resources marked as
141 * IORESOURCE_MEM and IORESOURCE_BUSY and looking for system RAM and/or a
142 * resource described not as IORES_DESC_NONE (e.g. IORES_DESC_ACPI_TABLES).
143 *
144 * After that, deal with misc other ranges in __ioremap_check_other() which do
145 * not fall into the above category.
146 */
__ioremap_check_mem(resource_size_t addr,unsigned long size,struct ioremap_desc * desc)147 static void __ioremap_check_mem(resource_size_t addr, unsigned long size,
148 struct ioremap_desc *desc)
149 {
150 u64 start, end;
151
152 start = (u64)addr;
153 end = start + size - 1;
154 memset(desc, 0, sizeof(struct ioremap_desc));
155
156 walk_mem_res(start, end, desc, __ioremap_collect_map_flags);
157
158 __ioremap_check_other(addr, desc);
159 }
160
161 /*
162 * Remap an arbitrary physical address space into the kernel virtual
163 * address space. It transparently creates kernel huge I/O mapping when
164 * the physical address is aligned by a huge page size (1GB or 2MB) and
165 * the requested size is at least the huge page size.
166 *
167 * NOTE: MTRRs can override PAT memory types with a 4KB granularity.
168 * Therefore, the mapping code falls back to use a smaller page toward 4KB
169 * when a mapping range is covered by non-WB type of MTRRs.
170 *
171 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
172 * have to convert them into an offset in a page-aligned mapping, but the
173 * caller shouldn't need to know that small detail.
174 */
175 static void __iomem *
__ioremap_caller(resource_size_t phys_addr,unsigned long size,enum page_cache_mode pcm,void * caller,bool encrypted)176 __ioremap_caller(resource_size_t phys_addr, unsigned long size,
177 enum page_cache_mode pcm, void *caller, bool encrypted)
178 {
179 unsigned long offset, vaddr;
180 resource_size_t last_addr;
181 const resource_size_t unaligned_phys_addr = phys_addr;
182 const unsigned long unaligned_size = size;
183 struct ioremap_desc io_desc;
184 struct vm_struct *area;
185 enum page_cache_mode new_pcm;
186 pgprot_t prot;
187 int retval;
188 void __iomem *ret_addr;
189
190 /* Don't allow wraparound or zero size */
191 last_addr = phys_addr + size - 1;
192 if (!size || last_addr < phys_addr)
193 return NULL;
194
195 if (!phys_addr_valid(phys_addr)) {
196 printk(KERN_WARNING "ioremap: invalid physical address %llx\n",
197 (unsigned long long)phys_addr);
198 WARN_ON_ONCE(1);
199 return NULL;
200 }
201
202 __ioremap_check_mem(phys_addr, size, &io_desc);
203
204 /*
205 * Don't allow anybody to remap normal RAM that we're using..
206 */
207 if (io_desc.flags & IORES_MAP_SYSTEM_RAM) {
208 WARN_ONCE(1, "ioremap on RAM at %pa - %pa\n",
209 &phys_addr, &last_addr);
210 return NULL;
211 }
212
213 /*
214 * Mappings have to be page-aligned
215 */
216 offset = phys_addr & ~PAGE_MASK;
217 phys_addr &= PHYSICAL_PAGE_MASK;
218 size = PAGE_ALIGN(last_addr+1) - phys_addr;
219
220 retval = memtype_reserve(phys_addr, (u64)phys_addr + size,
221 pcm, &new_pcm);
222 if (retval) {
223 printk(KERN_ERR "ioremap memtype_reserve failed %d\n", retval);
224 return NULL;
225 }
226
227 if (pcm != new_pcm) {
228 if (!is_new_memtype_allowed(phys_addr, size, pcm, new_pcm)) {
229 printk(KERN_ERR
230 "ioremap error for 0x%llx-0x%llx, requested 0x%x, got 0x%x\n",
231 (unsigned long long)phys_addr,
232 (unsigned long long)(phys_addr + size),
233 pcm, new_pcm);
234 goto err_free_memtype;
235 }
236 pcm = new_pcm;
237 }
238
239 /*
240 * If the page being mapped is in memory and SEV is active then
241 * make sure the memory encryption attribute is enabled in the
242 * resulting mapping.
243 */
244 prot = PAGE_KERNEL_IO;
245 if ((io_desc.flags & IORES_MAP_ENCRYPTED) || encrypted)
246 prot = pgprot_encrypted(prot);
247
248 switch (pcm) {
249 case _PAGE_CACHE_MODE_UC:
250 default:
251 prot = __pgprot(pgprot_val(prot) |
252 cachemode2protval(_PAGE_CACHE_MODE_UC));
253 break;
254 case _PAGE_CACHE_MODE_UC_MINUS:
255 prot = __pgprot(pgprot_val(prot) |
256 cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS));
257 break;
258 case _PAGE_CACHE_MODE_WC:
259 prot = __pgprot(pgprot_val(prot) |
260 cachemode2protval(_PAGE_CACHE_MODE_WC));
261 break;
262 case _PAGE_CACHE_MODE_WT:
263 prot = __pgprot(pgprot_val(prot) |
264 cachemode2protval(_PAGE_CACHE_MODE_WT));
265 break;
266 case _PAGE_CACHE_MODE_WB:
267 break;
268 }
269
270 /*
271 * Ok, go for it..
272 */
273 area = get_vm_area_caller(size, VM_IOREMAP, caller);
274 if (!area)
275 goto err_free_memtype;
276 area->phys_addr = phys_addr;
277 vaddr = (unsigned long) area->addr;
278
279 if (memtype_kernel_map_sync(phys_addr, size, pcm))
280 goto err_free_area;
281
282 if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot))
283 goto err_free_area;
284
285 ret_addr = (void __iomem *) (vaddr + offset);
286 mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
287
288 /*
289 * Check if the request spans more than any BAR in the iomem resource
290 * tree.
291 */
292 if (iomem_map_sanity_check(unaligned_phys_addr, unaligned_size))
293 pr_warn("caller %pS mapping multiple BARs\n", caller);
294
295 return ret_addr;
296 err_free_area:
297 free_vm_area(area);
298 err_free_memtype:
299 memtype_free(phys_addr, phys_addr + size);
300 return NULL;
301 }
302
303 /**
304 * ioremap - map bus memory into CPU space
305 * @phys_addr: bus address of the memory
306 * @size: size of the resource to map
307 *
308 * ioremap performs a platform specific sequence of operations to
309 * make bus memory CPU accessible via the readb/readw/readl/writeb/
310 * writew/writel functions and the other mmio helpers. The returned
311 * address is not guaranteed to be usable directly as a virtual
312 * address.
313 *
314 * This version of ioremap ensures that the memory is marked uncachable
315 * on the CPU as well as honouring existing caching rules from things like
316 * the PCI bus. Note that there are other caches and buffers on many
317 * busses. In particular driver authors should read up on PCI writes
318 *
319 * It's useful if some control registers are in such an area and
320 * write combining or read caching is not desirable:
321 *
322 * Must be freed with iounmap.
323 */
ioremap(resource_size_t phys_addr,unsigned long size)324 void __iomem *ioremap(resource_size_t phys_addr, unsigned long size)
325 {
326 /*
327 * Ideally, this should be:
328 * pat_enabled() ? _PAGE_CACHE_MODE_UC : _PAGE_CACHE_MODE_UC_MINUS;
329 *
330 * Till we fix all X drivers to use ioremap_wc(), we will use
331 * UC MINUS. Drivers that are certain they need or can already
332 * be converted over to strong UC can use ioremap_uc().
333 */
334 enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC_MINUS;
335
336 return __ioremap_caller(phys_addr, size, pcm,
337 __builtin_return_address(0), false);
338 }
339 EXPORT_SYMBOL(ioremap);
340
341 /**
342 * ioremap_uc - map bus memory into CPU space as strongly uncachable
343 * @phys_addr: bus address of the memory
344 * @size: size of the resource to map
345 *
346 * ioremap_uc performs a platform specific sequence of operations to
347 * make bus memory CPU accessible via the readb/readw/readl/writeb/
348 * writew/writel functions and the other mmio helpers. The returned
349 * address is not guaranteed to be usable directly as a virtual
350 * address.
351 *
352 * This version of ioremap ensures that the memory is marked with a strong
353 * preference as completely uncachable on the CPU when possible. For non-PAT
354 * systems this ends up setting page-attribute flags PCD=1, PWT=1. For PAT
355 * systems this will set the PAT entry for the pages as strong UC. This call
356 * will honor existing caching rules from things like the PCI bus. Note that
357 * there are other caches and buffers on many busses. In particular driver
358 * authors should read up on PCI writes.
359 *
360 * It's useful if some control registers are in such an area and
361 * write combining or read caching is not desirable:
362 *
363 * Must be freed with iounmap.
364 */
ioremap_uc(resource_size_t phys_addr,unsigned long size)365 void __iomem *ioremap_uc(resource_size_t phys_addr, unsigned long size)
366 {
367 enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC;
368
369 return __ioremap_caller(phys_addr, size, pcm,
370 __builtin_return_address(0), false);
371 }
372 EXPORT_SYMBOL_GPL(ioremap_uc);
373
374 /**
375 * ioremap_wc - map memory into CPU space write combined
376 * @phys_addr: bus address of the memory
377 * @size: size of the resource to map
378 *
379 * This version of ioremap ensures that the memory is marked write combining.
380 * Write combining allows faster writes to some hardware devices.
381 *
382 * Must be freed with iounmap.
383 */
ioremap_wc(resource_size_t phys_addr,unsigned long size)384 void __iomem *ioremap_wc(resource_size_t phys_addr, unsigned long size)
385 {
386 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WC,
387 __builtin_return_address(0), false);
388 }
389 EXPORT_SYMBOL(ioremap_wc);
390
391 /**
392 * ioremap_wt - map memory into CPU space write through
393 * @phys_addr: bus address of the memory
394 * @size: size of the resource to map
395 *
396 * This version of ioremap ensures that the memory is marked write through.
397 * Write through stores data into memory while keeping the cache up-to-date.
398 *
399 * Must be freed with iounmap.
400 */
ioremap_wt(resource_size_t phys_addr,unsigned long size)401 void __iomem *ioremap_wt(resource_size_t phys_addr, unsigned long size)
402 {
403 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WT,
404 __builtin_return_address(0), false);
405 }
406 EXPORT_SYMBOL(ioremap_wt);
407
ioremap_encrypted(resource_size_t phys_addr,unsigned long size)408 void __iomem *ioremap_encrypted(resource_size_t phys_addr, unsigned long size)
409 {
410 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WB,
411 __builtin_return_address(0), true);
412 }
413 EXPORT_SYMBOL(ioremap_encrypted);
414
ioremap_cache(resource_size_t phys_addr,unsigned long size)415 void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
416 {
417 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WB,
418 __builtin_return_address(0), false);
419 }
420 EXPORT_SYMBOL(ioremap_cache);
421
ioremap_prot(resource_size_t phys_addr,unsigned long size,unsigned long prot_val)422 void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size,
423 unsigned long prot_val)
424 {
425 return __ioremap_caller(phys_addr, size,
426 pgprot2cachemode(__pgprot(prot_val)),
427 __builtin_return_address(0), false);
428 }
429 EXPORT_SYMBOL(ioremap_prot);
430
431 /**
432 * iounmap - Free a IO remapping
433 * @addr: virtual address from ioremap_*
434 *
435 * Caller must ensure there is only one unmapping for the same pointer.
436 */
iounmap(volatile void __iomem * addr)437 void iounmap(volatile void __iomem *addr)
438 {
439 struct vm_struct *p, *o;
440
441 if ((void __force *)addr <= high_memory)
442 return;
443
444 /*
445 * The PCI/ISA range special-casing was removed from __ioremap()
446 * so this check, in theory, can be removed. However, there are
447 * cases where iounmap() is called for addresses not obtained via
448 * ioremap() (vga16fb for example). Add a warning so that these
449 * cases can be caught and fixed.
450 */
451 if ((void __force *)addr >= phys_to_virt(ISA_START_ADDRESS) &&
452 (void __force *)addr < phys_to_virt(ISA_END_ADDRESS)) {
453 WARN(1, "iounmap() called for ISA range not obtained using ioremap()\n");
454 return;
455 }
456
457 mmiotrace_iounmap(addr);
458
459 addr = (volatile void __iomem *)
460 (PAGE_MASK & (unsigned long __force)addr);
461
462 /* Use the vm area unlocked, assuming the caller
463 ensures there isn't another iounmap for the same address
464 in parallel. Reuse of the virtual address is prevented by
465 leaving it in the global lists until we're done with it.
466 cpa takes care of the direct mappings. */
467 p = find_vm_area((void __force *)addr);
468
469 if (!p) {
470 printk(KERN_ERR "iounmap: bad address %p\n", addr);
471 dump_stack();
472 return;
473 }
474
475 memtype_free(p->phys_addr, p->phys_addr + get_vm_area_size(p));
476
477 /* Finally remove it */
478 o = remove_vm_area((void __force *)addr);
479 BUG_ON(p != o || o == NULL);
480 kfree(p);
481 }
482 EXPORT_SYMBOL(iounmap);
483
484 /*
485 * Convert a physical pointer to a virtual kernel pointer for /dev/mem
486 * access
487 */
xlate_dev_mem_ptr(phys_addr_t phys)488 void *xlate_dev_mem_ptr(phys_addr_t phys)
489 {
490 unsigned long start = phys & PAGE_MASK;
491 unsigned long offset = phys & ~PAGE_MASK;
492 void *vaddr;
493
494 /* memremap() maps if RAM, otherwise falls back to ioremap() */
495 vaddr = memremap(start, PAGE_SIZE, MEMREMAP_WB);
496
497 /* Only add the offset on success and return NULL if memremap() failed */
498 if (vaddr)
499 vaddr += offset;
500
501 return vaddr;
502 }
503
unxlate_dev_mem_ptr(phys_addr_t phys,void * addr)504 void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr)
505 {
506 memunmap((void *)((unsigned long)addr & PAGE_MASK));
507 }
508
509 /*
510 * Examine the physical address to determine if it is an area of memory
511 * that should be mapped decrypted. If the memory is not part of the
512 * kernel usable area it was accessed and created decrypted, so these
513 * areas should be mapped decrypted. And since the encryption key can
514 * change across reboots, persistent memory should also be mapped
515 * decrypted.
516 *
517 * If SEV is active, that implies that BIOS/UEFI also ran encrypted so
518 * only persistent memory should be mapped decrypted.
519 */
memremap_should_map_decrypted(resource_size_t phys_addr,unsigned long size)520 static bool memremap_should_map_decrypted(resource_size_t phys_addr,
521 unsigned long size)
522 {
523 int is_pmem;
524
525 /*
526 * Check if the address is part of a persistent memory region.
527 * This check covers areas added by E820, EFI and ACPI.
528 */
529 is_pmem = region_intersects(phys_addr, size, IORESOURCE_MEM,
530 IORES_DESC_PERSISTENT_MEMORY);
531 if (is_pmem != REGION_DISJOINT)
532 return true;
533
534 /*
535 * Check if the non-volatile attribute is set for an EFI
536 * reserved area.
537 */
538 if (efi_enabled(EFI_BOOT)) {
539 switch (efi_mem_type(phys_addr)) {
540 case EFI_RESERVED_TYPE:
541 if (efi_mem_attributes(phys_addr) & EFI_MEMORY_NV)
542 return true;
543 break;
544 default:
545 break;
546 }
547 }
548
549 /* Check if the address is outside kernel usable area */
550 switch (e820__get_entry_type(phys_addr, phys_addr + size - 1)) {
551 case E820_TYPE_RESERVED:
552 case E820_TYPE_ACPI:
553 case E820_TYPE_NVS:
554 case E820_TYPE_UNUSABLE:
555 /* For SEV, these areas are encrypted */
556 if (sev_active())
557 break;
558 fallthrough;
559
560 case E820_TYPE_PRAM:
561 return true;
562 default:
563 break;
564 }
565
566 return false;
567 }
568
569 /*
570 * Examine the physical address to determine if it is EFI data. Check
571 * it against the boot params structure and EFI tables and memory types.
572 */
memremap_is_efi_data(resource_size_t phys_addr,unsigned long size)573 static bool memremap_is_efi_data(resource_size_t phys_addr,
574 unsigned long size)
575 {
576 u64 paddr;
577
578 /* Check if the address is part of EFI boot/runtime data */
579 if (!efi_enabled(EFI_BOOT))
580 return false;
581
582 paddr = boot_params.efi_info.efi_memmap_hi;
583 paddr <<= 32;
584 paddr |= boot_params.efi_info.efi_memmap;
585 if (phys_addr == paddr)
586 return true;
587
588 paddr = boot_params.efi_info.efi_systab_hi;
589 paddr <<= 32;
590 paddr |= boot_params.efi_info.efi_systab;
591 if (phys_addr == paddr)
592 return true;
593
594 if (efi_is_table_address(phys_addr))
595 return true;
596
597 switch (efi_mem_type(phys_addr)) {
598 case EFI_BOOT_SERVICES_DATA:
599 case EFI_RUNTIME_SERVICES_DATA:
600 return true;
601 default:
602 break;
603 }
604
605 return false;
606 }
607
608 /*
609 * Examine the physical address to determine if it is boot data by checking
610 * it against the boot params setup_data chain.
611 */
memremap_is_setup_data(resource_size_t phys_addr,unsigned long size)612 static bool memremap_is_setup_data(resource_size_t phys_addr,
613 unsigned long size)
614 {
615 struct setup_data *data;
616 u64 paddr, paddr_next;
617
618 paddr = boot_params.hdr.setup_data;
619 while (paddr) {
620 unsigned int len;
621
622 if (phys_addr == paddr)
623 return true;
624
625 data = memremap(paddr, sizeof(*data),
626 MEMREMAP_WB | MEMREMAP_DEC);
627
628 paddr_next = data->next;
629 len = data->len;
630
631 if ((phys_addr > paddr) && (phys_addr < (paddr + len))) {
632 memunmap(data);
633 return true;
634 }
635
636 if (data->type == SETUP_INDIRECT &&
637 ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT) {
638 paddr = ((struct setup_indirect *)data->data)->addr;
639 len = ((struct setup_indirect *)data->data)->len;
640 }
641
642 memunmap(data);
643
644 if ((phys_addr > paddr) && (phys_addr < (paddr + len)))
645 return true;
646
647 paddr = paddr_next;
648 }
649
650 return false;
651 }
652
653 /*
654 * Examine the physical address to determine if it is boot data by checking
655 * it against the boot params setup_data chain (early boot version).
656 */
early_memremap_is_setup_data(resource_size_t phys_addr,unsigned long size)657 static bool __init early_memremap_is_setup_data(resource_size_t phys_addr,
658 unsigned long size)
659 {
660 struct setup_data *data;
661 u64 paddr, paddr_next;
662
663 paddr = boot_params.hdr.setup_data;
664 while (paddr) {
665 unsigned int len;
666
667 if (phys_addr == paddr)
668 return true;
669
670 data = early_memremap_decrypted(paddr, sizeof(*data));
671
672 paddr_next = data->next;
673 len = data->len;
674
675 early_memunmap(data, sizeof(*data));
676
677 if ((phys_addr > paddr) && (phys_addr < (paddr + len)))
678 return true;
679
680 paddr = paddr_next;
681 }
682
683 return false;
684 }
685
686 /*
687 * Architecture function to determine if RAM remap is allowed. By default, a
688 * RAM remap will map the data as encrypted. Determine if a RAM remap should
689 * not be done so that the data will be mapped decrypted.
690 */
arch_memremap_can_ram_remap(resource_size_t phys_addr,unsigned long size,unsigned long flags)691 bool arch_memremap_can_ram_remap(resource_size_t phys_addr, unsigned long size,
692 unsigned long flags)
693 {
694 if (!mem_encrypt_active())
695 return true;
696
697 if (flags & MEMREMAP_ENC)
698 return true;
699
700 if (flags & MEMREMAP_DEC)
701 return false;
702
703 if (sme_active()) {
704 if (memremap_is_setup_data(phys_addr, size) ||
705 memremap_is_efi_data(phys_addr, size))
706 return false;
707 }
708
709 return !memremap_should_map_decrypted(phys_addr, size);
710 }
711
712 /*
713 * Architecture override of __weak function to adjust the protection attributes
714 * used when remapping memory. By default, early_memremap() will map the data
715 * as encrypted. Determine if an encrypted mapping should not be done and set
716 * the appropriate protection attributes.
717 */
early_memremap_pgprot_adjust(resource_size_t phys_addr,unsigned long size,pgprot_t prot)718 pgprot_t __init early_memremap_pgprot_adjust(resource_size_t phys_addr,
719 unsigned long size,
720 pgprot_t prot)
721 {
722 bool encrypted_prot;
723
724 if (!mem_encrypt_active())
725 return prot;
726
727 encrypted_prot = true;
728
729 if (sme_active()) {
730 if (early_memremap_is_setup_data(phys_addr, size) ||
731 memremap_is_efi_data(phys_addr, size))
732 encrypted_prot = false;
733 }
734
735 if (encrypted_prot && memremap_should_map_decrypted(phys_addr, size))
736 encrypted_prot = false;
737
738 return encrypted_prot ? pgprot_encrypted(prot)
739 : pgprot_decrypted(prot);
740 }
741
phys_mem_access_encrypted(unsigned long phys_addr,unsigned long size)742 bool phys_mem_access_encrypted(unsigned long phys_addr, unsigned long size)
743 {
744 return arch_memremap_can_ram_remap(phys_addr, size, 0);
745 }
746
747 #ifdef CONFIG_AMD_MEM_ENCRYPT
748 /* Remap memory with encryption */
early_memremap_encrypted(resource_size_t phys_addr,unsigned long size)749 void __init *early_memremap_encrypted(resource_size_t phys_addr,
750 unsigned long size)
751 {
752 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_ENC);
753 }
754
755 /*
756 * Remap memory with encryption and write-protected - cannot be called
757 * before pat_init() is called
758 */
early_memremap_encrypted_wp(resource_size_t phys_addr,unsigned long size)759 void __init *early_memremap_encrypted_wp(resource_size_t phys_addr,
760 unsigned long size)
761 {
762 if (!x86_has_pat_wp())
763 return NULL;
764 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_ENC_WP);
765 }
766
767 /* Remap memory without encryption */
early_memremap_decrypted(resource_size_t phys_addr,unsigned long size)768 void __init *early_memremap_decrypted(resource_size_t phys_addr,
769 unsigned long size)
770 {
771 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_NOENC);
772 }
773
774 /*
775 * Remap memory without encryption and write-protected - cannot be called
776 * before pat_init() is called
777 */
early_memremap_decrypted_wp(resource_size_t phys_addr,unsigned long size)778 void __init *early_memremap_decrypted_wp(resource_size_t phys_addr,
779 unsigned long size)
780 {
781 if (!x86_has_pat_wp())
782 return NULL;
783 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_NOENC_WP);
784 }
785 #endif /* CONFIG_AMD_MEM_ENCRYPT */
786
787 static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss;
788
early_ioremap_pmd(unsigned long addr)789 static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
790 {
791 /* Don't assume we're using swapper_pg_dir at this point */
792 pgd_t *base = __va(read_cr3_pa());
793 pgd_t *pgd = &base[pgd_index(addr)];
794 p4d_t *p4d = p4d_offset(pgd, addr);
795 pud_t *pud = pud_offset(p4d, addr);
796 pmd_t *pmd = pmd_offset(pud, addr);
797
798 return pmd;
799 }
800
early_ioremap_pte(unsigned long addr)801 static inline pte_t * __init early_ioremap_pte(unsigned long addr)
802 {
803 return &bm_pte[pte_index(addr)];
804 }
805
is_early_ioremap_ptep(pte_t * ptep)806 bool __init is_early_ioremap_ptep(pte_t *ptep)
807 {
808 return ptep >= &bm_pte[0] && ptep < &bm_pte[PAGE_SIZE/sizeof(pte_t)];
809 }
810
early_ioremap_init(void)811 void __init early_ioremap_init(void)
812 {
813 pmd_t *pmd;
814
815 #ifdef CONFIG_X86_64
816 BUILD_BUG_ON((fix_to_virt(0) + PAGE_SIZE) & ((1 << PMD_SHIFT) - 1));
817 #else
818 WARN_ON((fix_to_virt(0) + PAGE_SIZE) & ((1 << PMD_SHIFT) - 1));
819 #endif
820
821 early_ioremap_setup();
822
823 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
824 memset(bm_pte, 0, sizeof(bm_pte));
825 pmd_populate_kernel(&init_mm, pmd, bm_pte);
826
827 /*
828 * The boot-ioremap range spans multiple pmds, for which
829 * we are not prepared:
830 */
831 #define __FIXADDR_TOP (-PAGE_SIZE)
832 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
833 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
834 #undef __FIXADDR_TOP
835 if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
836 WARN_ON(1);
837 printk(KERN_WARNING "pmd %p != %p\n",
838 pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
839 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
840 fix_to_virt(FIX_BTMAP_BEGIN));
841 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n",
842 fix_to_virt(FIX_BTMAP_END));
843
844 printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
845 printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n",
846 FIX_BTMAP_BEGIN);
847 }
848 }
849
__early_set_fixmap(enum fixed_addresses idx,phys_addr_t phys,pgprot_t flags)850 void __init __early_set_fixmap(enum fixed_addresses idx,
851 phys_addr_t phys, pgprot_t flags)
852 {
853 unsigned long addr = __fix_to_virt(idx);
854 pte_t *pte;
855
856 if (idx >= __end_of_fixed_addresses) {
857 BUG();
858 return;
859 }
860 pte = early_ioremap_pte(addr);
861
862 /* Sanitize 'prot' against any unsupported bits: */
863 pgprot_val(flags) &= __supported_pte_mask;
864
865 if (pgprot_val(flags))
866 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
867 else
868 pte_clear(&init_mm, addr, pte);
869 flush_tlb_one_kernel(addr);
870 }
871