1084cfca1SRichard Henderson /* 2084cfca1SRichard Henderson * Flush the host cpu caches. 3084cfca1SRichard Henderson * 4084cfca1SRichard Henderson * This work is licensed under the terms of the GNU GPL, version 2 or later. 5084cfca1SRichard Henderson * See the COPYING file in the top-level directory. 6084cfca1SRichard Henderson */ 7084cfca1SRichard Henderson 8084cfca1SRichard Henderson #include "qemu/osdep.h" 9084cfca1SRichard Henderson #include "qemu/cacheflush.h" 10*ad768e6fSPeter Maydell #include "qemu/cacheinfo.h" 11664a7973SRichard Henderson #include "qemu/bitops.h" 12084cfca1SRichard Henderson 13084cfca1SRichard Henderson 14084cfca1SRichard Henderson #if defined(__i386__) || defined(__x86_64__) || defined(__s390__) 15084cfca1SRichard Henderson 16084cfca1SRichard Henderson /* Caches are coherent and do not require flushing; symbol inline. */ 17084cfca1SRichard Henderson 18664a7973SRichard Henderson #elif defined(__aarch64__) 19664a7973SRichard Henderson 20664a7973SRichard Henderson #ifdef CONFIG_DARWIN 21664a7973SRichard Henderson /* Apple does not expose CTR_EL0, so we must use system interfaces. */ 22664a7973SRichard Henderson extern void sys_icache_invalidate(void *start, size_t len); 23664a7973SRichard Henderson extern void sys_dcache_flush(void *start, size_t len); 24664a7973SRichard Henderson void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 25664a7973SRichard Henderson { 26664a7973SRichard Henderson sys_dcache_flush((void *)rw, len); 27664a7973SRichard Henderson sys_icache_invalidate((void *)rx, len); 28664a7973SRichard Henderson } 29664a7973SRichard Henderson #else 30664a7973SRichard Henderson 31664a7973SRichard Henderson /* 32664a7973SRichard Henderson * TODO: unify this with cacheinfo.c. 33664a7973SRichard Henderson * We want to save the whole contents of CTR_EL0, so that we 34664a7973SRichard Henderson * have more than the linesize, but also IDC and DIC. 35664a7973SRichard Henderson */ 36acd15fc2SGan Qixin static uint64_t save_ctr_el0; 37664a7973SRichard Henderson static void __attribute__((constructor)) init_ctr_el0(void) 38664a7973SRichard Henderson { 39664a7973SRichard Henderson asm volatile("mrs\t%0, ctr_el0" : "=r"(save_ctr_el0)); 40664a7973SRichard Henderson } 41664a7973SRichard Henderson 42664a7973SRichard Henderson /* 43664a7973SRichard Henderson * This is a copy of gcc's __aarch64_sync_cache_range, modified 44664a7973SRichard Henderson * to fit this three-operand interface. 45664a7973SRichard Henderson */ 46664a7973SRichard Henderson void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 47664a7973SRichard Henderson { 48664a7973SRichard Henderson const unsigned CTR_IDC = 1u << 28; 49664a7973SRichard Henderson const unsigned CTR_DIC = 1u << 29; 50acd15fc2SGan Qixin const uint64_t ctr_el0 = save_ctr_el0; 51acd15fc2SGan Qixin const uintptr_t icache_lsize = 4 << extract64(ctr_el0, 0, 4); 52acd15fc2SGan Qixin const uintptr_t dcache_lsize = 4 << extract64(ctr_el0, 16, 4); 53664a7973SRichard Henderson uintptr_t p; 54664a7973SRichard Henderson 55664a7973SRichard Henderson /* 56664a7973SRichard Henderson * If CTR_EL0.IDC is enabled, Data cache clean to the Point of Unification 57664a7973SRichard Henderson * is not required for instruction to data coherence. 58664a7973SRichard Henderson */ 59664a7973SRichard Henderson if (!(ctr_el0 & CTR_IDC)) { 60664a7973SRichard Henderson /* 61664a7973SRichard Henderson * Loop over the address range, clearing one cache line at once. 62664a7973SRichard Henderson * Data cache must be flushed to unification first to make sure 63664a7973SRichard Henderson * the instruction cache fetches the updated data. 64664a7973SRichard Henderson */ 65664a7973SRichard Henderson for (p = rw & -dcache_lsize; p < rw + len; p += dcache_lsize) { 66664a7973SRichard Henderson asm volatile("dc\tcvau, %0" : : "r" (p) : "memory"); 67664a7973SRichard Henderson } 68664a7973SRichard Henderson asm volatile("dsb\tish" : : : "memory"); 69664a7973SRichard Henderson } 70664a7973SRichard Henderson 71664a7973SRichard Henderson /* 72664a7973SRichard Henderson * If CTR_EL0.DIC is enabled, Instruction cache cleaning to the Point 73664a7973SRichard Henderson * of Unification is not required for instruction to data coherence. 74664a7973SRichard Henderson */ 75664a7973SRichard Henderson if (!(ctr_el0 & CTR_DIC)) { 76664a7973SRichard Henderson for (p = rx & -icache_lsize; p < rx + len; p += icache_lsize) { 77664a7973SRichard Henderson asm volatile("ic\tivau, %0" : : "r"(p) : "memory"); 78664a7973SRichard Henderson } 79664a7973SRichard Henderson asm volatile ("dsb\tish" : : : "memory"); 80664a7973SRichard Henderson } 81664a7973SRichard Henderson 82664a7973SRichard Henderson asm volatile("isb" : : : "memory"); 83664a7973SRichard Henderson } 84664a7973SRichard Henderson #endif /* CONFIG_DARWIN */ 85664a7973SRichard Henderson 86084cfca1SRichard Henderson #elif defined(__mips__) 87084cfca1SRichard Henderson 88084cfca1SRichard Henderson #ifdef __OpenBSD__ 89084cfca1SRichard Henderson #include <machine/sysarch.h> 90084cfca1SRichard Henderson #else 91084cfca1SRichard Henderson #include <sys/cachectl.h> 92084cfca1SRichard Henderson #endif 93084cfca1SRichard Henderson 941da8de39SRichard Henderson void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 95084cfca1SRichard Henderson { 961da8de39SRichard Henderson if (rx != rw) { 971da8de39SRichard Henderson cacheflush((void *)rw, len, DCACHE); 981da8de39SRichard Henderson } 991da8de39SRichard Henderson cacheflush((void *)rx, len, ICACHE); 100084cfca1SRichard Henderson } 101084cfca1SRichard Henderson 102084cfca1SRichard Henderson #elif defined(__powerpc__) 103084cfca1SRichard Henderson 1041da8de39SRichard Henderson void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 105084cfca1SRichard Henderson { 1061da8de39SRichard Henderson uintptr_t p, b, e; 107084cfca1SRichard Henderson size_t dsize = qemu_dcache_linesize; 108084cfca1SRichard Henderson size_t isize = qemu_icache_linesize; 109084cfca1SRichard Henderson 1101da8de39SRichard Henderson b = rw & ~(dsize - 1); 1111da8de39SRichard Henderson e = (rw + len + dsize - 1) & ~(dsize - 1); 1121da8de39SRichard Henderson for (p = b; p < e; p += dsize) { 113084cfca1SRichard Henderson asm volatile ("dcbst 0,%0" : : "r"(p) : "memory"); 114084cfca1SRichard Henderson } 115084cfca1SRichard Henderson asm volatile ("sync" : : : "memory"); 116084cfca1SRichard Henderson 1171da8de39SRichard Henderson b = rx & ~(isize - 1); 1181da8de39SRichard Henderson e = (rx + len + isize - 1) & ~(isize - 1); 1191da8de39SRichard Henderson for (p = b; p < e; p += isize) { 120084cfca1SRichard Henderson asm volatile ("icbi 0,%0" : : "r"(p) : "memory"); 121084cfca1SRichard Henderson } 122084cfca1SRichard Henderson asm volatile ("sync" : : : "memory"); 123084cfca1SRichard Henderson asm volatile ("isync" : : : "memory"); 124084cfca1SRichard Henderson } 125084cfca1SRichard Henderson 126084cfca1SRichard Henderson #elif defined(__sparc__) 127084cfca1SRichard Henderson 1281da8de39SRichard Henderson void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 129084cfca1SRichard Henderson { 1301da8de39SRichard Henderson /* No additional data flush to the RW virtual address required. */ 1311da8de39SRichard Henderson uintptr_t p, end = (rx + len + 7) & -8; 1321da8de39SRichard Henderson for (p = rx & -8; p < end; p += 8) { 133084cfca1SRichard Henderson __asm__ __volatile__("flush\t%0" : : "r" (p)); 134084cfca1SRichard Henderson } 135084cfca1SRichard Henderson } 136084cfca1SRichard Henderson 137084cfca1SRichard Henderson #else 138084cfca1SRichard Henderson 1391da8de39SRichard Henderson void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 140084cfca1SRichard Henderson { 1411da8de39SRichard Henderson if (rw != rx) { 1421da8de39SRichard Henderson __builtin___clear_cache((char *)rw, (char *)rw + len); 1431da8de39SRichard Henderson } 1441da8de39SRichard Henderson __builtin___clear_cache((char *)rx, (char *)rx + len); 145084cfca1SRichard Henderson } 146084cfca1SRichard Henderson 147084cfca1SRichard Henderson #endif 148