1084cfca1SRichard Henderson /* 2084cfca1SRichard Henderson * Flush the host cpu caches. 3084cfca1SRichard Henderson * 4084cfca1SRichard Henderson * This work is licensed under the terms of the GNU GPL, version 2 or later. 5084cfca1SRichard Henderson * See the COPYING file in the top-level directory. 6084cfca1SRichard Henderson */ 7084cfca1SRichard Henderson 8084cfca1SRichard Henderson #include "qemu/osdep.h" 9084cfca1SRichard Henderson #include "qemu/cacheflush.h" 10664a7973SRichard Henderson #include "qemu/bitops.h" 11084cfca1SRichard Henderson 12084cfca1SRichard Henderson 13084cfca1SRichard Henderson #if defined(__i386__) || defined(__x86_64__) || defined(__s390__) 14084cfca1SRichard Henderson 15084cfca1SRichard Henderson /* Caches are coherent and do not require flushing; symbol inline. */ 16084cfca1SRichard Henderson 17664a7973SRichard Henderson #elif defined(__aarch64__) 18664a7973SRichard Henderson 19664a7973SRichard Henderson #ifdef CONFIG_DARWIN 20664a7973SRichard Henderson /* Apple does not expose CTR_EL0, so we must use system interfaces. */ 21664a7973SRichard Henderson extern void sys_icache_invalidate(void *start, size_t len); 22664a7973SRichard Henderson extern void sys_dcache_flush(void *start, size_t len); 23664a7973SRichard Henderson void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 24664a7973SRichard Henderson { 25664a7973SRichard Henderson sys_dcache_flush((void *)rw, len); 26664a7973SRichard Henderson sys_icache_invalidate((void *)rx, len); 27664a7973SRichard Henderson } 28664a7973SRichard Henderson #else 29664a7973SRichard Henderson 30664a7973SRichard Henderson /* 31664a7973SRichard Henderson * TODO: unify this with cacheinfo.c. 32664a7973SRichard Henderson * We want to save the whole contents of CTR_EL0, so that we 33664a7973SRichard Henderson * have more than the linesize, but also IDC and DIC. 34664a7973SRichard Henderson */ 35*acd15fc2SGan Qixin static uint64_t save_ctr_el0; 36664a7973SRichard Henderson static void __attribute__((constructor)) init_ctr_el0(void) 37664a7973SRichard Henderson { 38664a7973SRichard Henderson asm volatile("mrs\t%0, ctr_el0" : "=r"(save_ctr_el0)); 39664a7973SRichard Henderson } 40664a7973SRichard Henderson 41664a7973SRichard Henderson /* 42664a7973SRichard Henderson * This is a copy of gcc's __aarch64_sync_cache_range, modified 43664a7973SRichard Henderson * to fit this three-operand interface. 44664a7973SRichard Henderson */ 45664a7973SRichard Henderson void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 46664a7973SRichard Henderson { 47664a7973SRichard Henderson const unsigned CTR_IDC = 1u << 28; 48664a7973SRichard Henderson const unsigned CTR_DIC = 1u << 29; 49*acd15fc2SGan Qixin const uint64_t ctr_el0 = save_ctr_el0; 50*acd15fc2SGan Qixin const uintptr_t icache_lsize = 4 << extract64(ctr_el0, 0, 4); 51*acd15fc2SGan Qixin const uintptr_t dcache_lsize = 4 << extract64(ctr_el0, 16, 4); 52664a7973SRichard Henderson uintptr_t p; 53664a7973SRichard Henderson 54664a7973SRichard Henderson /* 55664a7973SRichard Henderson * If CTR_EL0.IDC is enabled, Data cache clean to the Point of Unification 56664a7973SRichard Henderson * is not required for instruction to data coherence. 57664a7973SRichard Henderson */ 58664a7973SRichard Henderson if (!(ctr_el0 & CTR_IDC)) { 59664a7973SRichard Henderson /* 60664a7973SRichard Henderson * Loop over the address range, clearing one cache line at once. 61664a7973SRichard Henderson * Data cache must be flushed to unification first to make sure 62664a7973SRichard Henderson * the instruction cache fetches the updated data. 63664a7973SRichard Henderson */ 64664a7973SRichard Henderson for (p = rw & -dcache_lsize; p < rw + len; p += dcache_lsize) { 65664a7973SRichard Henderson asm volatile("dc\tcvau, %0" : : "r" (p) : "memory"); 66664a7973SRichard Henderson } 67664a7973SRichard Henderson asm volatile("dsb\tish" : : : "memory"); 68664a7973SRichard Henderson } 69664a7973SRichard Henderson 70664a7973SRichard Henderson /* 71664a7973SRichard Henderson * If CTR_EL0.DIC is enabled, Instruction cache cleaning to the Point 72664a7973SRichard Henderson * of Unification is not required for instruction to data coherence. 73664a7973SRichard Henderson */ 74664a7973SRichard Henderson if (!(ctr_el0 & CTR_DIC)) { 75664a7973SRichard Henderson for (p = rx & -icache_lsize; p < rx + len; p += icache_lsize) { 76664a7973SRichard Henderson asm volatile("ic\tivau, %0" : : "r"(p) : "memory"); 77664a7973SRichard Henderson } 78664a7973SRichard Henderson asm volatile ("dsb\tish" : : : "memory"); 79664a7973SRichard Henderson } 80664a7973SRichard Henderson 81664a7973SRichard Henderson asm volatile("isb" : : : "memory"); 82664a7973SRichard Henderson } 83664a7973SRichard Henderson #endif /* CONFIG_DARWIN */ 84664a7973SRichard Henderson 85084cfca1SRichard Henderson #elif defined(__mips__) 86084cfca1SRichard Henderson 87084cfca1SRichard Henderson #ifdef __OpenBSD__ 88084cfca1SRichard Henderson #include <machine/sysarch.h> 89084cfca1SRichard Henderson #else 90084cfca1SRichard Henderson #include <sys/cachectl.h> 91084cfca1SRichard Henderson #endif 92084cfca1SRichard Henderson 931da8de39SRichard Henderson void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 94084cfca1SRichard Henderson { 951da8de39SRichard Henderson if (rx != rw) { 961da8de39SRichard Henderson cacheflush((void *)rw, len, DCACHE); 971da8de39SRichard Henderson } 981da8de39SRichard Henderson cacheflush((void *)rx, len, ICACHE); 99084cfca1SRichard Henderson } 100084cfca1SRichard Henderson 101084cfca1SRichard Henderson #elif defined(__powerpc__) 102084cfca1SRichard Henderson 1031da8de39SRichard Henderson void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 104084cfca1SRichard Henderson { 1051da8de39SRichard Henderson uintptr_t p, b, e; 106084cfca1SRichard Henderson size_t dsize = qemu_dcache_linesize; 107084cfca1SRichard Henderson size_t isize = qemu_icache_linesize; 108084cfca1SRichard Henderson 1091da8de39SRichard Henderson b = rw & ~(dsize - 1); 1101da8de39SRichard Henderson e = (rw + len + dsize - 1) & ~(dsize - 1); 1111da8de39SRichard Henderson for (p = b; p < e; p += dsize) { 112084cfca1SRichard Henderson asm volatile ("dcbst 0,%0" : : "r"(p) : "memory"); 113084cfca1SRichard Henderson } 114084cfca1SRichard Henderson asm volatile ("sync" : : : "memory"); 115084cfca1SRichard Henderson 1161da8de39SRichard Henderson b = rx & ~(isize - 1); 1171da8de39SRichard Henderson e = (rx + len + isize - 1) & ~(isize - 1); 1181da8de39SRichard Henderson for (p = b; p < e; p += isize) { 119084cfca1SRichard Henderson asm volatile ("icbi 0,%0" : : "r"(p) : "memory"); 120084cfca1SRichard Henderson } 121084cfca1SRichard Henderson asm volatile ("sync" : : : "memory"); 122084cfca1SRichard Henderson asm volatile ("isync" : : : "memory"); 123084cfca1SRichard Henderson } 124084cfca1SRichard Henderson 125084cfca1SRichard Henderson #elif defined(__sparc__) 126084cfca1SRichard Henderson 1271da8de39SRichard Henderson void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 128084cfca1SRichard Henderson { 1291da8de39SRichard Henderson /* No additional data flush to the RW virtual address required. */ 1301da8de39SRichard Henderson uintptr_t p, end = (rx + len + 7) & -8; 1311da8de39SRichard Henderson for (p = rx & -8; p < end; p += 8) { 132084cfca1SRichard Henderson __asm__ __volatile__("flush\t%0" : : "r" (p)); 133084cfca1SRichard Henderson } 134084cfca1SRichard Henderson } 135084cfca1SRichard Henderson 136084cfca1SRichard Henderson #else 137084cfca1SRichard Henderson 1381da8de39SRichard Henderson void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 139084cfca1SRichard Henderson { 1401da8de39SRichard Henderson if (rw != rx) { 1411da8de39SRichard Henderson __builtin___clear_cache((char *)rw, (char *)rw + len); 1421da8de39SRichard Henderson } 1431da8de39SRichard Henderson __builtin___clear_cache((char *)rx, (char *)rx + len); 144084cfca1SRichard Henderson } 145084cfca1SRichard Henderson 146084cfca1SRichard Henderson #endif 147