1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6 #ifndef _ASM_RISCV_PGTABLE_64_H 7 #define _ASM_RISCV_PGTABLE_64_H 8 9 #include <linux/bits.h> 10 #include <linux/const.h> 11 #include <asm/errata_list.h> 12 13 extern bool pgtable_l4_enabled; 14 extern bool pgtable_l5_enabled; 15 16 #define PGDIR_SHIFT_L3 30 17 #define PGDIR_SHIFT_L4 39 18 #define PGDIR_SHIFT_L5 48 19 #define PGDIR_SIZE_L3 (_AC(1, UL) << PGDIR_SHIFT_L3) 20 21 #define PGDIR_SHIFT (pgtable_l5_enabled ? PGDIR_SHIFT_L5 : \ 22 (pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3)) 23 /* Size of region mapped by a page global directory */ 24 #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) 25 #define PGDIR_MASK (~(PGDIR_SIZE - 1)) 26 27 /* p4d is folded into pgd in case of 4-level page table */ 28 #define P4D_SHIFT 39 29 #define P4D_SIZE (_AC(1, UL) << P4D_SHIFT) 30 #define P4D_MASK (~(P4D_SIZE - 1)) 31 32 /* pud is folded into pgd in case of 3-level page table */ 33 #define PUD_SHIFT 30 34 #define PUD_SIZE (_AC(1, UL) << PUD_SHIFT) 35 #define PUD_MASK (~(PUD_SIZE - 1)) 36 37 #define PMD_SHIFT 21 38 /* Size of region mapped by a page middle directory */ 39 #define PMD_SIZE (_AC(1, UL) << PMD_SHIFT) 40 #define PMD_MASK (~(PMD_SIZE - 1)) 41 42 /* Page 4th Directory entry */ 43 typedef struct { 44 unsigned long p4d; 45 } p4d_t; 46 47 #define p4d_val(x) ((x).p4d) 48 #define __p4d(x) ((p4d_t) { (x) }) 49 #define PTRS_PER_P4D (PAGE_SIZE / sizeof(p4d_t)) 50 51 /* Page Upper Directory entry */ 52 typedef struct { 53 unsigned long pud; 54 } pud_t; 55 56 #define pud_val(x) ((x).pud) 57 #define __pud(x) ((pud_t) { (x) }) 58 #define PTRS_PER_PUD (PAGE_SIZE / sizeof(pud_t)) 59 60 /* Page Middle Directory entry */ 61 typedef struct { 62 unsigned long pmd; 63 } pmd_t; 64 65 #define pmd_val(x) ((x).pmd) 66 #define __pmd(x) ((pmd_t) { (x) }) 67 68 #define PTRS_PER_PMD (PAGE_SIZE / sizeof(pmd_t)) 69 70 /* 71 * rv64 PTE format: 72 * | 63 | 62 61 | 60 54 | 53 10 | 9 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 73 * N MT RSV PFN reserved for SW D A G U X W R V 74 */ 75 #define _PAGE_PFN_MASK GENMASK(53, 10) 76 77 /* 78 * [62:61] Svpbmt Memory Type definitions: 79 * 80 * 00 - PMA Normal Cacheable, No change to implied PMA memory type 81 * 01 - NC Non-cacheable, idempotent, weakly-ordered Main Memory 82 * 10 - IO Non-cacheable, non-idempotent, strongly-ordered I/O memory 83 * 11 - Rsvd Reserved for future standard use 84 */ 85 #define _PAGE_NOCACHE_SVPBMT (1UL << 61) 86 #define _PAGE_IO_SVPBMT (1UL << 62) 87 #define _PAGE_MTMASK_SVPBMT (_PAGE_NOCACHE_SVPBMT | _PAGE_IO_SVPBMT) 88 89 /* 90 * [63:59] T-Head Memory Type definitions: 91 * 92 * 00000 - NC Weakly-ordered, Non-cacheable, Non-bufferable, Non-shareable, Non-trustable 93 * 01110 - PMA Weakly-ordered, Cacheable, Bufferable, Shareable, Non-trustable 94 * 10000 - IO Strongly-ordered, Non-cacheable, Non-bufferable, Non-shareable, Non-trustable 95 */ 96 #define _PAGE_PMA_THEAD ((1UL << 62) | (1UL << 61) | (1UL << 60)) 97 #define _PAGE_NOCACHE_THEAD 0UL 98 #define _PAGE_IO_THEAD (1UL << 63) 99 #define _PAGE_MTMASK_THEAD (_PAGE_PMA_THEAD | _PAGE_IO_THEAD | (1UL << 59)) 100 101 static inline u64 riscv_page_mtmask(void) 102 { 103 u64 val; 104 105 ALT_SVPBMT(val, _PAGE_MTMASK); 106 return val; 107 } 108 109 static inline u64 riscv_page_nocache(void) 110 { 111 u64 val; 112 113 ALT_SVPBMT(val, _PAGE_NOCACHE); 114 return val; 115 } 116 117 static inline u64 riscv_page_io(void) 118 { 119 u64 val; 120 121 ALT_SVPBMT(val, _PAGE_IO); 122 return val; 123 } 124 125 #define _PAGE_NOCACHE riscv_page_nocache() 126 #define _PAGE_IO riscv_page_io() 127 #define _PAGE_MTMASK riscv_page_mtmask() 128 129 /* Set of bits to preserve across pte_modify() */ 130 #define _PAGE_CHG_MASK (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ | \ 131 _PAGE_WRITE | _PAGE_EXEC | \ 132 _PAGE_USER | _PAGE_GLOBAL | \ 133 _PAGE_MTMASK)) 134 135 static inline int pud_present(pud_t pud) 136 { 137 return (pud_val(pud) & _PAGE_PRESENT); 138 } 139 140 static inline int pud_none(pud_t pud) 141 { 142 return (pud_val(pud) == 0); 143 } 144 145 static inline int pud_bad(pud_t pud) 146 { 147 return !pud_present(pud); 148 } 149 150 #define pud_leaf pud_leaf 151 static inline int pud_leaf(pud_t pud) 152 { 153 return pud_present(pud) && (pud_val(pud) & _PAGE_LEAF); 154 } 155 156 static inline int pud_user(pud_t pud) 157 { 158 return pud_val(pud) & _PAGE_USER; 159 } 160 161 static inline void set_pud(pud_t *pudp, pud_t pud) 162 { 163 *pudp = pud; 164 } 165 166 static inline void pud_clear(pud_t *pudp) 167 { 168 set_pud(pudp, __pud(0)); 169 } 170 171 static inline pud_t pfn_pud(unsigned long pfn, pgprot_t prot) 172 { 173 return __pud((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); 174 } 175 176 static inline unsigned long _pud_pfn(pud_t pud) 177 { 178 return __page_val_to_pfn(pud_val(pud)); 179 } 180 181 static inline pmd_t *pud_pgtable(pud_t pud) 182 { 183 return (pmd_t *)pfn_to_virt(__page_val_to_pfn(pud_val(pud))); 184 } 185 186 static inline struct page *pud_page(pud_t pud) 187 { 188 return pfn_to_page(__page_val_to_pfn(pud_val(pud))); 189 } 190 191 #define mm_p4d_folded mm_p4d_folded 192 static inline bool mm_p4d_folded(struct mm_struct *mm) 193 { 194 if (pgtable_l5_enabled) 195 return false; 196 197 return true; 198 } 199 200 #define mm_pud_folded mm_pud_folded 201 static inline bool mm_pud_folded(struct mm_struct *mm) 202 { 203 if (pgtable_l4_enabled) 204 return false; 205 206 return true; 207 } 208 209 #define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) 210 211 static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot) 212 { 213 unsigned long prot_val = pgprot_val(prot); 214 215 ALT_THEAD_PMA(prot_val); 216 217 return __pmd((pfn << _PAGE_PFN_SHIFT) | prot_val); 218 } 219 220 static inline unsigned long _pmd_pfn(pmd_t pmd) 221 { 222 return __page_val_to_pfn(pmd_val(pmd)); 223 } 224 225 #define mk_pmd(page, prot) pfn_pmd(page_to_pfn(page), prot) 226 227 #define pmd_ERROR(e) \ 228 pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) 229 230 #define pud_ERROR(e) \ 231 pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) 232 233 #define p4d_ERROR(e) \ 234 pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e)) 235 236 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) 237 { 238 if (pgtable_l4_enabled) 239 *p4dp = p4d; 240 else 241 set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); 242 } 243 244 static inline int p4d_none(p4d_t p4d) 245 { 246 if (pgtable_l4_enabled) 247 return (p4d_val(p4d) == 0); 248 249 return 0; 250 } 251 252 static inline int p4d_present(p4d_t p4d) 253 { 254 if (pgtable_l4_enabled) 255 return (p4d_val(p4d) & _PAGE_PRESENT); 256 257 return 1; 258 } 259 260 static inline int p4d_bad(p4d_t p4d) 261 { 262 if (pgtable_l4_enabled) 263 return !p4d_present(p4d); 264 265 return 0; 266 } 267 268 static inline void p4d_clear(p4d_t *p4d) 269 { 270 if (pgtable_l4_enabled) 271 set_p4d(p4d, __p4d(0)); 272 } 273 274 static inline p4d_t pfn_p4d(unsigned long pfn, pgprot_t prot) 275 { 276 return __p4d((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); 277 } 278 279 static inline unsigned long _p4d_pfn(p4d_t p4d) 280 { 281 return __page_val_to_pfn(p4d_val(p4d)); 282 } 283 284 static inline pud_t *p4d_pgtable(p4d_t p4d) 285 { 286 if (pgtable_l4_enabled) 287 return (pud_t *)pfn_to_virt(__page_val_to_pfn(p4d_val(p4d))); 288 289 return (pud_t *)pud_pgtable((pud_t) { p4d_val(p4d) }); 290 } 291 #define p4d_page_vaddr(p4d) ((unsigned long)p4d_pgtable(p4d)) 292 293 static inline struct page *p4d_page(p4d_t p4d) 294 { 295 return pfn_to_page(__page_val_to_pfn(p4d_val(p4d))); 296 } 297 298 #define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) 299 300 #define pud_offset pud_offset 301 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) 302 { 303 if (pgtable_l4_enabled) 304 return p4d_pgtable(*p4d) + pud_index(address); 305 306 return (pud_t *)p4d; 307 } 308 309 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) 310 { 311 if (pgtable_l5_enabled) 312 *pgdp = pgd; 313 else 314 set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) }); 315 } 316 317 static inline int pgd_none(pgd_t pgd) 318 { 319 if (pgtable_l5_enabled) 320 return (pgd_val(pgd) == 0); 321 322 return 0; 323 } 324 325 static inline int pgd_present(pgd_t pgd) 326 { 327 if (pgtable_l5_enabled) 328 return (pgd_val(pgd) & _PAGE_PRESENT); 329 330 return 1; 331 } 332 333 static inline int pgd_bad(pgd_t pgd) 334 { 335 if (pgtable_l5_enabled) 336 return !pgd_present(pgd); 337 338 return 0; 339 } 340 341 static inline void pgd_clear(pgd_t *pgd) 342 { 343 if (pgtable_l5_enabled) 344 set_pgd(pgd, __pgd(0)); 345 } 346 347 static inline p4d_t *pgd_pgtable(pgd_t pgd) 348 { 349 if (pgtable_l5_enabled) 350 return (p4d_t *)pfn_to_virt(__page_val_to_pfn(pgd_val(pgd))); 351 352 return (p4d_t *)p4d_pgtable((p4d_t) { pgd_val(pgd) }); 353 } 354 #define pgd_page_vaddr(pgd) ((unsigned long)pgd_pgtable(pgd)) 355 356 static inline struct page *pgd_page(pgd_t pgd) 357 { 358 return pfn_to_page(__page_val_to_pfn(pgd_val(pgd))); 359 } 360 #define pgd_page(pgd) pgd_page(pgd) 361 362 #define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1)) 363 364 #define p4d_offset p4d_offset 365 static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) 366 { 367 if (pgtable_l5_enabled) 368 return pgd_pgtable(*pgd) + p4d_index(address); 369 370 return (p4d_t *)pgd; 371 } 372 373 #endif /* _ASM_RISCV_PGTABLE_64_H */ 374