1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * SiFive composable cache controller Driver 4 * 5 * Copyright (C) 2018-2022 SiFive, Inc. 6 * 7 */ 8 9 #define pr_fmt(fmt) "CCACHE: " fmt 10 11 #include <linux/align.h> 12 #include <linux/debugfs.h> 13 #include <linux/interrupt.h> 14 #include <linux/of_irq.h> 15 #include <linux/of_address.h> 16 #include <linux/device.h> 17 #include <linux/bitfield.h> 18 #include <linux/platform_device.h> 19 #include <linux/property.h> 20 #include <asm/cacheflush.h> 21 #include <asm/cacheinfo.h> 22 #include <asm/dma-noncoherent.h> 23 #include <soc/sifive/sifive_ccache.h> 24 25 #define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100 26 #define SIFIVE_CCACHE_DIRECCFIX_HIGH 0x104 27 #define SIFIVE_CCACHE_DIRECCFIX_COUNT 0x108 28 29 #define SIFIVE_CCACHE_DIRECCFAIL_LOW 0x120 30 #define SIFIVE_CCACHE_DIRECCFAIL_HIGH 0x124 31 #define SIFIVE_CCACHE_DIRECCFAIL_COUNT 0x128 32 33 #define SIFIVE_CCACHE_DATECCFIX_LOW 0x140 34 #define SIFIVE_CCACHE_DATECCFIX_HIGH 0x144 35 #define SIFIVE_CCACHE_DATECCFIX_COUNT 0x148 36 37 #define SIFIVE_CCACHE_DATECCFAIL_LOW 0x160 38 #define SIFIVE_CCACHE_DATECCFAIL_HIGH 0x164 39 #define SIFIVE_CCACHE_DATECCFAIL_COUNT 0x168 40 41 #define SIFIVE_CCACHE_CONFIG 0x00 42 #define SIFIVE_CCACHE_CONFIG_BANK_MASK GENMASK_ULL(7, 0) 43 #define SIFIVE_CCACHE_CONFIG_WAYS_MASK GENMASK_ULL(15, 8) 44 #define SIFIVE_CCACHE_CONFIG_SETS_MASK GENMASK_ULL(23, 16) 45 #define SIFIVE_CCACHE_CONFIG_BLKS_MASK GENMASK_ULL(31, 24) 46 47 #define SIFIVE_CCACHE_FLUSH64 0x200 48 #define SIFIVE_CCACHE_FLUSH32 0x240 49 50 #define SIFIVE_CCACHE_WAYENABLE 0x08 51 #define SIFIVE_CCACHE_ECCINJECTERR 0x40 52 53 #define SIFIVE_CCACHE_MAX_ECCINTR 4 54 #define SIFIVE_CCACHE_LINE_SIZE 64 55 56 static void __iomem *ccache_base; 57 static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR]; 58 static struct riscv_cacheinfo_ops ccache_cache_ops; 59 static int level; 60 61 enum { 62 DIR_CORR = 0, 63 DATA_CORR, 64 DATA_UNCORR, 65 DIR_UNCORR, 66 }; 67 68 enum { 69 QUIRK_NONSTANDARD_CACHE_OPS = BIT(0), 70 QUIRK_BROKEN_DATA_UNCORR = BIT(1), 71 }; 72 73 #ifdef CONFIG_DEBUG_FS 74 static struct dentry *sifive_test; 75 76 static ssize_t ccache_write(struct file *file, const char __user *data, 77 size_t count, loff_t *ppos) 78 { 79 unsigned int val; 80 81 if (kstrtouint_from_user(data, count, 0, &val)) 82 return -EINVAL; 83 if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF)) 84 writel(val, ccache_base + SIFIVE_CCACHE_ECCINJECTERR); 85 else 86 return -EINVAL; 87 return count; 88 } 89 90 static const struct file_operations ccache_fops = { 91 .owner = THIS_MODULE, 92 .open = simple_open, 93 .write = ccache_write 94 }; 95 96 static void setup_sifive_debug(void) 97 { 98 sifive_test = debugfs_create_dir("sifive_ccache_cache", NULL); 99 100 debugfs_create_file("sifive_debug_inject_error", 0200, 101 sifive_test, NULL, &ccache_fops); 102 } 103 #endif 104 105 static void ccache_config_read(void) 106 { 107 u32 cfg; 108 109 cfg = readl(ccache_base + SIFIVE_CCACHE_CONFIG); 110 pr_info("%llu banks, %llu ways, sets/bank=%llu, bytes/block=%llu\n", 111 FIELD_GET(SIFIVE_CCACHE_CONFIG_BANK_MASK, cfg), 112 FIELD_GET(SIFIVE_CCACHE_CONFIG_WAYS_MASK, cfg), 113 BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_SETS_MASK, cfg)), 114 BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_BLKS_MASK, cfg))); 115 116 cfg = readl(ccache_base + SIFIVE_CCACHE_WAYENABLE); 117 pr_info("Index of the largest way enabled: %u\n", cfg); 118 } 119 120 static const struct of_device_id sifive_ccache_ids[] = { 121 { .compatible = "sifive,fu540-c000-ccache" }, 122 { .compatible = "sifive,fu740-c000-ccache" }, 123 { .compatible = "starfive,jh7100-ccache", 124 .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS | QUIRK_BROKEN_DATA_UNCORR) }, 125 { .compatible = "sifive,ccache0" }, 126 { /* end of table */ } 127 }; 128 129 static ATOMIC_NOTIFIER_HEAD(ccache_err_chain); 130 131 int register_sifive_ccache_error_notifier(struct notifier_block *nb) 132 { 133 return atomic_notifier_chain_register(&ccache_err_chain, nb); 134 } 135 EXPORT_SYMBOL_GPL(register_sifive_ccache_error_notifier); 136 137 int unregister_sifive_ccache_error_notifier(struct notifier_block *nb) 138 { 139 return atomic_notifier_chain_unregister(&ccache_err_chain, nb); 140 } 141 EXPORT_SYMBOL_GPL(unregister_sifive_ccache_error_notifier); 142 143 #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS 144 static void ccache_flush_range(phys_addr_t start, size_t len) 145 { 146 phys_addr_t end = start + len; 147 phys_addr_t line; 148 149 if (!len) 150 return; 151 152 mb(); 153 for (line = ALIGN_DOWN(start, SIFIVE_CCACHE_LINE_SIZE); line < end; 154 line += SIFIVE_CCACHE_LINE_SIZE) { 155 #ifdef CONFIG_32BIT 156 writel(line >> 4, ccache_base + SIFIVE_CCACHE_FLUSH32); 157 #else 158 writeq(line, ccache_base + SIFIVE_CCACHE_FLUSH64); 159 #endif 160 mb(); 161 } 162 } 163 164 static const struct riscv_nonstd_cache_ops ccache_mgmt_ops __initconst = { 165 .wback = &ccache_flush_range, 166 .inv = &ccache_flush_range, 167 .wback_inv = &ccache_flush_range, 168 }; 169 #endif /* CONFIG_RISCV_NONSTANDARD_CACHE_OPS */ 170 171 static int ccache_largest_wayenabled(void) 172 { 173 return readl(ccache_base + SIFIVE_CCACHE_WAYENABLE) & 0xFF; 174 } 175 176 static ssize_t number_of_ways_enabled_show(struct device *dev, 177 struct device_attribute *attr, 178 char *buf) 179 { 180 return sprintf(buf, "%u\n", ccache_largest_wayenabled()); 181 } 182 183 static DEVICE_ATTR_RO(number_of_ways_enabled); 184 185 static struct attribute *priv_attrs[] = { 186 &dev_attr_number_of_ways_enabled.attr, 187 NULL, 188 }; 189 190 static const struct attribute_group priv_attr_group = { 191 .attrs = priv_attrs, 192 }; 193 194 static const struct attribute_group *ccache_get_priv_group(struct cacheinfo 195 *this_leaf) 196 { 197 /* We want to use private group for composable cache only */ 198 if (this_leaf->level == level) 199 return &priv_attr_group; 200 else 201 return NULL; 202 } 203 204 static irqreturn_t ccache_int_handler(int irq, void *device) 205 { 206 unsigned int add_h, add_l; 207 208 if (irq == g_irq[DIR_CORR]) { 209 add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_HIGH); 210 add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_LOW); 211 pr_err("DirError @ 0x%08X.%08X\n", add_h, add_l); 212 /* Reading this register clears the DirError interrupt sig */ 213 readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_COUNT); 214 atomic_notifier_call_chain(&ccache_err_chain, 215 SIFIVE_CCACHE_ERR_TYPE_CE, 216 "DirECCFix"); 217 } 218 if (irq == g_irq[DIR_UNCORR]) { 219 add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_HIGH); 220 add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_LOW); 221 /* Reading this register clears the DirFail interrupt sig */ 222 readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_COUNT); 223 atomic_notifier_call_chain(&ccache_err_chain, 224 SIFIVE_CCACHE_ERR_TYPE_UE, 225 "DirECCFail"); 226 panic("CCACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l); 227 } 228 if (irq == g_irq[DATA_CORR]) { 229 add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_HIGH); 230 add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_LOW); 231 pr_err("DataError @ 0x%08X.%08X\n", add_h, add_l); 232 /* Reading this register clears the DataError interrupt sig */ 233 readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_COUNT); 234 atomic_notifier_call_chain(&ccache_err_chain, 235 SIFIVE_CCACHE_ERR_TYPE_CE, 236 "DatECCFix"); 237 } 238 if (irq == g_irq[DATA_UNCORR]) { 239 add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_HIGH); 240 add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_LOW); 241 pr_err("DataFail @ 0x%08X.%08X\n", add_h, add_l); 242 /* Reading this register clears the DataFail interrupt sig */ 243 readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_COUNT); 244 atomic_notifier_call_chain(&ccache_err_chain, 245 SIFIVE_CCACHE_ERR_TYPE_UE, 246 "DatECCFail"); 247 } 248 249 return IRQ_HANDLED; 250 } 251 252 static int sifive_ccache_probe(struct platform_device *pdev) 253 { 254 struct device *dev = &pdev->dev; 255 unsigned long quirks; 256 int intr_num, rc; 257 258 quirks = (unsigned long)device_get_match_data(dev); 259 260 intr_num = platform_irq_count(pdev); 261 if (!intr_num) 262 return dev_err_probe(dev, -ENODEV, "No interrupts property\n"); 263 264 for (int i = 0; i < intr_num; i++) { 265 if (i == DATA_UNCORR && (quirks & QUIRK_BROKEN_DATA_UNCORR)) 266 continue; 267 268 g_irq[i] = platform_get_irq(pdev, i); 269 if (g_irq[i] < 0) 270 return g_irq[i]; 271 272 rc = devm_request_irq(dev, g_irq[i], ccache_int_handler, 0, "ccache_ecc", NULL); 273 if (rc) 274 return dev_err_probe(dev, rc, "Could not request IRQ %d\n", g_irq[i]); 275 } 276 277 return 0; 278 } 279 280 static struct platform_driver sifive_ccache_driver = { 281 .probe = sifive_ccache_probe, 282 .driver = { 283 .name = "sifive_ccache", 284 .of_match_table = sifive_ccache_ids, 285 }, 286 }; 287 288 static int __init sifive_ccache_init(void) 289 { 290 struct device_node *np; 291 struct resource res; 292 const struct of_device_id *match; 293 unsigned long quirks; 294 int rc; 295 296 np = of_find_matching_node_and_match(NULL, sifive_ccache_ids, &match); 297 if (!np) 298 return -ENODEV; 299 300 quirks = (uintptr_t)match->data; 301 302 if (of_address_to_resource(np, 0, &res)) { 303 rc = -ENODEV; 304 goto err_node_put; 305 } 306 307 ccache_base = ioremap(res.start, resource_size(&res)); 308 if (!ccache_base) { 309 rc = -ENOMEM; 310 goto err_node_put; 311 } 312 313 if (of_property_read_u32(np, "cache-level", &level)) { 314 rc = -ENOENT; 315 goto err_unmap; 316 } 317 318 #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS 319 if (quirks & QUIRK_NONSTANDARD_CACHE_OPS) { 320 riscv_cbom_block_size = SIFIVE_CCACHE_LINE_SIZE; 321 riscv_noncoherent_supported(); 322 riscv_noncoherent_register_cache_ops(&ccache_mgmt_ops); 323 } 324 #endif 325 326 ccache_config_read(); 327 328 ccache_cache_ops.get_priv_group = ccache_get_priv_group; 329 riscv_set_cacheinfo_ops(&ccache_cache_ops); 330 331 #ifdef CONFIG_DEBUG_FS 332 setup_sifive_debug(); 333 #endif 334 335 rc = platform_driver_register(&sifive_ccache_driver); 336 if (rc) 337 goto err_unmap; 338 339 of_node_put(np); 340 341 return 0; 342 343 err_unmap: 344 iounmap(ccache_base); 345 err_node_put: 346 of_node_put(np); 347 return rc; 348 } 349 350 arch_initcall(sifive_ccache_init); 351