1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell CN10K LLC-TAD perf driver 3 * 4 * Copyright (C) 2021 Marvell 5 */ 6 7 #define pr_fmt(fmt) "tad_pmu: " fmt 8 9 #include <linux/module.h> 10 #include <linux/of.h> 11 #include <linux/of_address.h> 12 #include <linux/of_device.h> 13 #include <linux/cpuhotplug.h> 14 #include <linux/perf_event.h> 15 #include <linux/platform_device.h> 16 17 #define TAD_PFC_OFFSET 0x800 18 #define TAD_PFC(counter) (TAD_PFC_OFFSET | (counter << 3)) 19 #define TAD_PRF_OFFSET 0x900 20 #define TAD_PRF(counter) (TAD_PRF_OFFSET | (counter << 3)) 21 #define TAD_PRF_CNTSEL_MASK 0xFF 22 #define TAD_MAX_COUNTERS 8 23 24 #define to_tad_pmu(p) (container_of(p, struct tad_pmu, pmu)) 25 26 struct tad_region { 27 void __iomem *base; 28 }; 29 30 struct tad_pmu { 31 struct pmu pmu; 32 struct tad_region *regions; 33 u32 region_cnt; 34 unsigned int cpu; 35 struct hlist_node node; 36 struct perf_event *events[TAD_MAX_COUNTERS]; 37 DECLARE_BITMAP(counters_map, TAD_MAX_COUNTERS); 38 }; 39 40 static int tad_pmu_cpuhp_state; 41 42 static void tad_pmu_event_counter_read(struct perf_event *event) 43 { 44 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 45 struct hw_perf_event *hwc = &event->hw; 46 u32 counter_idx = hwc->idx; 47 u64 prev, new; 48 int i; 49 50 do { 51 prev = local64_read(&hwc->prev_count); 52 for (i = 0, new = 0; i < tad_pmu->region_cnt; i++) 53 new += readq(tad_pmu->regions[i].base + 54 TAD_PFC(counter_idx)); 55 } while (local64_cmpxchg(&hwc->prev_count, prev, new) != prev); 56 57 local64_add(new - prev, &event->count); 58 } 59 60 static void tad_pmu_event_counter_stop(struct perf_event *event, int flags) 61 { 62 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 63 struct hw_perf_event *hwc = &event->hw; 64 u32 counter_idx = hwc->idx; 65 int i; 66 67 /* TAD()_PFC() stop counting on the write 68 * which sets TAD()_PRF()[CNTSEL] == 0 69 */ 70 for (i = 0; i < tad_pmu->region_cnt; i++) { 71 writeq_relaxed(0, tad_pmu->regions[i].base + 72 TAD_PRF(counter_idx)); 73 } 74 75 tad_pmu_event_counter_read(event); 76 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 77 } 78 79 static void tad_pmu_event_counter_start(struct perf_event *event, int flags) 80 { 81 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 82 struct hw_perf_event *hwc = &event->hw; 83 u32 event_idx = event->attr.config; 84 u32 counter_idx = hwc->idx; 85 u64 reg_val; 86 int i; 87 88 hwc->state = 0; 89 90 /* Typically TAD_PFC() are zeroed to start counting */ 91 for (i = 0; i < tad_pmu->region_cnt; i++) 92 writeq_relaxed(0, tad_pmu->regions[i].base + 93 TAD_PFC(counter_idx)); 94 95 /* TAD()_PFC() start counting on the write 96 * which sets TAD()_PRF()[CNTSEL] != 0 97 */ 98 for (i = 0; i < tad_pmu->region_cnt; i++) { 99 reg_val = event_idx & 0xFF; 100 writeq_relaxed(reg_val, tad_pmu->regions[i].base + 101 TAD_PRF(counter_idx)); 102 } 103 } 104 105 static void tad_pmu_event_counter_del(struct perf_event *event, int flags) 106 { 107 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 108 struct hw_perf_event *hwc = &event->hw; 109 int idx = hwc->idx; 110 111 tad_pmu_event_counter_stop(event, flags | PERF_EF_UPDATE); 112 tad_pmu->events[idx] = NULL; 113 clear_bit(idx, tad_pmu->counters_map); 114 } 115 116 static int tad_pmu_event_counter_add(struct perf_event *event, int flags) 117 { 118 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 119 struct hw_perf_event *hwc = &event->hw; 120 int idx; 121 122 /* Get a free counter for this event */ 123 idx = find_first_zero_bit(tad_pmu->counters_map, TAD_MAX_COUNTERS); 124 if (idx == TAD_MAX_COUNTERS) 125 return -EAGAIN; 126 127 set_bit(idx, tad_pmu->counters_map); 128 129 hwc->idx = idx; 130 hwc->state = PERF_HES_STOPPED; 131 tad_pmu->events[idx] = event; 132 133 if (flags & PERF_EF_START) 134 tad_pmu_event_counter_start(event, flags); 135 136 return 0; 137 } 138 139 static int tad_pmu_event_init(struct perf_event *event) 140 { 141 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 142 143 if (event->attr.type != event->pmu->type) 144 return -ENOENT; 145 146 if (!event->attr.disabled) 147 return -EINVAL; 148 149 if (event->state != PERF_EVENT_STATE_OFF) 150 return -EINVAL; 151 152 event->cpu = tad_pmu->cpu; 153 event->hw.idx = -1; 154 event->hw.config_base = event->attr.config; 155 156 return 0; 157 } 158 159 static ssize_t tad_pmu_event_show(struct device *dev, 160 struct device_attribute *attr, char *page) 161 { 162 struct perf_pmu_events_attr *pmu_attr; 163 164 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); 165 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); 166 } 167 168 #define TAD_PMU_EVENT_ATTR(name, config) \ 169 PMU_EVENT_ATTR_ID(name, tad_pmu_event_show, config) 170 171 static struct attribute *tad_pmu_event_attrs[] = { 172 TAD_PMU_EVENT_ATTR(tad_none, 0x0), 173 TAD_PMU_EVENT_ATTR(tad_req_msh_in_any, 0x1), 174 TAD_PMU_EVENT_ATTR(tad_req_msh_in_mn, 0x2), 175 TAD_PMU_EVENT_ATTR(tad_req_msh_in_exlmn, 0x3), 176 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_any, 0x4), 177 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_mn, 0x5), 178 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_exlmn, 0x6), 179 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_dss, 0x7), 180 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_retry_dss, 0x8), 181 TAD_PMU_EVENT_ATTR(tad_dat_msh_in_any, 0x9), 182 TAD_PMU_EVENT_ATTR(tad_dat_msh_in_dss, 0xa), 183 TAD_PMU_EVENT_ATTR(tad_req_msh_out_any, 0xb), 184 TAD_PMU_EVENT_ATTR(tad_req_msh_out_dss_rd, 0xc), 185 TAD_PMU_EVENT_ATTR(tad_req_msh_out_dss_wr, 0xd), 186 TAD_PMU_EVENT_ATTR(tad_req_msh_out_evict, 0xe), 187 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_any, 0xf), 188 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_retry_exlmn, 0x10), 189 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_retry_mn, 0x11), 190 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_exlmn, 0x12), 191 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_mn, 0x13), 192 TAD_PMU_EVENT_ATTR(tad_snp_msh_out_any, 0x14), 193 TAD_PMU_EVENT_ATTR(tad_snp_msh_out_mn, 0x15), 194 TAD_PMU_EVENT_ATTR(tad_snp_msh_out_exlmn, 0x16), 195 TAD_PMU_EVENT_ATTR(tad_dat_msh_out_any, 0x17), 196 TAD_PMU_EVENT_ATTR(tad_dat_msh_out_fill, 0x18), 197 TAD_PMU_EVENT_ATTR(tad_dat_msh_out_dss, 0x19), 198 TAD_PMU_EVENT_ATTR(tad_alloc_dtg, 0x1a), 199 TAD_PMU_EVENT_ATTR(tad_alloc_ltg, 0x1b), 200 TAD_PMU_EVENT_ATTR(tad_alloc_any, 0x1c), 201 TAD_PMU_EVENT_ATTR(tad_hit_dtg, 0x1d), 202 TAD_PMU_EVENT_ATTR(tad_hit_ltg, 0x1e), 203 TAD_PMU_EVENT_ATTR(tad_hit_any, 0x1f), 204 TAD_PMU_EVENT_ATTR(tad_tag_rd, 0x20), 205 TAD_PMU_EVENT_ATTR(tad_dat_rd, 0x21), 206 TAD_PMU_EVENT_ATTR(tad_dat_rd_byp, 0x22), 207 TAD_PMU_EVENT_ATTR(tad_ifb_occ, 0x23), 208 TAD_PMU_EVENT_ATTR(tad_req_occ, 0x24), 209 NULL 210 }; 211 212 static const struct attribute_group tad_pmu_events_attr_group = { 213 .name = "events", 214 .attrs = tad_pmu_event_attrs, 215 }; 216 217 PMU_FORMAT_ATTR(event, "config:0-7"); 218 219 static struct attribute *tad_pmu_format_attrs[] = { 220 &format_attr_event.attr, 221 NULL 222 }; 223 224 static struct attribute_group tad_pmu_format_attr_group = { 225 .name = "format", 226 .attrs = tad_pmu_format_attrs, 227 }; 228 229 static ssize_t tad_pmu_cpumask_show(struct device *dev, 230 struct device_attribute *attr, char *buf) 231 { 232 struct tad_pmu *tad_pmu = to_tad_pmu(dev_get_drvdata(dev)); 233 234 return cpumap_print_to_pagebuf(true, buf, cpumask_of(tad_pmu->cpu)); 235 } 236 237 static DEVICE_ATTR(cpumask, 0444, tad_pmu_cpumask_show, NULL); 238 239 static struct attribute *tad_pmu_cpumask_attrs[] = { 240 &dev_attr_cpumask.attr, 241 NULL 242 }; 243 244 static struct attribute_group tad_pmu_cpumask_attr_group = { 245 .attrs = tad_pmu_cpumask_attrs, 246 }; 247 248 static const struct attribute_group *tad_pmu_attr_groups[] = { 249 &tad_pmu_events_attr_group, 250 &tad_pmu_format_attr_group, 251 &tad_pmu_cpumask_attr_group, 252 NULL 253 }; 254 255 static int tad_pmu_probe(struct platform_device *pdev) 256 { 257 struct device_node *node = pdev->dev.of_node; 258 struct tad_region *regions; 259 struct tad_pmu *tad_pmu; 260 struct resource *res; 261 u32 tad_pmu_page_size; 262 u32 tad_page_size; 263 u32 tad_cnt; 264 int i, ret; 265 char *name; 266 267 tad_pmu = devm_kzalloc(&pdev->dev, sizeof(*tad_pmu), GFP_KERNEL); 268 if (!tad_pmu) 269 return -ENOMEM; 270 271 platform_set_drvdata(pdev, tad_pmu); 272 273 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 274 if (!res) { 275 dev_err(&pdev->dev, "Mem resource not found\n"); 276 return -ENODEV; 277 } 278 279 ret = of_property_read_u32(node, "marvell,tad-page-size", 280 &tad_page_size); 281 if (ret) { 282 dev_err(&pdev->dev, "Can't find tad-page-size property\n"); 283 return ret; 284 } 285 286 ret = of_property_read_u32(node, "marvell,tad-pmu-page-size", 287 &tad_pmu_page_size); 288 if (ret) { 289 dev_err(&pdev->dev, "Can't find tad-pmu-page-size property\n"); 290 return ret; 291 } 292 293 ret = of_property_read_u32(node, "marvell,tad-cnt", &tad_cnt); 294 if (ret) { 295 dev_err(&pdev->dev, "Can't find tad-cnt property\n"); 296 return ret; 297 } 298 299 regions = devm_kcalloc(&pdev->dev, tad_cnt, 300 sizeof(*regions), GFP_KERNEL); 301 if (!regions) 302 return -ENOMEM; 303 304 /* ioremap the distributed TAD pmu regions */ 305 for (i = 0; i < tad_cnt && res->start < res->end; i++) { 306 regions[i].base = devm_ioremap(&pdev->dev, 307 res->start, 308 tad_pmu_page_size); 309 if (!regions[i].base) { 310 dev_err(&pdev->dev, "TAD%d ioremap fail\n", i); 311 return -ENOMEM; 312 } 313 res->start += tad_page_size; 314 } 315 316 tad_pmu->regions = regions; 317 tad_pmu->region_cnt = tad_cnt; 318 319 tad_pmu->pmu = (struct pmu) { 320 321 .module = THIS_MODULE, 322 .attr_groups = tad_pmu_attr_groups, 323 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | 324 PERF_PMU_CAP_NO_INTERRUPT, 325 .task_ctx_nr = perf_invalid_context, 326 327 .event_init = tad_pmu_event_init, 328 .add = tad_pmu_event_counter_add, 329 .del = tad_pmu_event_counter_del, 330 .start = tad_pmu_event_counter_start, 331 .stop = tad_pmu_event_counter_stop, 332 .read = tad_pmu_event_counter_read, 333 }; 334 335 tad_pmu->cpu = raw_smp_processor_id(); 336 337 /* Register pmu instance for cpu hotplug */ 338 ret = cpuhp_state_add_instance_nocalls(tad_pmu_cpuhp_state, 339 &tad_pmu->node); 340 if (ret) { 341 dev_err(&pdev->dev, "Error %d registering hotplug\n", ret); 342 return ret; 343 } 344 345 name = "tad"; 346 ret = perf_pmu_register(&tad_pmu->pmu, name, -1); 347 if (ret) 348 cpuhp_state_remove_instance_nocalls(tad_pmu_cpuhp_state, 349 &tad_pmu->node); 350 351 return ret; 352 } 353 354 static int tad_pmu_remove(struct platform_device *pdev) 355 { 356 struct tad_pmu *pmu = platform_get_drvdata(pdev); 357 358 cpuhp_state_remove_instance_nocalls(tad_pmu_cpuhp_state, 359 &pmu->node); 360 perf_pmu_unregister(&pmu->pmu); 361 362 return 0; 363 } 364 365 #ifdef CONFIG_OF 366 static const struct of_device_id tad_pmu_of_match[] = { 367 { .compatible = "marvell,cn10k-tad-pmu", }, 368 {}, 369 }; 370 #endif 371 372 static struct platform_driver tad_pmu_driver = { 373 .driver = { 374 .name = "cn10k_tad_pmu", 375 .of_match_table = of_match_ptr(tad_pmu_of_match), 376 .suppress_bind_attrs = true, 377 }, 378 .probe = tad_pmu_probe, 379 .remove = tad_pmu_remove, 380 }; 381 382 static int tad_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 383 { 384 struct tad_pmu *pmu = hlist_entry_safe(node, struct tad_pmu, node); 385 unsigned int target; 386 387 if (cpu != pmu->cpu) 388 return 0; 389 390 target = cpumask_any_but(cpu_online_mask, cpu); 391 if (target >= nr_cpu_ids) 392 return 0; 393 394 perf_pmu_migrate_context(&pmu->pmu, cpu, target); 395 pmu->cpu = target; 396 397 return 0; 398 } 399 400 static int __init tad_pmu_init(void) 401 { 402 int ret; 403 404 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 405 "perf/cn10k/tadpmu:online", 406 NULL, 407 tad_pmu_offline_cpu); 408 if (ret < 0) 409 return ret; 410 tad_pmu_cpuhp_state = ret; 411 return platform_driver_register(&tad_pmu_driver); 412 } 413 414 static void __exit tad_pmu_exit(void) 415 { 416 platform_driver_unregister(&tad_pmu_driver); 417 cpuhp_remove_multi_state(tad_pmu_cpuhp_state); 418 } 419 420 module_init(tad_pmu_init); 421 module_exit(tad_pmu_exit); 422 423 MODULE_DESCRIPTION("Marvell CN10K LLC-TAD Perf driver"); 424 MODULE_AUTHOR("Bhaskara Budiredla <bbudiredla@marvell.com>"); 425 MODULE_LICENSE("GPL v2"); 426