1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Zhaoxin PMU; like Intel Architectural PerfMon-v2
4 */
5
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8 #include <linux/stddef.h>
9 #include <linux/types.h>
10 #include <linux/init.h>
11 #include <linux/slab.h>
12 #include <linux/export.h>
13 #include <linux/nmi.h>
14
15 #include <asm/cpufeature.h>
16 #include <asm/hardirq.h>
17 #include <asm/apic.h>
18
19 #include "../perf_event.h"
20
21 /*
22 * Zhaoxin PerfMon, used on zxc and later.
23 */
24 static u64 zx_pmon_event_map[PERF_COUNT_HW_MAX] __read_mostly = {
25
26 [PERF_COUNT_HW_CPU_CYCLES] = 0x0082,
27 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
28 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0515,
29 [PERF_COUNT_HW_CACHE_MISSES] = 0x051a,
30 [PERF_COUNT_HW_BUS_CYCLES] = 0x0083,
31 };
32
33 static struct event_constraint zxc_event_constraints[] __read_mostly = {
34
35 FIXED_EVENT_CONSTRAINT(0x0082, 1), /* unhalted core clock cycles */
36 EVENT_CONSTRAINT_END
37 };
38
39 static struct event_constraint zxd_event_constraints[] __read_mostly = {
40
41 FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* retired instructions */
42 FIXED_EVENT_CONSTRAINT(0x0082, 1), /* unhalted core clock cycles */
43 FIXED_EVENT_CONSTRAINT(0x0083, 2), /* unhalted bus clock cycles */
44 EVENT_CONSTRAINT_END
45 };
46
47 static __initconst const u64 zxd_hw_cache_event_ids
48 [PERF_COUNT_HW_CACHE_MAX]
49 [PERF_COUNT_HW_CACHE_OP_MAX]
50 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
51 [C(L1D)] = {
52 [C(OP_READ)] = {
53 [C(RESULT_ACCESS)] = 0x0042,
54 [C(RESULT_MISS)] = 0x0538,
55 },
56 [C(OP_WRITE)] = {
57 [C(RESULT_ACCESS)] = 0x0043,
58 [C(RESULT_MISS)] = 0x0562,
59 },
60 [C(OP_PREFETCH)] = {
61 [C(RESULT_ACCESS)] = -1,
62 [C(RESULT_MISS)] = -1,
63 },
64 },
65 [C(L1I)] = {
66 [C(OP_READ)] = {
67 [C(RESULT_ACCESS)] = 0x0300,
68 [C(RESULT_MISS)] = 0x0301,
69 },
70 [C(OP_WRITE)] = {
71 [C(RESULT_ACCESS)] = -1,
72 [C(RESULT_MISS)] = -1,
73 },
74 [C(OP_PREFETCH)] = {
75 [C(RESULT_ACCESS)] = 0x030a,
76 [C(RESULT_MISS)] = 0x030b,
77 },
78 },
79 [C(LL)] = {
80 [C(OP_READ)] = {
81 [C(RESULT_ACCESS)] = -1,
82 [C(RESULT_MISS)] = -1,
83 },
84 [C(OP_WRITE)] = {
85 [C(RESULT_ACCESS)] = -1,
86 [C(RESULT_MISS)] = -1,
87 },
88 [C(OP_PREFETCH)] = {
89 [C(RESULT_ACCESS)] = -1,
90 [C(RESULT_MISS)] = -1,
91 },
92 },
93 [C(DTLB)] = {
94 [C(OP_READ)] = {
95 [C(RESULT_ACCESS)] = 0x0042,
96 [C(RESULT_MISS)] = 0x052c,
97 },
98 [C(OP_WRITE)] = {
99 [C(RESULT_ACCESS)] = 0x0043,
100 [C(RESULT_MISS)] = 0x0530,
101 },
102 [C(OP_PREFETCH)] = {
103 [C(RESULT_ACCESS)] = 0x0564,
104 [C(RESULT_MISS)] = 0x0565,
105 },
106 },
107 [C(ITLB)] = {
108 [C(OP_READ)] = {
109 [C(RESULT_ACCESS)] = 0x00c0,
110 [C(RESULT_MISS)] = 0x0534,
111 },
112 [C(OP_WRITE)] = {
113 [C(RESULT_ACCESS)] = -1,
114 [C(RESULT_MISS)] = -1,
115 },
116 [C(OP_PREFETCH)] = {
117 [C(RESULT_ACCESS)] = -1,
118 [C(RESULT_MISS)] = -1,
119 },
120 },
121 [C(BPU)] = {
122 [C(OP_READ)] = {
123 [C(RESULT_ACCESS)] = 0x0700,
124 [C(RESULT_MISS)] = 0x0709,
125 },
126 [C(OP_WRITE)] = {
127 [C(RESULT_ACCESS)] = -1,
128 [C(RESULT_MISS)] = -1,
129 },
130 [C(OP_PREFETCH)] = {
131 [C(RESULT_ACCESS)] = -1,
132 [C(RESULT_MISS)] = -1,
133 },
134 },
135 [C(NODE)] = {
136 [C(OP_READ)] = {
137 [C(RESULT_ACCESS)] = -1,
138 [C(RESULT_MISS)] = -1,
139 },
140 [C(OP_WRITE)] = {
141 [C(RESULT_ACCESS)] = -1,
142 [C(RESULT_MISS)] = -1,
143 },
144 [C(OP_PREFETCH)] = {
145 [C(RESULT_ACCESS)] = -1,
146 [C(RESULT_MISS)] = -1,
147 },
148 },
149 };
150
151 static __initconst const u64 zxe_hw_cache_event_ids
152 [PERF_COUNT_HW_CACHE_MAX]
153 [PERF_COUNT_HW_CACHE_OP_MAX]
154 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
155 [C(L1D)] = {
156 [C(OP_READ)] = {
157 [C(RESULT_ACCESS)] = 0x0568,
158 [C(RESULT_MISS)] = 0x054b,
159 },
160 [C(OP_WRITE)] = {
161 [C(RESULT_ACCESS)] = 0x0669,
162 [C(RESULT_MISS)] = 0x0562,
163 },
164 [C(OP_PREFETCH)] = {
165 [C(RESULT_ACCESS)] = -1,
166 [C(RESULT_MISS)] = -1,
167 },
168 },
169 [C(L1I)] = {
170 [C(OP_READ)] = {
171 [C(RESULT_ACCESS)] = 0x0300,
172 [C(RESULT_MISS)] = 0x0301,
173 },
174 [C(OP_WRITE)] = {
175 [C(RESULT_ACCESS)] = -1,
176 [C(RESULT_MISS)] = -1,
177 },
178 [C(OP_PREFETCH)] = {
179 [C(RESULT_ACCESS)] = 0x030a,
180 [C(RESULT_MISS)] = 0x030b,
181 },
182 },
183 [C(LL)] = {
184 [C(OP_READ)] = {
185 [C(RESULT_ACCESS)] = 0x0,
186 [C(RESULT_MISS)] = 0x0,
187 },
188 [C(OP_WRITE)] = {
189 [C(RESULT_ACCESS)] = 0x0,
190 [C(RESULT_MISS)] = 0x0,
191 },
192 [C(OP_PREFETCH)] = {
193 [C(RESULT_ACCESS)] = 0x0,
194 [C(RESULT_MISS)] = 0x0,
195 },
196 },
197 [C(DTLB)] = {
198 [C(OP_READ)] = {
199 [C(RESULT_ACCESS)] = 0x0568,
200 [C(RESULT_MISS)] = 0x052c,
201 },
202 [C(OP_WRITE)] = {
203 [C(RESULT_ACCESS)] = 0x0669,
204 [C(RESULT_MISS)] = 0x0530,
205 },
206 [C(OP_PREFETCH)] = {
207 [C(RESULT_ACCESS)] = 0x0564,
208 [C(RESULT_MISS)] = 0x0565,
209 },
210 },
211 [C(ITLB)] = {
212 [C(OP_READ)] = {
213 [C(RESULT_ACCESS)] = 0x00c0,
214 [C(RESULT_MISS)] = 0x0534,
215 },
216 [C(OP_WRITE)] = {
217 [C(RESULT_ACCESS)] = -1,
218 [C(RESULT_MISS)] = -1,
219 },
220 [C(OP_PREFETCH)] = {
221 [C(RESULT_ACCESS)] = -1,
222 [C(RESULT_MISS)] = -1,
223 },
224 },
225 [C(BPU)] = {
226 [C(OP_READ)] = {
227 [C(RESULT_ACCESS)] = 0x0028,
228 [C(RESULT_MISS)] = 0x0029,
229 },
230 [C(OP_WRITE)] = {
231 [C(RESULT_ACCESS)] = -1,
232 [C(RESULT_MISS)] = -1,
233 },
234 [C(OP_PREFETCH)] = {
235 [C(RESULT_ACCESS)] = -1,
236 [C(RESULT_MISS)] = -1,
237 },
238 },
239 [C(NODE)] = {
240 [C(OP_READ)] = {
241 [C(RESULT_ACCESS)] = -1,
242 [C(RESULT_MISS)] = -1,
243 },
244 [C(OP_WRITE)] = {
245 [C(RESULT_ACCESS)] = -1,
246 [C(RESULT_MISS)] = -1,
247 },
248 [C(OP_PREFETCH)] = {
249 [C(RESULT_ACCESS)] = -1,
250 [C(RESULT_MISS)] = -1,
251 },
252 },
253 };
254
zhaoxin_pmu_disable_all(void)255 static void zhaoxin_pmu_disable_all(void)
256 {
257 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
258 }
259
zhaoxin_pmu_enable_all(int added)260 static void zhaoxin_pmu_enable_all(int added)
261 {
262 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
263 }
264
zhaoxin_pmu_get_status(void)265 static inline u64 zhaoxin_pmu_get_status(void)
266 {
267 u64 status;
268
269 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
270
271 return status;
272 }
273
zhaoxin_pmu_ack_status(u64 ack)274 static inline void zhaoxin_pmu_ack_status(u64 ack)
275 {
276 wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
277 }
278
zxc_pmu_ack_status(u64 ack)279 static inline void zxc_pmu_ack_status(u64 ack)
280 {
281 /*
282 * ZXC needs global control enabled in order to clear status bits.
283 */
284 zhaoxin_pmu_enable_all(0);
285 zhaoxin_pmu_ack_status(ack);
286 zhaoxin_pmu_disable_all();
287 }
288
zhaoxin_pmu_disable_fixed(struct hw_perf_event * hwc)289 static void zhaoxin_pmu_disable_fixed(struct hw_perf_event *hwc)
290 {
291 int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
292 u64 ctrl_val, mask;
293
294 mask = 0xfULL << (idx * 4);
295
296 rdmsrl(hwc->config_base, ctrl_val);
297 ctrl_val &= ~mask;
298 wrmsrl(hwc->config_base, ctrl_val);
299 }
300
zhaoxin_pmu_disable_event(struct perf_event * event)301 static void zhaoxin_pmu_disable_event(struct perf_event *event)
302 {
303 struct hw_perf_event *hwc = &event->hw;
304
305 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
306 zhaoxin_pmu_disable_fixed(hwc);
307 return;
308 }
309
310 x86_pmu_disable_event(event);
311 }
312
zhaoxin_pmu_enable_fixed(struct hw_perf_event * hwc)313 static void zhaoxin_pmu_enable_fixed(struct hw_perf_event *hwc)
314 {
315 int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
316 u64 ctrl_val, bits, mask;
317
318 /*
319 * Enable IRQ generation (0x8),
320 * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
321 * if requested:
322 */
323 bits = 0x8ULL;
324 if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
325 bits |= 0x2;
326 if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
327 bits |= 0x1;
328
329 bits <<= (idx * 4);
330 mask = 0xfULL << (idx * 4);
331
332 rdmsrl(hwc->config_base, ctrl_val);
333 ctrl_val &= ~mask;
334 ctrl_val |= bits;
335 wrmsrl(hwc->config_base, ctrl_val);
336 }
337
zhaoxin_pmu_enable_event(struct perf_event * event)338 static void zhaoxin_pmu_enable_event(struct perf_event *event)
339 {
340 struct hw_perf_event *hwc = &event->hw;
341
342 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
343 zhaoxin_pmu_enable_fixed(hwc);
344 return;
345 }
346
347 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
348 }
349
350 /*
351 * This handler is triggered by the local APIC, so the APIC IRQ handling
352 * rules apply:
353 */
zhaoxin_pmu_handle_irq(struct pt_regs * regs)354 static int zhaoxin_pmu_handle_irq(struct pt_regs *regs)
355 {
356 struct perf_sample_data data;
357 struct cpu_hw_events *cpuc;
358 int handled = 0;
359 u64 status;
360 int bit;
361
362 cpuc = this_cpu_ptr(&cpu_hw_events);
363 apic_write(APIC_LVTPC, APIC_DM_NMI);
364 zhaoxin_pmu_disable_all();
365 status = zhaoxin_pmu_get_status();
366 if (!status)
367 goto done;
368
369 again:
370 if (x86_pmu.enabled_ack)
371 zxc_pmu_ack_status(status);
372 else
373 zhaoxin_pmu_ack_status(status);
374
375 inc_irq_stat(apic_perf_irqs);
376
377 /*
378 * CondChgd bit 63 doesn't mean any overflow status. Ignore
379 * and clear the bit.
380 */
381 if (__test_and_clear_bit(63, (unsigned long *)&status)) {
382 if (!status)
383 goto done;
384 }
385
386 for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
387 struct perf_event *event = cpuc->events[bit];
388
389 handled++;
390
391 if (!test_bit(bit, cpuc->active_mask))
392 continue;
393
394 x86_perf_event_update(event);
395 perf_sample_data_init(&data, 0, event->hw.last_period);
396
397 if (!x86_perf_event_set_period(event))
398 continue;
399
400 if (perf_event_overflow(event, &data, regs))
401 x86_pmu_stop(event, 0);
402 }
403
404 /*
405 * Repeat if there is more work to be done:
406 */
407 status = zhaoxin_pmu_get_status();
408 if (status)
409 goto again;
410
411 done:
412 zhaoxin_pmu_enable_all(0);
413 return handled;
414 }
415
zhaoxin_pmu_event_map(int hw_event)416 static u64 zhaoxin_pmu_event_map(int hw_event)
417 {
418 return zx_pmon_event_map[hw_event];
419 }
420
421 static struct event_constraint *
zhaoxin_get_event_constraints(struct cpu_hw_events * cpuc,int idx,struct perf_event * event)422 zhaoxin_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
423 struct perf_event *event)
424 {
425 struct event_constraint *c;
426
427 if (x86_pmu.event_constraints) {
428 for_each_event_constraint(c, x86_pmu.event_constraints) {
429 if ((event->hw.config & c->cmask) == c->code)
430 return c;
431 }
432 }
433
434 return &unconstrained;
435 }
436
437 PMU_FORMAT_ATTR(event, "config:0-7");
438 PMU_FORMAT_ATTR(umask, "config:8-15");
439 PMU_FORMAT_ATTR(edge, "config:18");
440 PMU_FORMAT_ATTR(inv, "config:23");
441 PMU_FORMAT_ATTR(cmask, "config:24-31");
442
443 static struct attribute *zx_arch_formats_attr[] = {
444 &format_attr_event.attr,
445 &format_attr_umask.attr,
446 &format_attr_edge.attr,
447 &format_attr_inv.attr,
448 &format_attr_cmask.attr,
449 NULL,
450 };
451
zhaoxin_event_sysfs_show(char * page,u64 config)452 static ssize_t zhaoxin_event_sysfs_show(char *page, u64 config)
453 {
454 u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT);
455
456 return x86_event_sysfs_show(page, config, event);
457 }
458
459 static const struct x86_pmu zhaoxin_pmu __initconst = {
460 .name = "zhaoxin",
461 .handle_irq = zhaoxin_pmu_handle_irq,
462 .disable_all = zhaoxin_pmu_disable_all,
463 .enable_all = zhaoxin_pmu_enable_all,
464 .enable = zhaoxin_pmu_enable_event,
465 .disable = zhaoxin_pmu_disable_event,
466 .hw_config = x86_pmu_hw_config,
467 .schedule_events = x86_schedule_events,
468 .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
469 .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
470 .event_map = zhaoxin_pmu_event_map,
471 .max_events = ARRAY_SIZE(zx_pmon_event_map),
472 .apic = 1,
473 /*
474 * For zxd/zxe, read/write operation for PMCx MSR is 48 bits.
475 */
476 .max_period = (1ULL << 47) - 1,
477 .get_event_constraints = zhaoxin_get_event_constraints,
478
479 .format_attrs = zx_arch_formats_attr,
480 .events_sysfs_show = zhaoxin_event_sysfs_show,
481 };
482
483 static const struct { int id; char *name; } zx_arch_events_map[] __initconst = {
484 { PERF_COUNT_HW_CPU_CYCLES, "cpu cycles" },
485 { PERF_COUNT_HW_INSTRUCTIONS, "instructions" },
486 { PERF_COUNT_HW_BUS_CYCLES, "bus cycles" },
487 { PERF_COUNT_HW_CACHE_REFERENCES, "cache references" },
488 { PERF_COUNT_HW_CACHE_MISSES, "cache misses" },
489 { PERF_COUNT_HW_BRANCH_INSTRUCTIONS, "branch instructions" },
490 { PERF_COUNT_HW_BRANCH_MISSES, "branch misses" },
491 };
492
zhaoxin_arch_events_quirk(void)493 static __init void zhaoxin_arch_events_quirk(void)
494 {
495 int bit;
496
497 /* disable event that reported as not present by cpuid */
498 for_each_set_bit(bit, x86_pmu.events_mask, ARRAY_SIZE(zx_arch_events_map)) {
499 zx_pmon_event_map[zx_arch_events_map[bit].id] = 0;
500 pr_warn("CPUID marked event: \'%s\' unavailable\n",
501 zx_arch_events_map[bit].name);
502 }
503 }
504
zhaoxin_pmu_init(void)505 __init int zhaoxin_pmu_init(void)
506 {
507 union cpuid10_edx edx;
508 union cpuid10_eax eax;
509 union cpuid10_ebx ebx;
510 struct event_constraint *c;
511 unsigned int unused;
512 int version;
513
514 pr_info("Welcome to zhaoxin pmu!\n");
515
516 /*
517 * Check whether the Architectural PerfMon supports
518 * hw_event or not.
519 */
520 cpuid(10, &eax.full, &ebx.full, &unused, &edx.full);
521
522 if (eax.split.mask_length < ARCH_PERFMON_EVENTS_COUNT - 1)
523 return -ENODEV;
524
525 version = eax.split.version_id;
526 if (version != 2)
527 return -ENODEV;
528
529 x86_pmu = zhaoxin_pmu;
530 pr_info("Version check pass!\n");
531
532 x86_pmu.version = version;
533 x86_pmu.num_counters = eax.split.num_counters;
534 x86_pmu.cntval_bits = eax.split.bit_width;
535 x86_pmu.cntval_mask = (1ULL << eax.split.bit_width) - 1;
536 x86_pmu.events_maskl = ebx.full;
537 x86_pmu.events_mask_len = eax.split.mask_length;
538
539 x86_pmu.num_counters_fixed = edx.split.num_counters_fixed;
540 x86_add_quirk(zhaoxin_arch_events_quirk);
541
542 switch (boot_cpu_data.x86) {
543 case 0x06:
544 if (boot_cpu_data.x86_model == 0x0f || boot_cpu_data.x86_model == 0x19) {
545
546 x86_pmu.max_period = x86_pmu.cntval_mask >> 1;
547
548 /* Clearing status works only if the global control is enable on zxc. */
549 x86_pmu.enabled_ack = 1;
550
551 x86_pmu.event_constraints = zxc_event_constraints;
552 zx_pmon_event_map[PERF_COUNT_HW_INSTRUCTIONS] = 0;
553 zx_pmon_event_map[PERF_COUNT_HW_CACHE_REFERENCES] = 0;
554 zx_pmon_event_map[PERF_COUNT_HW_CACHE_MISSES] = 0;
555 zx_pmon_event_map[PERF_COUNT_HW_BUS_CYCLES] = 0;
556
557 pr_cont("ZXC events, ");
558 break;
559 }
560 return -ENODEV;
561
562 case 0x07:
563 zx_pmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
564 X86_CONFIG(.event = 0x01, .umask = 0x01, .inv = 0x01, .cmask = 0x01);
565
566 zx_pmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
567 X86_CONFIG(.event = 0x0f, .umask = 0x04, .inv = 0, .cmask = 0);
568
569 switch (boot_cpu_data.x86_model) {
570 case 0x1b:
571 memcpy(hw_cache_event_ids, zxd_hw_cache_event_ids,
572 sizeof(hw_cache_event_ids));
573
574 x86_pmu.event_constraints = zxd_event_constraints;
575
576 zx_pmon_event_map[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x0700;
577 zx_pmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x0709;
578
579 pr_cont("ZXD events, ");
580 break;
581 case 0x3b:
582 memcpy(hw_cache_event_ids, zxe_hw_cache_event_ids,
583 sizeof(hw_cache_event_ids));
584
585 x86_pmu.event_constraints = zxd_event_constraints;
586
587 zx_pmon_event_map[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x0028;
588 zx_pmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x0029;
589
590 pr_cont("ZXE events, ");
591 break;
592 default:
593 return -ENODEV;
594 }
595 break;
596
597 default:
598 return -ENODEV;
599 }
600
601 x86_pmu.intel_ctrl = (1 << (x86_pmu.num_counters)) - 1;
602 x86_pmu.intel_ctrl |= ((1LL << x86_pmu.num_counters_fixed)-1) << INTEL_PMC_IDX_FIXED;
603
604 if (x86_pmu.event_constraints) {
605 for_each_event_constraint(c, x86_pmu.event_constraints) {
606 c->idxmsk64 |= (1ULL << x86_pmu.num_counters) - 1;
607 c->weight += x86_pmu.num_counters;
608 }
609 }
610
611 return 0;
612 }
613
614