17cf83e22SBharat Bhushan // SPDX-License-Identifier: GPL-2.0
27cf83e22SBharat Bhushan /* Marvell CN10K DRAM Subsystem (DSS) Performance Monitor Driver
37cf83e22SBharat Bhushan *
47cf83e22SBharat Bhushan * Copyright (C) 2021 Marvell.
57cf83e22SBharat Bhushan */
67cf83e22SBharat Bhushan
77cf83e22SBharat Bhushan #include <linux/init.h>
87cf83e22SBharat Bhushan #include <linux/io.h>
97cf83e22SBharat Bhushan #include <linux/module.h>
107cf83e22SBharat Bhushan #include <linux/of.h>
117cf83e22SBharat Bhushan #include <linux/perf_event.h>
1235a43326SBharat Bhushan #include <linux/hrtimer.h>
13e85930f0SGowthami Thiagarajan #include <linux/acpi.h>
14918dc87bSRob Herring #include <linux/platform_device.h>
157cf83e22SBharat Bhushan
167cf83e22SBharat Bhushan /* Performance Counters Operating Mode Control Registers */
177cf83e22SBharat Bhushan #define DDRC_PERF_CNT_OP_MODE_CTRL 0x8020
187cf83e22SBharat Bhushan #define OP_MODE_CTRL_VAL_MANNUAL 0x1
197cf83e22SBharat Bhushan
207cf83e22SBharat Bhushan /* Performance Counters Start Operation Control Registers */
217cf83e22SBharat Bhushan #define DDRC_PERF_CNT_START_OP_CTRL 0x8028
227cf83e22SBharat Bhushan #define START_OP_CTRL_VAL_START 0x1ULL
237cf83e22SBharat Bhushan #define START_OP_CTRL_VAL_ACTIVE 0x2
247cf83e22SBharat Bhushan
257cf83e22SBharat Bhushan /* Performance Counters End Operation Control Registers */
267cf83e22SBharat Bhushan #define DDRC_PERF_CNT_END_OP_CTRL 0x8030
277cf83e22SBharat Bhushan #define END_OP_CTRL_VAL_END 0x1ULL
287cf83e22SBharat Bhushan
297cf83e22SBharat Bhushan /* Performance Counters End Status Registers */
307cf83e22SBharat Bhushan #define DDRC_PERF_CNT_END_STATUS 0x8038
317cf83e22SBharat Bhushan #define END_STATUS_VAL_END_TIMER_MODE_END 0x1
327cf83e22SBharat Bhushan
337cf83e22SBharat Bhushan /* Performance Counters Configuration Registers */
347cf83e22SBharat Bhushan #define DDRC_PERF_CFG_BASE 0x8040
357cf83e22SBharat Bhushan
367cf83e22SBharat Bhushan /* 8 Generic event counter + 2 fixed event counters */
377cf83e22SBharat Bhushan #define DDRC_PERF_NUM_GEN_COUNTERS 8
387cf83e22SBharat Bhushan #define DDRC_PERF_NUM_FIX_COUNTERS 2
397cf83e22SBharat Bhushan #define DDRC_PERF_READ_COUNTER_IDX DDRC_PERF_NUM_GEN_COUNTERS
407cf83e22SBharat Bhushan #define DDRC_PERF_WRITE_COUNTER_IDX (DDRC_PERF_NUM_GEN_COUNTERS + 1)
417cf83e22SBharat Bhushan #define DDRC_PERF_NUM_COUNTERS (DDRC_PERF_NUM_GEN_COUNTERS + \
427cf83e22SBharat Bhushan DDRC_PERF_NUM_FIX_COUNTERS)
437cf83e22SBharat Bhushan
447cf83e22SBharat Bhushan /* Generic event counter registers */
457cf83e22SBharat Bhushan #define DDRC_PERF_CFG(n) (DDRC_PERF_CFG_BASE + 8 * (n))
467cf83e22SBharat Bhushan #define EVENT_ENABLE BIT_ULL(63)
477cf83e22SBharat Bhushan
487cf83e22SBharat Bhushan /* Two dedicated event counters for DDR reads and writes */
497cf83e22SBharat Bhushan #define EVENT_DDR_READS 101
507cf83e22SBharat Bhushan #define EVENT_DDR_WRITES 100
517cf83e22SBharat Bhushan
527cf83e22SBharat Bhushan /*
537cf83e22SBharat Bhushan * programmable events IDs in programmable event counters.
547cf83e22SBharat Bhushan * DO NOT change these event-id numbers, they are used to
557cf83e22SBharat Bhushan * program event bitmap in h/w.
567cf83e22SBharat Bhushan */
577cf83e22SBharat Bhushan #define EVENT_OP_IS_ZQLATCH 55
587cf83e22SBharat Bhushan #define EVENT_OP_IS_ZQSTART 54
597cf83e22SBharat Bhushan #define EVENT_OP_IS_TCR_MRR 53
607cf83e22SBharat Bhushan #define EVENT_OP_IS_DQSOSC_MRR 52
617cf83e22SBharat Bhushan #define EVENT_OP_IS_DQSOSC_MPC 51
627cf83e22SBharat Bhushan #define EVENT_VISIBLE_WIN_LIMIT_REACHED_WR 50
637cf83e22SBharat Bhushan #define EVENT_VISIBLE_WIN_LIMIT_REACHED_RD 49
647cf83e22SBharat Bhushan #define EVENT_BSM_STARVATION 48
657cf83e22SBharat Bhushan #define EVENT_BSM_ALLOC 47
667cf83e22SBharat Bhushan #define EVENT_LPR_REQ_WITH_NOCREDIT 46
677cf83e22SBharat Bhushan #define EVENT_HPR_REQ_WITH_NOCREDIT 45
687cf83e22SBharat Bhushan #define EVENT_OP_IS_ZQCS 44
697cf83e22SBharat Bhushan #define EVENT_OP_IS_ZQCL 43
707cf83e22SBharat Bhushan #define EVENT_OP_IS_LOAD_MODE 42
717cf83e22SBharat Bhushan #define EVENT_OP_IS_SPEC_REF 41
727cf83e22SBharat Bhushan #define EVENT_OP_IS_CRIT_REF 40
737cf83e22SBharat Bhushan #define EVENT_OP_IS_REFRESH 39
747cf83e22SBharat Bhushan #define EVENT_OP_IS_ENTER_MPSM 35
757cf83e22SBharat Bhushan #define EVENT_OP_IS_ENTER_POWERDOWN 31
767cf83e22SBharat Bhushan #define EVENT_OP_IS_ENTER_SELFREF 27
777cf83e22SBharat Bhushan #define EVENT_WAW_HAZARD 26
787cf83e22SBharat Bhushan #define EVENT_RAW_HAZARD 25
797cf83e22SBharat Bhushan #define EVENT_WAR_HAZARD 24
807cf83e22SBharat Bhushan #define EVENT_WRITE_COMBINE 23
817cf83e22SBharat Bhushan #define EVENT_RDWR_TRANSITIONS 22
827cf83e22SBharat Bhushan #define EVENT_PRECHARGE_FOR_OTHER 21
837cf83e22SBharat Bhushan #define EVENT_PRECHARGE_FOR_RDWR 20
847cf83e22SBharat Bhushan #define EVENT_OP_IS_PRECHARGE 19
857cf83e22SBharat Bhushan #define EVENT_OP_IS_MWR 18
867cf83e22SBharat Bhushan #define EVENT_OP_IS_WR 17
877cf83e22SBharat Bhushan #define EVENT_OP_IS_RD 16
887cf83e22SBharat Bhushan #define EVENT_OP_IS_RD_ACTIVATE 15
897cf83e22SBharat Bhushan #define EVENT_OP_IS_RD_OR_WR 14
907cf83e22SBharat Bhushan #define EVENT_OP_IS_ACTIVATE 13
917cf83e22SBharat Bhushan #define EVENT_WR_XACT_WHEN_CRITICAL 12
927cf83e22SBharat Bhushan #define EVENT_LPR_XACT_WHEN_CRITICAL 11
937cf83e22SBharat Bhushan #define EVENT_HPR_XACT_WHEN_CRITICAL 10
947cf83e22SBharat Bhushan #define EVENT_DFI_RD_DATA_CYCLES 9
957cf83e22SBharat Bhushan #define EVENT_DFI_WR_DATA_CYCLES 8
967cf83e22SBharat Bhushan #define EVENT_ACT_BYPASS 7
977cf83e22SBharat Bhushan #define EVENT_READ_BYPASS 6
987cf83e22SBharat Bhushan #define EVENT_HIF_HI_PRI_RD 5
997cf83e22SBharat Bhushan #define EVENT_HIF_RMW 4
1007cf83e22SBharat Bhushan #define EVENT_HIF_RD 3
1017cf83e22SBharat Bhushan #define EVENT_HIF_WR 2
1027cf83e22SBharat Bhushan #define EVENT_HIF_RD_OR_WR 1
1037cf83e22SBharat Bhushan
1047cf83e22SBharat Bhushan /* Event counter value registers */
1057cf83e22SBharat Bhushan #define DDRC_PERF_CNT_VALUE_BASE 0x8080
1067cf83e22SBharat Bhushan #define DDRC_PERF_CNT_VALUE(n) (DDRC_PERF_CNT_VALUE_BASE + 8 * (n))
1077cf83e22SBharat Bhushan
1087cf83e22SBharat Bhushan /* Fixed event counter enable/disable register */
1097cf83e22SBharat Bhushan #define DDRC_PERF_CNT_FREERUN_EN 0x80C0
1107cf83e22SBharat Bhushan #define DDRC_PERF_FREERUN_WRITE_EN 0x1
1117cf83e22SBharat Bhushan #define DDRC_PERF_FREERUN_READ_EN 0x2
1127cf83e22SBharat Bhushan
1137cf83e22SBharat Bhushan /* Fixed event counter control register */
1147cf83e22SBharat Bhushan #define DDRC_PERF_CNT_FREERUN_CTRL 0x80C8
1157cf83e22SBharat Bhushan #define DDRC_FREERUN_WRITE_CNT_CLR 0x1
1167cf83e22SBharat Bhushan #define DDRC_FREERUN_READ_CNT_CLR 0x2
1177cf83e22SBharat Bhushan
1187cf83e22SBharat Bhushan /* Fixed event counter value register */
1197cf83e22SBharat Bhushan #define DDRC_PERF_CNT_VALUE_WR_OP 0x80D0
1207cf83e22SBharat Bhushan #define DDRC_PERF_CNT_VALUE_RD_OP 0x80D8
1217cf83e22SBharat Bhushan #define DDRC_PERF_CNT_VALUE_OVERFLOW BIT_ULL(48)
1227cf83e22SBharat Bhushan #define DDRC_PERF_CNT_MAX_VALUE GENMASK_ULL(48, 0)
1237cf83e22SBharat Bhushan
1247cf83e22SBharat Bhushan struct cn10k_ddr_pmu {
1257cf83e22SBharat Bhushan struct pmu pmu;
1267cf83e22SBharat Bhushan void __iomem *base;
1277cf83e22SBharat Bhushan unsigned int cpu;
1287cf83e22SBharat Bhushan struct device *dev;
1297cf83e22SBharat Bhushan int active_events;
1307cf83e22SBharat Bhushan struct perf_event *events[DDRC_PERF_NUM_COUNTERS];
13135a43326SBharat Bhushan struct hrtimer hrtimer;
13268fa55f0SBharat Bhushan struct hlist_node node;
1337cf83e22SBharat Bhushan };
1347cf83e22SBharat Bhushan
1357cf83e22SBharat Bhushan #define to_cn10k_ddr_pmu(p) container_of(p, struct cn10k_ddr_pmu, pmu)
1367cf83e22SBharat Bhushan
cn10k_ddr_pmu_event_show(struct device * dev,struct device_attribute * attr,char * page)1377cf83e22SBharat Bhushan static ssize_t cn10k_ddr_pmu_event_show(struct device *dev,
1387cf83e22SBharat Bhushan struct device_attribute *attr,
1397cf83e22SBharat Bhushan char *page)
1407cf83e22SBharat Bhushan {
1417cf83e22SBharat Bhushan struct perf_pmu_events_attr *pmu_attr;
1427cf83e22SBharat Bhushan
1437cf83e22SBharat Bhushan pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
1447cf83e22SBharat Bhushan return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
1457cf83e22SBharat Bhushan
1467cf83e22SBharat Bhushan }
1477cf83e22SBharat Bhushan
1487cf83e22SBharat Bhushan #define CN10K_DDR_PMU_EVENT_ATTR(_name, _id) \
1497cf83e22SBharat Bhushan PMU_EVENT_ATTR_ID(_name, cn10k_ddr_pmu_event_show, _id)
1507cf83e22SBharat Bhushan
1517cf83e22SBharat Bhushan static struct attribute *cn10k_ddr_perf_events_attrs[] = {
1527cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR),
1537cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR),
1547cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD),
1557cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW),
1567cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD),
1577cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS),
1587cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS),
1597cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_wr_data_access, EVENT_DFI_WR_DATA_CYCLES),
1607cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_rd_data_access, EVENT_DFI_RD_DATA_CYCLES),
1617cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access,
1627cf83e22SBharat Bhushan EVENT_HPR_XACT_WHEN_CRITICAL),
1637cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access,
1647cf83e22SBharat Bhushan EVENT_LPR_XACT_WHEN_CRITICAL),
1657cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access,
1667cf83e22SBharat Bhushan EVENT_WR_XACT_WHEN_CRITICAL),
1677cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE),
1687cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access, EVENT_OP_IS_RD_OR_WR),
1697cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access, EVENT_OP_IS_RD_ACTIVATE),
1707cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD),
1717cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR),
1727cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR),
1737cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE),
1747cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr, EVENT_PRECHARGE_FOR_RDWR),
1757cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other,
1767cf83e22SBharat Bhushan EVENT_PRECHARGE_FOR_OTHER),
1777cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS),
1787cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE),
1797cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD),
1807cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD),
1817cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD),
1827cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF),
1837cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown, EVENT_OP_IS_ENTER_POWERDOWN),
1847cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_mpsm, EVENT_OP_IS_ENTER_MPSM),
1857cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH),
1867cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF),
1877cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF),
1887cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE),
1897cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_zqcl, EVENT_OP_IS_ZQCL),
1907cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_wr_access, EVENT_OP_IS_ZQCS),
1917cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_hpr_req_with_nocredit,
1927cf83e22SBharat Bhushan EVENT_HPR_REQ_WITH_NOCREDIT),
1937cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_lpr_req_with_nocredit,
1947cf83e22SBharat Bhushan EVENT_LPR_REQ_WITH_NOCREDIT),
1957cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_alloc, EVENT_BSM_ALLOC),
1967cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_starvation, EVENT_BSM_STARVATION),
1977cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd,
1987cf83e22SBharat Bhushan EVENT_VISIBLE_WIN_LIMIT_REACHED_RD),
1997cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr,
2007cf83e22SBharat Bhushan EVENT_VISIBLE_WIN_LIMIT_REACHED_WR),
2017cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_OP_IS_DQSOSC_MPC),
2027cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_OP_IS_DQSOSC_MRR),
2037cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_OP_IS_TCR_MRR),
2047cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_OP_IS_ZQSTART),
2057cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_OP_IS_ZQLATCH),
2067cf83e22SBharat Bhushan /* Free run event counters */
2077cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS),
2087cf83e22SBharat Bhushan CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES),
2097cf83e22SBharat Bhushan NULL
2107cf83e22SBharat Bhushan };
2117cf83e22SBharat Bhushan
2127cf83e22SBharat Bhushan static struct attribute_group cn10k_ddr_perf_events_attr_group = {
2137cf83e22SBharat Bhushan .name = "events",
2147cf83e22SBharat Bhushan .attrs = cn10k_ddr_perf_events_attrs,
2157cf83e22SBharat Bhushan };
2167cf83e22SBharat Bhushan
2177cf83e22SBharat Bhushan PMU_FORMAT_ATTR(event, "config:0-8");
2187cf83e22SBharat Bhushan
2197cf83e22SBharat Bhushan static struct attribute *cn10k_ddr_perf_format_attrs[] = {
2207cf83e22SBharat Bhushan &format_attr_event.attr,
2217cf83e22SBharat Bhushan NULL,
2227cf83e22SBharat Bhushan };
2237cf83e22SBharat Bhushan
2247cf83e22SBharat Bhushan static struct attribute_group cn10k_ddr_perf_format_attr_group = {
2257cf83e22SBharat Bhushan .name = "format",
2267cf83e22SBharat Bhushan .attrs = cn10k_ddr_perf_format_attrs,
2277cf83e22SBharat Bhushan };
2287cf83e22SBharat Bhushan
cn10k_ddr_perf_cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)2297cf83e22SBharat Bhushan static ssize_t cn10k_ddr_perf_cpumask_show(struct device *dev,
2307cf83e22SBharat Bhushan struct device_attribute *attr,
2317cf83e22SBharat Bhushan char *buf)
2327cf83e22SBharat Bhushan {
2337cf83e22SBharat Bhushan struct cn10k_ddr_pmu *pmu = dev_get_drvdata(dev);
2347cf83e22SBharat Bhushan
2357cf83e22SBharat Bhushan return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
2367cf83e22SBharat Bhushan }
2377cf83e22SBharat Bhushan
2387cf83e22SBharat Bhushan static struct device_attribute cn10k_ddr_perf_cpumask_attr =
2397cf83e22SBharat Bhushan __ATTR(cpumask, 0444, cn10k_ddr_perf_cpumask_show, NULL);
2407cf83e22SBharat Bhushan
2417cf83e22SBharat Bhushan static struct attribute *cn10k_ddr_perf_cpumask_attrs[] = {
2427cf83e22SBharat Bhushan &cn10k_ddr_perf_cpumask_attr.attr,
2437cf83e22SBharat Bhushan NULL,
2447cf83e22SBharat Bhushan };
2457cf83e22SBharat Bhushan
2467cf83e22SBharat Bhushan static struct attribute_group cn10k_ddr_perf_cpumask_attr_group = {
2477cf83e22SBharat Bhushan .attrs = cn10k_ddr_perf_cpumask_attrs,
2487cf83e22SBharat Bhushan };
2497cf83e22SBharat Bhushan
2507cf83e22SBharat Bhushan static const struct attribute_group *cn10k_attr_groups[] = {
2517cf83e22SBharat Bhushan &cn10k_ddr_perf_events_attr_group,
2527cf83e22SBharat Bhushan &cn10k_ddr_perf_format_attr_group,
2537cf83e22SBharat Bhushan &cn10k_ddr_perf_cpumask_attr_group,
2547cf83e22SBharat Bhushan NULL,
2557cf83e22SBharat Bhushan };
2567cf83e22SBharat Bhushan
25735a43326SBharat Bhushan /* Default poll timeout is 100 sec, which is very sufficient for
25835a43326SBharat Bhushan * 48 bit counter incremented max at 5.6 GT/s, which may take many
25935a43326SBharat Bhushan * hours to overflow.
26035a43326SBharat Bhushan */
26135a43326SBharat Bhushan static unsigned long cn10k_ddr_pmu_poll_period_sec = 100;
26235a43326SBharat Bhushan module_param_named(poll_period_sec, cn10k_ddr_pmu_poll_period_sec, ulong, 0644);
26335a43326SBharat Bhushan
cn10k_ddr_pmu_timer_period(void)26435a43326SBharat Bhushan static ktime_t cn10k_ddr_pmu_timer_period(void)
26535a43326SBharat Bhushan {
26635a43326SBharat Bhushan return ms_to_ktime((u64)cn10k_ddr_pmu_poll_period_sec * USEC_PER_SEC);
26735a43326SBharat Bhushan }
26835a43326SBharat Bhushan
ddr_perf_get_event_bitmap(int eventid,u64 * event_bitmap)2697cf83e22SBharat Bhushan static int ddr_perf_get_event_bitmap(int eventid, u64 *event_bitmap)
2707cf83e22SBharat Bhushan {
2717cf83e22SBharat Bhushan switch (eventid) {
2727cf83e22SBharat Bhushan case EVENT_HIF_RD_OR_WR ... EVENT_WAW_HAZARD:
2737cf83e22SBharat Bhushan case EVENT_OP_IS_REFRESH ... EVENT_OP_IS_ZQLATCH:
2747cf83e22SBharat Bhushan *event_bitmap = (1ULL << (eventid - 1));
2757cf83e22SBharat Bhushan break;
2767cf83e22SBharat Bhushan case EVENT_OP_IS_ENTER_SELFREF:
2777cf83e22SBharat Bhushan case EVENT_OP_IS_ENTER_POWERDOWN:
2787cf83e22SBharat Bhushan case EVENT_OP_IS_ENTER_MPSM:
2797cf83e22SBharat Bhushan *event_bitmap = (0xFULL << (eventid - 1));
2807cf83e22SBharat Bhushan break;
2817cf83e22SBharat Bhushan default:
2827cf83e22SBharat Bhushan pr_err("%s Invalid eventid %d\n", __func__, eventid);
2837cf83e22SBharat Bhushan return -EINVAL;
2847cf83e22SBharat Bhushan }
2857cf83e22SBharat Bhushan
2867cf83e22SBharat Bhushan return 0;
2877cf83e22SBharat Bhushan }
2887cf83e22SBharat Bhushan
cn10k_ddr_perf_alloc_counter(struct cn10k_ddr_pmu * pmu,struct perf_event * event)2897cf83e22SBharat Bhushan static int cn10k_ddr_perf_alloc_counter(struct cn10k_ddr_pmu *pmu,
2907cf83e22SBharat Bhushan struct perf_event *event)
2917cf83e22SBharat Bhushan {
2927cf83e22SBharat Bhushan u8 config = event->attr.config;
2937cf83e22SBharat Bhushan int i;
2947cf83e22SBharat Bhushan
2957cf83e22SBharat Bhushan /* DDR read free-run counter index */
2967cf83e22SBharat Bhushan if (config == EVENT_DDR_READS) {
2977cf83e22SBharat Bhushan pmu->events[DDRC_PERF_READ_COUNTER_IDX] = event;
2987cf83e22SBharat Bhushan return DDRC_PERF_READ_COUNTER_IDX;
2997cf83e22SBharat Bhushan }
3007cf83e22SBharat Bhushan
3017cf83e22SBharat Bhushan /* DDR write free-run counter index */
3027cf83e22SBharat Bhushan if (config == EVENT_DDR_WRITES) {
3037cf83e22SBharat Bhushan pmu->events[DDRC_PERF_WRITE_COUNTER_IDX] = event;
3047cf83e22SBharat Bhushan return DDRC_PERF_WRITE_COUNTER_IDX;
3057cf83e22SBharat Bhushan }
3067cf83e22SBharat Bhushan
3077cf83e22SBharat Bhushan /* Allocate DDR generic counters */
3087cf83e22SBharat Bhushan for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
3097cf83e22SBharat Bhushan if (pmu->events[i] == NULL) {
3107cf83e22SBharat Bhushan pmu->events[i] = event;
3117cf83e22SBharat Bhushan return i;
3127cf83e22SBharat Bhushan }
3137cf83e22SBharat Bhushan }
3147cf83e22SBharat Bhushan
3157cf83e22SBharat Bhushan return -ENOENT;
3167cf83e22SBharat Bhushan }
3177cf83e22SBharat Bhushan
cn10k_ddr_perf_free_counter(struct cn10k_ddr_pmu * pmu,int counter)3187cf83e22SBharat Bhushan static void cn10k_ddr_perf_free_counter(struct cn10k_ddr_pmu *pmu, int counter)
3197cf83e22SBharat Bhushan {
3207cf83e22SBharat Bhushan pmu->events[counter] = NULL;
3217cf83e22SBharat Bhushan }
3227cf83e22SBharat Bhushan
cn10k_ddr_perf_event_init(struct perf_event * event)3237cf83e22SBharat Bhushan static int cn10k_ddr_perf_event_init(struct perf_event *event)
3247cf83e22SBharat Bhushan {
3257cf83e22SBharat Bhushan struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
3267cf83e22SBharat Bhushan struct hw_perf_event *hwc = &event->hw;
3277cf83e22SBharat Bhushan
3287cf83e22SBharat Bhushan if (event->attr.type != event->pmu->type)
3297cf83e22SBharat Bhushan return -ENOENT;
3307cf83e22SBharat Bhushan
3317cf83e22SBharat Bhushan if (is_sampling_event(event)) {
3327cf83e22SBharat Bhushan dev_info(pmu->dev, "Sampling not supported!\n");
3337cf83e22SBharat Bhushan return -EOPNOTSUPP;
3347cf83e22SBharat Bhushan }
3357cf83e22SBharat Bhushan
3367cf83e22SBharat Bhushan if (event->cpu < 0) {
3377cf83e22SBharat Bhushan dev_warn(pmu->dev, "Can't provide per-task data!\n");
3387cf83e22SBharat Bhushan return -EOPNOTSUPP;
3397cf83e22SBharat Bhushan }
3407cf83e22SBharat Bhushan
3417cf83e22SBharat Bhushan /* We must NOT create groups containing mixed PMUs */
3427cf83e22SBharat Bhushan if (event->group_leader->pmu != event->pmu &&
3437cf83e22SBharat Bhushan !is_software_event(event->group_leader))
3447cf83e22SBharat Bhushan return -EINVAL;
3457cf83e22SBharat Bhushan
3467cf83e22SBharat Bhushan /* Set ownership of event to one CPU, same event can not be observed
3477cf83e22SBharat Bhushan * on multiple cpus at same time.
3487cf83e22SBharat Bhushan */
3497cf83e22SBharat Bhushan event->cpu = pmu->cpu;
3507cf83e22SBharat Bhushan hwc->idx = -1;
3517cf83e22SBharat Bhushan return 0;
3527cf83e22SBharat Bhushan }
3537cf83e22SBharat Bhushan
cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu * pmu,int counter,bool enable)3547cf83e22SBharat Bhushan static void cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu *pmu,
3557cf83e22SBharat Bhushan int counter, bool enable)
3567cf83e22SBharat Bhushan {
3577cf83e22SBharat Bhushan u32 reg;
3587cf83e22SBharat Bhushan u64 val;
3597cf83e22SBharat Bhushan
3607cf83e22SBharat Bhushan if (counter > DDRC_PERF_NUM_COUNTERS) {
3617cf83e22SBharat Bhushan pr_err("Error: unsupported counter %d\n", counter);
3627cf83e22SBharat Bhushan return;
3637cf83e22SBharat Bhushan }
3647cf83e22SBharat Bhushan
3657cf83e22SBharat Bhushan if (counter < DDRC_PERF_NUM_GEN_COUNTERS) {
3667cf83e22SBharat Bhushan reg = DDRC_PERF_CFG(counter);
3677cf83e22SBharat Bhushan val = readq_relaxed(pmu->base + reg);
3687cf83e22SBharat Bhushan
3697cf83e22SBharat Bhushan if (enable)
3707cf83e22SBharat Bhushan val |= EVENT_ENABLE;
3717cf83e22SBharat Bhushan else
3727cf83e22SBharat Bhushan val &= ~EVENT_ENABLE;
3737cf83e22SBharat Bhushan
3747cf83e22SBharat Bhushan writeq_relaxed(val, pmu->base + reg);
3757cf83e22SBharat Bhushan } else {
3767cf83e22SBharat Bhushan val = readq_relaxed(pmu->base + DDRC_PERF_CNT_FREERUN_EN);
3777cf83e22SBharat Bhushan if (enable) {
3787cf83e22SBharat Bhushan if (counter == DDRC_PERF_READ_COUNTER_IDX)
3797cf83e22SBharat Bhushan val |= DDRC_PERF_FREERUN_READ_EN;
3807cf83e22SBharat Bhushan else
3817cf83e22SBharat Bhushan val |= DDRC_PERF_FREERUN_WRITE_EN;
3827cf83e22SBharat Bhushan } else {
3837cf83e22SBharat Bhushan if (counter == DDRC_PERF_READ_COUNTER_IDX)
3847cf83e22SBharat Bhushan val &= ~DDRC_PERF_FREERUN_READ_EN;
3857cf83e22SBharat Bhushan else
3867cf83e22SBharat Bhushan val &= ~DDRC_PERF_FREERUN_WRITE_EN;
3877cf83e22SBharat Bhushan }
3887cf83e22SBharat Bhushan writeq_relaxed(val, pmu->base + DDRC_PERF_CNT_FREERUN_EN);
3897cf83e22SBharat Bhushan }
3907cf83e22SBharat Bhushan }
3917cf83e22SBharat Bhushan
cn10k_ddr_perf_read_counter(struct cn10k_ddr_pmu * pmu,int counter)3927cf83e22SBharat Bhushan static u64 cn10k_ddr_perf_read_counter(struct cn10k_ddr_pmu *pmu, int counter)
3937cf83e22SBharat Bhushan {
3947cf83e22SBharat Bhushan u64 val;
3957cf83e22SBharat Bhushan
3967cf83e22SBharat Bhushan if (counter == DDRC_PERF_READ_COUNTER_IDX)
3977cf83e22SBharat Bhushan return readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE_RD_OP);
3987cf83e22SBharat Bhushan
3997cf83e22SBharat Bhushan if (counter == DDRC_PERF_WRITE_COUNTER_IDX)
4007cf83e22SBharat Bhushan return readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE_WR_OP);
4017cf83e22SBharat Bhushan
4027cf83e22SBharat Bhushan val = readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE(counter));
4037cf83e22SBharat Bhushan return val;
4047cf83e22SBharat Bhushan }
4057cf83e22SBharat Bhushan
cn10k_ddr_perf_event_update(struct perf_event * event)4067cf83e22SBharat Bhushan static void cn10k_ddr_perf_event_update(struct perf_event *event)
4077cf83e22SBharat Bhushan {
4087cf83e22SBharat Bhushan struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
4097cf83e22SBharat Bhushan struct hw_perf_event *hwc = &event->hw;
4107cf83e22SBharat Bhushan u64 prev_count, new_count, mask;
4117cf83e22SBharat Bhushan
4127cf83e22SBharat Bhushan do {
4137cf83e22SBharat Bhushan prev_count = local64_read(&hwc->prev_count);
4147cf83e22SBharat Bhushan new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
4157cf83e22SBharat Bhushan } while (local64_xchg(&hwc->prev_count, new_count) != prev_count);
4167cf83e22SBharat Bhushan
4177cf83e22SBharat Bhushan mask = DDRC_PERF_CNT_MAX_VALUE;
4187cf83e22SBharat Bhushan
4197cf83e22SBharat Bhushan local64_add((new_count - prev_count) & mask, &event->count);
4207cf83e22SBharat Bhushan }
4217cf83e22SBharat Bhushan
cn10k_ddr_perf_event_start(struct perf_event * event,int flags)4227cf83e22SBharat Bhushan static void cn10k_ddr_perf_event_start(struct perf_event *event, int flags)
4237cf83e22SBharat Bhushan {
4247cf83e22SBharat Bhushan struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
4257cf83e22SBharat Bhushan struct hw_perf_event *hwc = &event->hw;
4267cf83e22SBharat Bhushan int counter = hwc->idx;
4277cf83e22SBharat Bhushan
4287cf83e22SBharat Bhushan local64_set(&hwc->prev_count, 0);
4297cf83e22SBharat Bhushan
4307cf83e22SBharat Bhushan cn10k_ddr_perf_counter_enable(pmu, counter, true);
4317cf83e22SBharat Bhushan
4327cf83e22SBharat Bhushan hwc->state = 0;
4337cf83e22SBharat Bhushan }
4347cf83e22SBharat Bhushan
cn10k_ddr_perf_event_add(struct perf_event * event,int flags)4357cf83e22SBharat Bhushan static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags)
4367cf83e22SBharat Bhushan {
4377cf83e22SBharat Bhushan struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
4387cf83e22SBharat Bhushan struct hw_perf_event *hwc = &event->hw;
4397cf83e22SBharat Bhushan u8 config = event->attr.config;
4407cf83e22SBharat Bhushan int counter, ret;
4417cf83e22SBharat Bhushan u32 reg_offset;
4427cf83e22SBharat Bhushan u64 val;
4437cf83e22SBharat Bhushan
4447cf83e22SBharat Bhushan counter = cn10k_ddr_perf_alloc_counter(pmu, event);
4457cf83e22SBharat Bhushan if (counter < 0)
4467cf83e22SBharat Bhushan return -EAGAIN;
4477cf83e22SBharat Bhushan
4487cf83e22SBharat Bhushan pmu->active_events++;
4497cf83e22SBharat Bhushan hwc->idx = counter;
4507cf83e22SBharat Bhushan
45135a43326SBharat Bhushan if (pmu->active_events == 1)
45235a43326SBharat Bhushan hrtimer_start(&pmu->hrtimer, cn10k_ddr_pmu_timer_period(),
45335a43326SBharat Bhushan HRTIMER_MODE_REL_PINNED);
45435a43326SBharat Bhushan
4557cf83e22SBharat Bhushan if (counter < DDRC_PERF_NUM_GEN_COUNTERS) {
4567cf83e22SBharat Bhushan /* Generic counters, configure event id */
4577cf83e22SBharat Bhushan reg_offset = DDRC_PERF_CFG(counter);
4587cf83e22SBharat Bhushan ret = ddr_perf_get_event_bitmap(config, &val);
4597cf83e22SBharat Bhushan if (ret)
4607cf83e22SBharat Bhushan return ret;
4617cf83e22SBharat Bhushan
4627cf83e22SBharat Bhushan writeq_relaxed(val, pmu->base + reg_offset);
4637cf83e22SBharat Bhushan } else {
4647cf83e22SBharat Bhushan /* fixed event counter, clear counter value */
4657cf83e22SBharat Bhushan if (counter == DDRC_PERF_READ_COUNTER_IDX)
4667cf83e22SBharat Bhushan val = DDRC_FREERUN_READ_CNT_CLR;
4677cf83e22SBharat Bhushan else
4687cf83e22SBharat Bhushan val = DDRC_FREERUN_WRITE_CNT_CLR;
4697cf83e22SBharat Bhushan
4707cf83e22SBharat Bhushan writeq_relaxed(val, pmu->base + DDRC_PERF_CNT_FREERUN_CTRL);
4717cf83e22SBharat Bhushan }
4727cf83e22SBharat Bhushan
4737cf83e22SBharat Bhushan hwc->state |= PERF_HES_STOPPED;
4747cf83e22SBharat Bhushan
4757cf83e22SBharat Bhushan if (flags & PERF_EF_START)
4767cf83e22SBharat Bhushan cn10k_ddr_perf_event_start(event, flags);
4777cf83e22SBharat Bhushan
4787cf83e22SBharat Bhushan return 0;
4797cf83e22SBharat Bhushan }
4807cf83e22SBharat Bhushan
cn10k_ddr_perf_event_stop(struct perf_event * event,int flags)4817cf83e22SBharat Bhushan static void cn10k_ddr_perf_event_stop(struct perf_event *event, int flags)
4827cf83e22SBharat Bhushan {
4837cf83e22SBharat Bhushan struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
4847cf83e22SBharat Bhushan struct hw_perf_event *hwc = &event->hw;
4857cf83e22SBharat Bhushan int counter = hwc->idx;
4867cf83e22SBharat Bhushan
4877cf83e22SBharat Bhushan cn10k_ddr_perf_counter_enable(pmu, counter, false);
4887cf83e22SBharat Bhushan
4897cf83e22SBharat Bhushan if (flags & PERF_EF_UPDATE)
4907cf83e22SBharat Bhushan cn10k_ddr_perf_event_update(event);
4917cf83e22SBharat Bhushan
4927cf83e22SBharat Bhushan hwc->state |= PERF_HES_STOPPED;
4937cf83e22SBharat Bhushan }
4947cf83e22SBharat Bhushan
cn10k_ddr_perf_event_del(struct perf_event * event,int flags)4957cf83e22SBharat Bhushan static void cn10k_ddr_perf_event_del(struct perf_event *event, int flags)
4967cf83e22SBharat Bhushan {
4977cf83e22SBharat Bhushan struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
4987cf83e22SBharat Bhushan struct hw_perf_event *hwc = &event->hw;
4997cf83e22SBharat Bhushan int counter = hwc->idx;
5007cf83e22SBharat Bhushan
5017cf83e22SBharat Bhushan cn10k_ddr_perf_event_stop(event, PERF_EF_UPDATE);
5027cf83e22SBharat Bhushan
5037cf83e22SBharat Bhushan cn10k_ddr_perf_free_counter(pmu, counter);
5047cf83e22SBharat Bhushan pmu->active_events--;
5057cf83e22SBharat Bhushan hwc->idx = -1;
50635a43326SBharat Bhushan
50735a43326SBharat Bhushan /* Cancel timer when no events to capture */
50835a43326SBharat Bhushan if (pmu->active_events == 0)
50935a43326SBharat Bhushan hrtimer_cancel(&pmu->hrtimer);
5107cf83e22SBharat Bhushan }
5117cf83e22SBharat Bhushan
cn10k_ddr_perf_pmu_enable(struct pmu * pmu)5127cf83e22SBharat Bhushan static void cn10k_ddr_perf_pmu_enable(struct pmu *pmu)
5137cf83e22SBharat Bhushan {
5147cf83e22SBharat Bhushan struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu);
5157cf83e22SBharat Bhushan
5167cf83e22SBharat Bhushan writeq_relaxed(START_OP_CTRL_VAL_START, ddr_pmu->base +
5177cf83e22SBharat Bhushan DDRC_PERF_CNT_START_OP_CTRL);
5187cf83e22SBharat Bhushan }
5197cf83e22SBharat Bhushan
cn10k_ddr_perf_pmu_disable(struct pmu * pmu)5207cf83e22SBharat Bhushan static void cn10k_ddr_perf_pmu_disable(struct pmu *pmu)
5217cf83e22SBharat Bhushan {
5227cf83e22SBharat Bhushan struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu);
5237cf83e22SBharat Bhushan
5247cf83e22SBharat Bhushan writeq_relaxed(END_OP_CTRL_VAL_END, ddr_pmu->base +
5257cf83e22SBharat Bhushan DDRC_PERF_CNT_END_OP_CTRL);
5267cf83e22SBharat Bhushan }
5277cf83e22SBharat Bhushan
cn10k_ddr_perf_event_update_all(struct cn10k_ddr_pmu * pmu)52835a43326SBharat Bhushan static void cn10k_ddr_perf_event_update_all(struct cn10k_ddr_pmu *pmu)
52935a43326SBharat Bhushan {
53035a43326SBharat Bhushan struct hw_perf_event *hwc;
53135a43326SBharat Bhushan int i;
53235a43326SBharat Bhushan
53335a43326SBharat Bhushan for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
53435a43326SBharat Bhushan if (pmu->events[i] == NULL)
53535a43326SBharat Bhushan continue;
53635a43326SBharat Bhushan
53735a43326SBharat Bhushan cn10k_ddr_perf_event_update(pmu->events[i]);
53835a43326SBharat Bhushan }
53935a43326SBharat Bhushan
54035a43326SBharat Bhushan /* Reset previous count as h/w counter are reset */
54135a43326SBharat Bhushan for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
54235a43326SBharat Bhushan if (pmu->events[i] == NULL)
54335a43326SBharat Bhushan continue;
54435a43326SBharat Bhushan
54535a43326SBharat Bhushan hwc = &pmu->events[i]->hw;
54635a43326SBharat Bhushan local64_set(&hwc->prev_count, 0);
54735a43326SBharat Bhushan }
54835a43326SBharat Bhushan }
54935a43326SBharat Bhushan
cn10k_ddr_pmu_overflow_handler(struct cn10k_ddr_pmu * pmu)55035a43326SBharat Bhushan static irqreturn_t cn10k_ddr_pmu_overflow_handler(struct cn10k_ddr_pmu *pmu)
55135a43326SBharat Bhushan {
55235a43326SBharat Bhushan struct perf_event *event;
55335a43326SBharat Bhushan struct hw_perf_event *hwc;
55435a43326SBharat Bhushan u64 prev_count, new_count;
55535a43326SBharat Bhushan u64 value;
55635a43326SBharat Bhushan int i;
55735a43326SBharat Bhushan
55835a43326SBharat Bhushan event = pmu->events[DDRC_PERF_READ_COUNTER_IDX];
55935a43326SBharat Bhushan if (event) {
56035a43326SBharat Bhushan hwc = &event->hw;
56135a43326SBharat Bhushan prev_count = local64_read(&hwc->prev_count);
56235a43326SBharat Bhushan new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
56335a43326SBharat Bhushan
56435a43326SBharat Bhushan /* Overflow condition is when new count less than
56535a43326SBharat Bhushan * previous count
56635a43326SBharat Bhushan */
56735a43326SBharat Bhushan if (new_count < prev_count)
56835a43326SBharat Bhushan cn10k_ddr_perf_event_update(event);
56935a43326SBharat Bhushan }
57035a43326SBharat Bhushan
57135a43326SBharat Bhushan event = pmu->events[DDRC_PERF_WRITE_COUNTER_IDX];
57235a43326SBharat Bhushan if (event) {
57335a43326SBharat Bhushan hwc = &event->hw;
57435a43326SBharat Bhushan prev_count = local64_read(&hwc->prev_count);
57535a43326SBharat Bhushan new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
57635a43326SBharat Bhushan
57735a43326SBharat Bhushan /* Overflow condition is when new count less than
57835a43326SBharat Bhushan * previous count
57935a43326SBharat Bhushan */
58035a43326SBharat Bhushan if (new_count < prev_count)
58135a43326SBharat Bhushan cn10k_ddr_perf_event_update(event);
58235a43326SBharat Bhushan }
58335a43326SBharat Bhushan
58435a43326SBharat Bhushan for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
58535a43326SBharat Bhushan if (pmu->events[i] == NULL)
58635a43326SBharat Bhushan continue;
58735a43326SBharat Bhushan
58835a43326SBharat Bhushan value = cn10k_ddr_perf_read_counter(pmu, i);
58935a43326SBharat Bhushan if (value == DDRC_PERF_CNT_MAX_VALUE) {
59035a43326SBharat Bhushan pr_info("Counter-(%d) reached max value\n", i);
59135a43326SBharat Bhushan cn10k_ddr_perf_event_update_all(pmu);
59235a43326SBharat Bhushan cn10k_ddr_perf_pmu_disable(&pmu->pmu);
59335a43326SBharat Bhushan cn10k_ddr_perf_pmu_enable(&pmu->pmu);
59435a43326SBharat Bhushan }
59535a43326SBharat Bhushan }
59635a43326SBharat Bhushan
59735a43326SBharat Bhushan return IRQ_HANDLED;
59835a43326SBharat Bhushan }
59935a43326SBharat Bhushan
cn10k_ddr_pmu_timer_handler(struct hrtimer * hrtimer)60035a43326SBharat Bhushan static enum hrtimer_restart cn10k_ddr_pmu_timer_handler(struct hrtimer *hrtimer)
60135a43326SBharat Bhushan {
60235a43326SBharat Bhushan struct cn10k_ddr_pmu *pmu = container_of(hrtimer, struct cn10k_ddr_pmu,
60335a43326SBharat Bhushan hrtimer);
60435a43326SBharat Bhushan unsigned long flags;
60535a43326SBharat Bhushan
60635a43326SBharat Bhushan local_irq_save(flags);
60735a43326SBharat Bhushan cn10k_ddr_pmu_overflow_handler(pmu);
60835a43326SBharat Bhushan local_irq_restore(flags);
60935a43326SBharat Bhushan
61035a43326SBharat Bhushan hrtimer_forward_now(hrtimer, cn10k_ddr_pmu_timer_period());
61135a43326SBharat Bhushan return HRTIMER_RESTART;
61235a43326SBharat Bhushan }
61335a43326SBharat Bhushan
cn10k_ddr_pmu_offline_cpu(unsigned int cpu,struct hlist_node * node)61468fa55f0SBharat Bhushan static int cn10k_ddr_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
61568fa55f0SBharat Bhushan {
61668fa55f0SBharat Bhushan struct cn10k_ddr_pmu *pmu = hlist_entry_safe(node, struct cn10k_ddr_pmu,
61768fa55f0SBharat Bhushan node);
61868fa55f0SBharat Bhushan unsigned int target;
61968fa55f0SBharat Bhushan
62068fa55f0SBharat Bhushan if (cpu != pmu->cpu)
62168fa55f0SBharat Bhushan return 0;
62268fa55f0SBharat Bhushan
62368fa55f0SBharat Bhushan target = cpumask_any_but(cpu_online_mask, cpu);
62468fa55f0SBharat Bhushan if (target >= nr_cpu_ids)
62568fa55f0SBharat Bhushan return 0;
62668fa55f0SBharat Bhushan
62768fa55f0SBharat Bhushan perf_pmu_migrate_context(&pmu->pmu, cpu, target);
62868fa55f0SBharat Bhushan pmu->cpu = target;
62968fa55f0SBharat Bhushan return 0;
63068fa55f0SBharat Bhushan }
63168fa55f0SBharat Bhushan
cn10k_ddr_perf_probe(struct platform_device * pdev)6327cf83e22SBharat Bhushan static int cn10k_ddr_perf_probe(struct platform_device *pdev)
6337cf83e22SBharat Bhushan {
6347cf83e22SBharat Bhushan struct cn10k_ddr_pmu *ddr_pmu;
6357cf83e22SBharat Bhushan struct resource *res;
6367cf83e22SBharat Bhushan void __iomem *base;
6377cf83e22SBharat Bhushan char *name;
6387cf83e22SBharat Bhushan int ret;
6397cf83e22SBharat Bhushan
6407cf83e22SBharat Bhushan ddr_pmu = devm_kzalloc(&pdev->dev, sizeof(*ddr_pmu), GFP_KERNEL);
6417cf83e22SBharat Bhushan if (!ddr_pmu)
6427cf83e22SBharat Bhushan return -ENOMEM;
6437cf83e22SBharat Bhushan
6447cf83e22SBharat Bhushan ddr_pmu->dev = &pdev->dev;
6457cf83e22SBharat Bhushan platform_set_drvdata(pdev, ddr_pmu);
6467cf83e22SBharat Bhushan
6477cf83e22SBharat Bhushan base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
6487cf83e22SBharat Bhushan if (IS_ERR(base))
6497cf83e22SBharat Bhushan return PTR_ERR(base);
6507cf83e22SBharat Bhushan
6517cf83e22SBharat Bhushan ddr_pmu->base = base;
6527cf83e22SBharat Bhushan
6537cf83e22SBharat Bhushan /* Setup the PMU counter to work in manual mode */
6547cf83e22SBharat Bhushan writeq_relaxed(OP_MODE_CTRL_VAL_MANNUAL, ddr_pmu->base +
6557cf83e22SBharat Bhushan DDRC_PERF_CNT_OP_MODE_CTRL);
6567cf83e22SBharat Bhushan
6577cf83e22SBharat Bhushan ddr_pmu->pmu = (struct pmu) {
6587cf83e22SBharat Bhushan .module = THIS_MODULE,
6597cf83e22SBharat Bhushan .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
6607cf83e22SBharat Bhushan .task_ctx_nr = perf_invalid_context,
6617cf83e22SBharat Bhushan .attr_groups = cn10k_attr_groups,
6627cf83e22SBharat Bhushan .event_init = cn10k_ddr_perf_event_init,
6637cf83e22SBharat Bhushan .add = cn10k_ddr_perf_event_add,
6647cf83e22SBharat Bhushan .del = cn10k_ddr_perf_event_del,
6657cf83e22SBharat Bhushan .start = cn10k_ddr_perf_event_start,
6667cf83e22SBharat Bhushan .stop = cn10k_ddr_perf_event_stop,
6677cf83e22SBharat Bhushan .read = cn10k_ddr_perf_event_update,
6687cf83e22SBharat Bhushan .pmu_enable = cn10k_ddr_perf_pmu_enable,
6697cf83e22SBharat Bhushan .pmu_disable = cn10k_ddr_perf_pmu_disable,
6707cf83e22SBharat Bhushan };
6717cf83e22SBharat Bhushan
6727cf83e22SBharat Bhushan /* Choose this cpu to collect perf data */
6737cf83e22SBharat Bhushan ddr_pmu->cpu = raw_smp_processor_id();
6747cf83e22SBharat Bhushan
6757cf83e22SBharat Bhushan name = devm_kasprintf(ddr_pmu->dev, GFP_KERNEL, "mrvl_ddr_pmu_%llx",
6767cf83e22SBharat Bhushan res->start);
6777cf83e22SBharat Bhushan if (!name)
6787cf83e22SBharat Bhushan return -ENOMEM;
6797cf83e22SBharat Bhushan
68035a43326SBharat Bhushan hrtimer_init(&ddr_pmu->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
68135a43326SBharat Bhushan ddr_pmu->hrtimer.function = cn10k_ddr_pmu_timer_handler;
68235a43326SBharat Bhushan
68368fa55f0SBharat Bhushan cpuhp_state_add_instance_nocalls(
68468fa55f0SBharat Bhushan CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
68568fa55f0SBharat Bhushan &ddr_pmu->node);
68668fa55f0SBharat Bhushan
6877cf83e22SBharat Bhushan ret = perf_pmu_register(&ddr_pmu->pmu, name, -1);
6887cf83e22SBharat Bhushan if (ret)
68968fa55f0SBharat Bhushan goto error;
6907cf83e22SBharat Bhushan
6917cf83e22SBharat Bhushan pr_info("CN10K DDR PMU Driver for ddrc@%llx\n", res->start);
6927cf83e22SBharat Bhushan return 0;
69368fa55f0SBharat Bhushan error:
69468fa55f0SBharat Bhushan cpuhp_state_remove_instance_nocalls(
69568fa55f0SBharat Bhushan CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
69668fa55f0SBharat Bhushan &ddr_pmu->node);
69768fa55f0SBharat Bhushan return ret;
6987cf83e22SBharat Bhushan }
6997cf83e22SBharat Bhushan
cn10k_ddr_perf_remove(struct platform_device * pdev)700*c802bd9eSUwe Kleine-König static void cn10k_ddr_perf_remove(struct platform_device *pdev)
7017cf83e22SBharat Bhushan {
7027cf83e22SBharat Bhushan struct cn10k_ddr_pmu *ddr_pmu = platform_get_drvdata(pdev);
7037cf83e22SBharat Bhushan
70468fa55f0SBharat Bhushan cpuhp_state_remove_instance_nocalls(
70568fa55f0SBharat Bhushan CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
70668fa55f0SBharat Bhushan &ddr_pmu->node);
70768fa55f0SBharat Bhushan
7087cf83e22SBharat Bhushan perf_pmu_unregister(&ddr_pmu->pmu);
7097cf83e22SBharat Bhushan }
7107cf83e22SBharat Bhushan
7117cf83e22SBharat Bhushan #ifdef CONFIG_OF
7127cf83e22SBharat Bhushan static const struct of_device_id cn10k_ddr_pmu_of_match[] = {
7137cf83e22SBharat Bhushan { .compatible = "marvell,cn10k-ddr-pmu", },
7147cf83e22SBharat Bhushan { },
7157cf83e22SBharat Bhushan };
7167cf83e22SBharat Bhushan MODULE_DEVICE_TABLE(of, cn10k_ddr_pmu_of_match);
7177cf83e22SBharat Bhushan #endif
7187cf83e22SBharat Bhushan
719e85930f0SGowthami Thiagarajan #ifdef CONFIG_ACPI
720e85930f0SGowthami Thiagarajan static const struct acpi_device_id cn10k_ddr_pmu_acpi_match[] = {
721e85930f0SGowthami Thiagarajan {"MRVL000A", 0},
722e85930f0SGowthami Thiagarajan {},
723e85930f0SGowthami Thiagarajan };
724e85930f0SGowthami Thiagarajan MODULE_DEVICE_TABLE(acpi, cn10k_ddr_pmu_acpi_match);
725e85930f0SGowthami Thiagarajan #endif
726e85930f0SGowthami Thiagarajan
7277cf83e22SBharat Bhushan static struct platform_driver cn10k_ddr_pmu_driver = {
7287cf83e22SBharat Bhushan .driver = {
7297cf83e22SBharat Bhushan .name = "cn10k-ddr-pmu",
7306676a42fSWill Deacon .of_match_table = of_match_ptr(cn10k_ddr_pmu_of_match),
731e85930f0SGowthami Thiagarajan .acpi_match_table = ACPI_PTR(cn10k_ddr_pmu_acpi_match),
7327cf83e22SBharat Bhushan .suppress_bind_attrs = true,
7337cf83e22SBharat Bhushan },
7347cf83e22SBharat Bhushan .probe = cn10k_ddr_perf_probe,
735*c802bd9eSUwe Kleine-König .remove_new = cn10k_ddr_perf_remove,
7367cf83e22SBharat Bhushan };
7377cf83e22SBharat Bhushan
cn10k_ddr_pmu_init(void)7387cf83e22SBharat Bhushan static int __init cn10k_ddr_pmu_init(void)
7397cf83e22SBharat Bhushan {
74068fa55f0SBharat Bhushan int ret;
74168fa55f0SBharat Bhushan
74268fa55f0SBharat Bhushan ret = cpuhp_setup_state_multi(
74368fa55f0SBharat Bhushan CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
74468fa55f0SBharat Bhushan "perf/marvell/cn10k/ddr:online", NULL,
74568fa55f0SBharat Bhushan cn10k_ddr_pmu_offline_cpu);
74668fa55f0SBharat Bhushan if (ret)
74768fa55f0SBharat Bhushan return ret;
74868fa55f0SBharat Bhushan
74968fa55f0SBharat Bhushan ret = platform_driver_register(&cn10k_ddr_pmu_driver);
75068fa55f0SBharat Bhushan if (ret)
75168fa55f0SBharat Bhushan cpuhp_remove_multi_state(
75268fa55f0SBharat Bhushan CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE);
75368fa55f0SBharat Bhushan return ret;
7547cf83e22SBharat Bhushan }
7557cf83e22SBharat Bhushan
cn10k_ddr_pmu_exit(void)7567cf83e22SBharat Bhushan static void __exit cn10k_ddr_pmu_exit(void)
7577cf83e22SBharat Bhushan {
7587cf83e22SBharat Bhushan platform_driver_unregister(&cn10k_ddr_pmu_driver);
75968fa55f0SBharat Bhushan cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE);
7607cf83e22SBharat Bhushan }
7617cf83e22SBharat Bhushan
7627cf83e22SBharat Bhushan module_init(cn10k_ddr_pmu_init);
7637cf83e22SBharat Bhushan module_exit(cn10k_ddr_pmu_exit);
7647cf83e22SBharat Bhushan
7657cf83e22SBharat Bhushan MODULE_AUTHOR("Bharat Bhushan <bbhushan2@marvell.com>");
7667cf83e22SBharat Bhushan MODULE_LICENSE("GPL v2");
767