1bc88bb2bSRuslan Bukin /*-
2bc88bb2bSRuslan Bukin * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
3bc88bb2bSRuslan Bukin * All rights reserved.
4bc88bb2bSRuslan Bukin *
5bc88bb2bSRuslan Bukin * This software was developed by the University of Cambridge Computer
6bc88bb2bSRuslan Bukin * Laboratory with support from ARM Ltd.
7bc88bb2bSRuslan Bukin *
8bc88bb2bSRuslan Bukin * Redistribution and use in source and binary forms, with or without
9bc88bb2bSRuslan Bukin * modification, are permitted provided that the following conditions
10bc88bb2bSRuslan Bukin * are met:
11bc88bb2bSRuslan Bukin * 1. Redistributions of source code must retain the above copyright
12bc88bb2bSRuslan Bukin * notice, this list of conditions and the following disclaimer.
13bc88bb2bSRuslan Bukin * 2. Redistributions in binary form must reproduce the above copyright
14bc88bb2bSRuslan Bukin * notice, this list of conditions and the following disclaimer in the
15bc88bb2bSRuslan Bukin * documentation and/or other materials provided with the distribution.
16bc88bb2bSRuslan Bukin *
17bc88bb2bSRuslan Bukin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18bc88bb2bSRuslan Bukin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19bc88bb2bSRuslan Bukin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20bc88bb2bSRuslan Bukin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21bc88bb2bSRuslan Bukin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22bc88bb2bSRuslan Bukin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23bc88bb2bSRuslan Bukin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24bc88bb2bSRuslan Bukin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25bc88bb2bSRuslan Bukin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26bc88bb2bSRuslan Bukin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27bc88bb2bSRuslan Bukin * SUCH DAMAGE.
28bc88bb2bSRuslan Bukin */
29bc88bb2bSRuslan Bukin
30bc88bb2bSRuslan Bukin #include <sys/param.h>
31bc88bb2bSRuslan Bukin #include <sys/systm.h>
32bc88bb2bSRuslan Bukin #include <sys/pmc.h>
33bc88bb2bSRuslan Bukin #include <sys/pmckern.h>
34bc88bb2bSRuslan Bukin
35bc88bb2bSRuslan Bukin #include <machine/pmc_mdep.h>
36bc88bb2bSRuslan Bukin #include <machine/cpu.h>
37bc88bb2bSRuslan Bukin
380a9a4d2cSEmmanuel Vadot #include "opt_acpi.h"
390a9a4d2cSEmmanuel Vadot
40bc88bb2bSRuslan Bukin static int arm64_npmcs;
41bc88bb2bSRuslan Bukin
42bc88bb2bSRuslan Bukin struct arm64_event_code_map {
43bc88bb2bSRuslan Bukin enum pmc_event pe_ev;
44bc88bb2bSRuslan Bukin uint8_t pe_code;
45bc88bb2bSRuslan Bukin };
46bc88bb2bSRuslan Bukin
47bc88bb2bSRuslan Bukin /*
48bc88bb2bSRuslan Bukin * Per-processor information.
49bc88bb2bSRuslan Bukin */
50bc88bb2bSRuslan Bukin struct arm64_cpu {
51bc88bb2bSRuslan Bukin struct pmc_hw *pc_arm64pmcs;
52bc88bb2bSRuslan Bukin };
53bc88bb2bSRuslan Bukin
54bc88bb2bSRuslan Bukin static struct arm64_cpu **arm64_pcpu;
55bc88bb2bSRuslan Bukin
56bc88bb2bSRuslan Bukin /*
57bc88bb2bSRuslan Bukin * Interrupt Enable Set Register
58bc88bb2bSRuslan Bukin */
59bc88bb2bSRuslan Bukin static __inline void
arm64_interrupt_enable(uint32_t pmc)60bc88bb2bSRuslan Bukin arm64_interrupt_enable(uint32_t pmc)
61bc88bb2bSRuslan Bukin {
62bc88bb2bSRuslan Bukin uint32_t reg;
63bc88bb2bSRuslan Bukin
64bc88bb2bSRuslan Bukin reg = (1 << pmc);
65fc232b89SAndrew Turner WRITE_SPECIALREG(pmintenset_el1, reg);
66bc88bb2bSRuslan Bukin
67bc88bb2bSRuslan Bukin isb();
68bc88bb2bSRuslan Bukin }
69bc88bb2bSRuslan Bukin
70bc88bb2bSRuslan Bukin /*
71bc88bb2bSRuslan Bukin * Interrupt Clear Set Register
72bc88bb2bSRuslan Bukin */
73bc88bb2bSRuslan Bukin static __inline void
arm64_interrupt_disable(uint32_t pmc)74bc88bb2bSRuslan Bukin arm64_interrupt_disable(uint32_t pmc)
75bc88bb2bSRuslan Bukin {
76bc88bb2bSRuslan Bukin uint32_t reg;
77bc88bb2bSRuslan Bukin
78bc88bb2bSRuslan Bukin reg = (1 << pmc);
79fc232b89SAndrew Turner WRITE_SPECIALREG(pmintenclr_el1, reg);
80bc88bb2bSRuslan Bukin
81bc88bb2bSRuslan Bukin isb();
82bc88bb2bSRuslan Bukin }
83bc88bb2bSRuslan Bukin
84bc88bb2bSRuslan Bukin /*
85bc88bb2bSRuslan Bukin * Counter Set Enable Register
86bc88bb2bSRuslan Bukin */
87bc88bb2bSRuslan Bukin static __inline void
arm64_counter_enable(unsigned int pmc)88bc88bb2bSRuslan Bukin arm64_counter_enable(unsigned int pmc)
89bc88bb2bSRuslan Bukin {
90bc88bb2bSRuslan Bukin uint32_t reg;
91bc88bb2bSRuslan Bukin
92bc88bb2bSRuslan Bukin reg = (1 << pmc);
93fc232b89SAndrew Turner WRITE_SPECIALREG(pmcntenset_el0, reg);
94bc88bb2bSRuslan Bukin
95bc88bb2bSRuslan Bukin isb();
96bc88bb2bSRuslan Bukin }
97bc88bb2bSRuslan Bukin
98bc88bb2bSRuslan Bukin /*
99bc88bb2bSRuslan Bukin * Counter Clear Enable Register
100bc88bb2bSRuslan Bukin */
101bc88bb2bSRuslan Bukin static __inline void
arm64_counter_disable(unsigned int pmc)102bc88bb2bSRuslan Bukin arm64_counter_disable(unsigned int pmc)
103bc88bb2bSRuslan Bukin {
104bc88bb2bSRuslan Bukin uint32_t reg;
105bc88bb2bSRuslan Bukin
106bc88bb2bSRuslan Bukin reg = (1 << pmc);
107fc232b89SAndrew Turner WRITE_SPECIALREG(pmcntenclr_el0, reg);
108bc88bb2bSRuslan Bukin
109bc88bb2bSRuslan Bukin isb();
110bc88bb2bSRuslan Bukin }
111bc88bb2bSRuslan Bukin
112bc88bb2bSRuslan Bukin /*
113bc88bb2bSRuslan Bukin * Performance Monitors Control Register
114bc88bb2bSRuslan Bukin */
115bc88bb2bSRuslan Bukin static uint32_t
arm64_pmcr_read(void)116bc88bb2bSRuslan Bukin arm64_pmcr_read(void)
117bc88bb2bSRuslan Bukin {
118bc88bb2bSRuslan Bukin uint32_t reg;
119bc88bb2bSRuslan Bukin
120fc232b89SAndrew Turner reg = READ_SPECIALREG(pmcr_el0);
121bc88bb2bSRuslan Bukin
122bc88bb2bSRuslan Bukin return (reg);
123bc88bb2bSRuslan Bukin }
124bc88bb2bSRuslan Bukin
125bc88bb2bSRuslan Bukin static void
arm64_pmcr_write(uint32_t reg)126bc88bb2bSRuslan Bukin arm64_pmcr_write(uint32_t reg)
127bc88bb2bSRuslan Bukin {
128bc88bb2bSRuslan Bukin
129fc232b89SAndrew Turner WRITE_SPECIALREG(pmcr_el0, reg);
130bc88bb2bSRuslan Bukin
131bc88bb2bSRuslan Bukin isb();
132bc88bb2bSRuslan Bukin }
133bc88bb2bSRuslan Bukin
134bc88bb2bSRuslan Bukin /*
135bc88bb2bSRuslan Bukin * Performance Count Register N
136bc88bb2bSRuslan Bukin */
137bc88bb2bSRuslan Bukin static uint32_t
arm64_pmcn_read(unsigned int pmc)138bc88bb2bSRuslan Bukin arm64_pmcn_read(unsigned int pmc)
139bc88bb2bSRuslan Bukin {
140bc88bb2bSRuslan Bukin
141bc88bb2bSRuslan Bukin KASSERT(pmc < arm64_npmcs, ("%s: illegal PMC number %d", __func__, pmc));
142bc88bb2bSRuslan Bukin
143fc232b89SAndrew Turner WRITE_SPECIALREG(pmselr_el0, pmc);
144bc88bb2bSRuslan Bukin
145bc88bb2bSRuslan Bukin isb();
146bc88bb2bSRuslan Bukin
147fc232b89SAndrew Turner return (READ_SPECIALREG(pmxevcntr_el0));
148bc88bb2bSRuslan Bukin }
149bc88bb2bSRuslan Bukin
150bc88bb2bSRuslan Bukin static void
arm64_pmcn_write(unsigned int pmc,uint32_t reg)151bc88bb2bSRuslan Bukin arm64_pmcn_write(unsigned int pmc, uint32_t reg)
152bc88bb2bSRuslan Bukin {
153bc88bb2bSRuslan Bukin
154bc88bb2bSRuslan Bukin KASSERT(pmc < arm64_npmcs, ("%s: illegal PMC number %d", __func__, pmc));
155bc88bb2bSRuslan Bukin
156fc232b89SAndrew Turner WRITE_SPECIALREG(pmselr_el0, pmc);
157fc232b89SAndrew Turner WRITE_SPECIALREG(pmxevcntr_el0, reg);
158bc88bb2bSRuslan Bukin
159bc88bb2bSRuslan Bukin isb();
160bc88bb2bSRuslan Bukin }
161bc88bb2bSRuslan Bukin
162bc88bb2bSRuslan Bukin static int
arm64_allocate_pmc(int cpu,int ri,struct pmc * pm,const struct pmc_op_pmcallocate * a)163bc88bb2bSRuslan Bukin arm64_allocate_pmc(int cpu, int ri, struct pmc *pm,
164bc88bb2bSRuslan Bukin const struct pmc_op_pmcallocate *a)
165bc88bb2bSRuslan Bukin {
1660e78510bSMitchell Horne uint32_t config;
167bc88bb2bSRuslan Bukin enum pmc_event pe;
168bc88bb2bSRuslan Bukin
169bc88bb2bSRuslan Bukin KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
170bc88bb2bSRuslan Bukin ("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
171bc88bb2bSRuslan Bukin KASSERT(ri >= 0 && ri < arm64_npmcs,
172bc88bb2bSRuslan Bukin ("[arm64,%d] illegal row index %d", __LINE__, ri));
173bc88bb2bSRuslan Bukin
174bc88bb2bSRuslan Bukin if (a->pm_class != PMC_CLASS_ARMV8) {
175bc88bb2bSRuslan Bukin return (EINVAL);
176bc88bb2bSRuslan Bukin }
177bc88bb2bSRuslan Bukin pe = a->pm_ev;
178bc88bb2bSRuslan Bukin
179c190fb35SMitchell Horne if ((a->pm_flags & PMC_F_EV_PMU) != 0) {
18027ea55fcSMitchell Horne config = a->pm_md.pm_md_config;
181c190fb35SMitchell Horne } else {
18227ea55fcSMitchell Horne config = (uint32_t)pe - PMC_EV_ARMV8_FIRST;
18324b2f4eaSAndrew Turner if (config > (PMC_EV_ARMV8_LAST - PMC_EV_ARMV8_FIRST))
18424b2f4eaSAndrew Turner return (EINVAL);
1858cc3815fSMitchell Horne }
1866bb7ba4aSJohn Baldwin
1876bb7ba4aSJohn Baldwin switch (a->pm_caps & (PMC_CAP_SYSTEM | PMC_CAP_USER)) {
1886bb7ba4aSJohn Baldwin case PMC_CAP_SYSTEM:
1896bb7ba4aSJohn Baldwin config |= PMEVTYPER_U;
1906bb7ba4aSJohn Baldwin break;
1916bb7ba4aSJohn Baldwin case PMC_CAP_USER:
1926bb7ba4aSJohn Baldwin config |= PMEVTYPER_P;
1936bb7ba4aSJohn Baldwin break;
1946bb7ba4aSJohn Baldwin default:
1956bb7ba4aSJohn Baldwin /*
1966bb7ba4aSJohn Baldwin * Trace both USER and SYSTEM if none are specified
1976bb7ba4aSJohn Baldwin * (default setting) or if both flags are specified
1986bb7ba4aSJohn Baldwin * (user explicitly requested both qualifiers).
1996bb7ba4aSJohn Baldwin */
2006bb7ba4aSJohn Baldwin break;
2016bb7ba4aSJohn Baldwin }
2026bb7ba4aSJohn Baldwin
203bc88bb2bSRuslan Bukin pm->pm_md.pm_arm64.pm_arm64_evsel = config;
204bc88bb2bSRuslan Bukin PMCDBG2(MDP, ALL, 2, "arm64-allocate ri=%d -> config=0x%x", ri, config);
205bc88bb2bSRuslan Bukin
2062129c8f6SMitchell Horne return (0);
207bc88bb2bSRuslan Bukin }
208bc88bb2bSRuslan Bukin
209bc88bb2bSRuslan Bukin
210bc88bb2bSRuslan Bukin static int
arm64_read_pmc(int cpu,int ri,struct pmc * pm,pmc_value_t * v)21139f92a76SMitchell Horne arm64_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
212bc88bb2bSRuslan Bukin {
213bc88bb2bSRuslan Bukin pmc_value_t tmp;
21412d05303SAndrew Turner register_t s;
21512d05303SAndrew Turner int reg;
216bc88bb2bSRuslan Bukin
217bc88bb2bSRuslan Bukin KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
218bc88bb2bSRuslan Bukin ("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
219bc88bb2bSRuslan Bukin KASSERT(ri >= 0 && ri < arm64_npmcs,
220bc88bb2bSRuslan Bukin ("[arm64,%d] illegal row index %d", __LINE__, ri));
221bc88bb2bSRuslan Bukin
22212d05303SAndrew Turner /*
22312d05303SAndrew Turner * Ensure we don't get interrupted while updating the overflow count.
22412d05303SAndrew Turner */
22512d05303SAndrew Turner s = intr_disable();
226bc88bb2bSRuslan Bukin tmp = arm64_pmcn_read(ri);
22712d05303SAndrew Turner reg = (1 << ri);
22812d05303SAndrew Turner if ((READ_SPECIALREG(pmovsclr_el0) & reg) != 0) {
22912d05303SAndrew Turner /* Clear Overflow Flag */
23012d05303SAndrew Turner WRITE_SPECIALREG(pmovsclr_el0, reg);
2316815909aSAndrew Turner pm->pm_pcpu_state[cpu].pps_overflowcnt++;
23212d05303SAndrew Turner
23312d05303SAndrew Turner /* Reread counter in case we raced. */
23412d05303SAndrew Turner tmp = arm64_pmcn_read(ri);
23512d05303SAndrew Turner }
2366815909aSAndrew Turner tmp += 0x100000000llu * pm->pm_pcpu_state[cpu].pps_overflowcnt;
23712d05303SAndrew Turner intr_restore(s);
238bc88bb2bSRuslan Bukin
239bc88bb2bSRuslan Bukin PMCDBG2(MDP, REA, 2, "arm64-read id=%d -> %jd", ri, tmp);
240e74c7ffcSJessica Clarke if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
241e74c7ffcSJessica Clarke /*
242e74c7ffcSJessica Clarke * Clamp value to 0 if the counter just overflowed,
243e74c7ffcSJessica Clarke * otherwise the returned reload count would wrap to a
244e74c7ffcSJessica Clarke * huge value.
245e74c7ffcSJessica Clarke */
246e74c7ffcSJessica Clarke if ((tmp & (1ull << 63)) == 0)
247e74c7ffcSJessica Clarke tmp = 0;
248bc88bb2bSRuslan Bukin else
249e74c7ffcSJessica Clarke tmp = ARMV8_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
250e74c7ffcSJessica Clarke }
251bc88bb2bSRuslan Bukin *v = tmp;
252bc88bb2bSRuslan Bukin
2532129c8f6SMitchell Horne return (0);
254bc88bb2bSRuslan Bukin }
255bc88bb2bSRuslan Bukin
256bc88bb2bSRuslan Bukin static int
arm64_write_pmc(int cpu,int ri,struct pmc * pm,pmc_value_t v)25739f92a76SMitchell Horne arm64_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
258bc88bb2bSRuslan Bukin {
259bc88bb2bSRuslan Bukin
260bc88bb2bSRuslan Bukin KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
261bc88bb2bSRuslan Bukin ("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
262bc88bb2bSRuslan Bukin KASSERT(ri >= 0 && ri < arm64_npmcs,
263bc88bb2bSRuslan Bukin ("[arm64,%d] illegal row-index %d", __LINE__, ri));
264bc88bb2bSRuslan Bukin
265bc88bb2bSRuslan Bukin if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
266bc88bb2bSRuslan Bukin v = ARMV8_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
267bc88bb2bSRuslan Bukin
268bc88bb2bSRuslan Bukin PMCDBG3(MDP, WRI, 1, "arm64-write cpu=%d ri=%d v=%jx", cpu, ri, v);
269bc88bb2bSRuslan Bukin
2706815909aSAndrew Turner pm->pm_pcpu_state[cpu].pps_overflowcnt = v >> 32;
271bc88bb2bSRuslan Bukin arm64_pmcn_write(ri, v);
272bc88bb2bSRuslan Bukin
2732129c8f6SMitchell Horne return (0);
274bc88bb2bSRuslan Bukin }
275bc88bb2bSRuslan Bukin
276bc88bb2bSRuslan Bukin static int
arm64_config_pmc(int cpu,int ri,struct pmc * pm)277bc88bb2bSRuslan Bukin arm64_config_pmc(int cpu, int ri, struct pmc *pm)
278bc88bb2bSRuslan Bukin {
279bc88bb2bSRuslan Bukin struct pmc_hw *phw;
280bc88bb2bSRuslan Bukin
281bc88bb2bSRuslan Bukin PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
282bc88bb2bSRuslan Bukin
283bc88bb2bSRuslan Bukin KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
284bc88bb2bSRuslan Bukin ("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
285bc88bb2bSRuslan Bukin KASSERT(ri >= 0 && ri < arm64_npmcs,
286bc88bb2bSRuslan Bukin ("[arm64,%d] illegal row-index %d", __LINE__, ri));
287bc88bb2bSRuslan Bukin
288bc88bb2bSRuslan Bukin phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri];
289bc88bb2bSRuslan Bukin
290bc88bb2bSRuslan Bukin KASSERT(pm == NULL || phw->phw_pmc == NULL,
291bc88bb2bSRuslan Bukin ("[arm64,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
292bc88bb2bSRuslan Bukin __LINE__, pm, phw->phw_pmc));
293bc88bb2bSRuslan Bukin
294bc88bb2bSRuslan Bukin phw->phw_pmc = pm;
295bc88bb2bSRuslan Bukin
2962129c8f6SMitchell Horne return (0);
297bc88bb2bSRuslan Bukin }
298bc88bb2bSRuslan Bukin
299bc88bb2bSRuslan Bukin static int
arm64_start_pmc(int cpu,int ri,struct pmc * pm)30039f92a76SMitchell Horne arm64_start_pmc(int cpu, int ri, struct pmc *pm)
301bc88bb2bSRuslan Bukin {
302bc88bb2bSRuslan Bukin uint32_t config;
303bc88bb2bSRuslan Bukin
304bc88bb2bSRuslan Bukin config = pm->pm_md.pm_arm64.pm_arm64_evsel;
305bc88bb2bSRuslan Bukin
306bc88bb2bSRuslan Bukin /*
307bc88bb2bSRuslan Bukin * Configure the event selection.
308bc88bb2bSRuslan Bukin */
309fc232b89SAndrew Turner WRITE_SPECIALREG(pmselr_el0, ri);
310fc232b89SAndrew Turner WRITE_SPECIALREG(pmxevtyper_el0, config);
311bc88bb2bSRuslan Bukin
312bc88bb2bSRuslan Bukin isb();
313bc88bb2bSRuslan Bukin
314bc88bb2bSRuslan Bukin /*
315bc88bb2bSRuslan Bukin * Enable the PMC.
316bc88bb2bSRuslan Bukin */
317bc88bb2bSRuslan Bukin arm64_interrupt_enable(ri);
318bc88bb2bSRuslan Bukin arm64_counter_enable(ri);
319bc88bb2bSRuslan Bukin
3202129c8f6SMitchell Horne return (0);
321bc88bb2bSRuslan Bukin }
322bc88bb2bSRuslan Bukin
323bc88bb2bSRuslan Bukin static int
arm64_stop_pmc(int cpu,int ri,struct pmc * pm __unused)32439f92a76SMitchell Horne arm64_stop_pmc(int cpu, int ri, struct pmc *pm __unused)
325bc88bb2bSRuslan Bukin {
326bc88bb2bSRuslan Bukin /*
327bc88bb2bSRuslan Bukin * Disable the PMCs.
328bc88bb2bSRuslan Bukin */
329bc88bb2bSRuslan Bukin arm64_counter_disable(ri);
330bc88bb2bSRuslan Bukin arm64_interrupt_disable(ri);
331bc88bb2bSRuslan Bukin
3322129c8f6SMitchell Horne return (0);
333bc88bb2bSRuslan Bukin }
334bc88bb2bSRuslan Bukin
335bc88bb2bSRuslan Bukin static int
arm64_release_pmc(int cpu,int ri,struct pmc * pmc)336bc88bb2bSRuslan Bukin arm64_release_pmc(int cpu, int ri, struct pmc *pmc)
337bc88bb2bSRuslan Bukin {
338e1988353SJohn Baldwin struct pmc_hw *phw __diagused;
339bc88bb2bSRuslan Bukin
340bc88bb2bSRuslan Bukin KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
341bc88bb2bSRuslan Bukin ("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
342bc88bb2bSRuslan Bukin KASSERT(ri >= 0 && ri < arm64_npmcs,
343bc88bb2bSRuslan Bukin ("[arm64,%d] illegal row-index %d", __LINE__, ri));
344bc88bb2bSRuslan Bukin
345bc88bb2bSRuslan Bukin phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri];
346bc88bb2bSRuslan Bukin KASSERT(phw->phw_pmc == NULL,
347bc88bb2bSRuslan Bukin ("[arm64,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
348bc88bb2bSRuslan Bukin
3492129c8f6SMitchell Horne return (0);
350bc88bb2bSRuslan Bukin }
351bc88bb2bSRuslan Bukin
352bc88bb2bSRuslan Bukin static int
arm64_intr(struct trapframe * tf)353eb7c9019SMatt Macy arm64_intr(struct trapframe *tf)
354bc88bb2bSRuslan Bukin {
355bc88bb2bSRuslan Bukin int retval, ri;
356bc88bb2bSRuslan Bukin struct pmc *pm;
357bc88bb2bSRuslan Bukin int error;
358eb7c9019SMatt Macy int reg, cpu;
359bc88bb2bSRuslan Bukin
3607f5336f6SMatt Macy cpu = curcpu;
361bc88bb2bSRuslan Bukin KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
362bc88bb2bSRuslan Bukin ("[arm64,%d] CPU %d out of range", __LINE__, cpu));
363bc88bb2bSRuslan Bukin
36413f5a307SMitchell Horne PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *)tf,
36513f5a307SMitchell Horne TRAPF_USERMODE(tf));
36613f5a307SMitchell Horne
367bc88bb2bSRuslan Bukin retval = 0;
368bc88bb2bSRuslan Bukin
369bc88bb2bSRuslan Bukin for (ri = 0; ri < arm64_npmcs; ri++) {
370bc88bb2bSRuslan Bukin pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc;
371bc88bb2bSRuslan Bukin if (pm == NULL)
372bc88bb2bSRuslan Bukin continue;
373bc88bb2bSRuslan Bukin /* Check if counter is overflowed */
374bc88bb2bSRuslan Bukin reg = (1 << ri);
375fc232b89SAndrew Turner if ((READ_SPECIALREG(pmovsclr_el0) & reg) == 0)
376bc88bb2bSRuslan Bukin continue;
377bc88bb2bSRuslan Bukin /* Clear Overflow Flag */
378fc232b89SAndrew Turner WRITE_SPECIALREG(pmovsclr_el0, reg);
379bc88bb2bSRuslan Bukin
380bc88bb2bSRuslan Bukin isb();
381bc88bb2bSRuslan Bukin
382bc88bb2bSRuslan Bukin retval = 1; /* Found an interrupting PMC. */
38312d05303SAndrew Turner
3846815909aSAndrew Turner pm->pm_pcpu_state[cpu].pps_overflowcnt += 1;
385e74c7ffcSJessica Clarke
386e74c7ffcSJessica Clarke if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
38712d05303SAndrew Turner continue;
38812d05303SAndrew Turner
389bc88bb2bSRuslan Bukin if (pm->pm_state != PMC_STATE_RUNNING)
390bc88bb2bSRuslan Bukin continue;
391bc88bb2bSRuslan Bukin
392eb7c9019SMatt Macy error = pmc_process_interrupt(PMC_HR, pm, tf);
393bc88bb2bSRuslan Bukin if (error)
39439f92a76SMitchell Horne arm64_stop_pmc(cpu, ri, pm);
395bc88bb2bSRuslan Bukin
396bc88bb2bSRuslan Bukin /* Reload sampling count */
39739f92a76SMitchell Horne arm64_write_pmc(cpu, ri, pm, pm->pm_sc.pm_reloadcount);
398bc88bb2bSRuslan Bukin }
399bc88bb2bSRuslan Bukin
400bc88bb2bSRuslan Bukin return (retval);
401bc88bb2bSRuslan Bukin }
402bc88bb2bSRuslan Bukin
403bc88bb2bSRuslan Bukin static int
arm64_describe(int cpu,int ri,struct pmc_info * pi,struct pmc ** ppmc)404bc88bb2bSRuslan Bukin arm64_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
405bc88bb2bSRuslan Bukin {
406bc88bb2bSRuslan Bukin struct pmc_hw *phw;
407bc88bb2bSRuslan Bukin
408bc88bb2bSRuslan Bukin KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
409bc88bb2bSRuslan Bukin ("[arm64,%d], illegal CPU %d", __LINE__, cpu));
410bc88bb2bSRuslan Bukin KASSERT(ri >= 0 && ri < arm64_npmcs,
411bc88bb2bSRuslan Bukin ("[arm64,%d] row-index %d out of range", __LINE__, ri));
412bc88bb2bSRuslan Bukin
413bc88bb2bSRuslan Bukin phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri];
41431610e34SMitchell Horne
41531610e34SMitchell Horne snprintf(pi->pm_name, sizeof(pi->pm_name), "ARMV8-%d", ri);
416bc88bb2bSRuslan Bukin pi->pm_class = PMC_CLASS_ARMV8;
41731610e34SMitchell Horne
418bc88bb2bSRuslan Bukin if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
419bc88bb2bSRuslan Bukin pi->pm_enabled = TRUE;
420bc88bb2bSRuslan Bukin *ppmc = phw->phw_pmc;
421bc88bb2bSRuslan Bukin } else {
422bc88bb2bSRuslan Bukin pi->pm_enabled = FALSE;
423bc88bb2bSRuslan Bukin *ppmc = NULL;
424bc88bb2bSRuslan Bukin }
425bc88bb2bSRuslan Bukin
426bc88bb2bSRuslan Bukin return (0);
427bc88bb2bSRuslan Bukin }
428bc88bb2bSRuslan Bukin
429bc88bb2bSRuslan Bukin static int
arm64_get_config(int cpu,int ri,struct pmc ** ppm)430bc88bb2bSRuslan Bukin arm64_get_config(int cpu, int ri, struct pmc **ppm)
431bc88bb2bSRuslan Bukin {
432bc88bb2bSRuslan Bukin
433bc88bb2bSRuslan Bukin *ppm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc;
434bc88bb2bSRuslan Bukin
435bc88bb2bSRuslan Bukin return (0);
436bc88bb2bSRuslan Bukin }
437bc88bb2bSRuslan Bukin
438bc88bb2bSRuslan Bukin static int
arm64_pcpu_init(struct pmc_mdep * md,int cpu)439bc88bb2bSRuslan Bukin arm64_pcpu_init(struct pmc_mdep *md, int cpu)
440bc88bb2bSRuslan Bukin {
441bc88bb2bSRuslan Bukin struct arm64_cpu *pac;
442bc88bb2bSRuslan Bukin struct pmc_hw *phw;
443bc88bb2bSRuslan Bukin struct pmc_cpu *pc;
444bc88bb2bSRuslan Bukin uint64_t pmcr;
445bc88bb2bSRuslan Bukin int first_ri;
446bc88bb2bSRuslan Bukin int i;
447bc88bb2bSRuslan Bukin
448bc88bb2bSRuslan Bukin KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
449bc88bb2bSRuslan Bukin ("[arm64,%d] wrong cpu number %d", __LINE__, cpu));
4508bdceaecSMitchell Horne PMCDBG0(MDP, INI, 1, "arm64-pcpu-init");
451bc88bb2bSRuslan Bukin
452bc88bb2bSRuslan Bukin arm64_pcpu[cpu] = pac = malloc(sizeof(struct arm64_cpu), M_PMC,
453bc88bb2bSRuslan Bukin M_WAITOK | M_ZERO);
454bc88bb2bSRuslan Bukin
455bc88bb2bSRuslan Bukin pac->pc_arm64pmcs = malloc(sizeof(struct pmc_hw) * arm64_npmcs,
456bc88bb2bSRuslan Bukin M_PMC, M_WAITOK | M_ZERO);
457bc88bb2bSRuslan Bukin pc = pmc_pcpu[cpu];
458bc88bb2bSRuslan Bukin first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8].pcd_ri;
459bc88bb2bSRuslan Bukin KASSERT(pc != NULL, ("[arm64,%d] NULL per-cpu pointer", __LINE__));
460bc88bb2bSRuslan Bukin
461bc88bb2bSRuslan Bukin for (i = 0, phw = pac->pc_arm64pmcs; i < arm64_npmcs; i++, phw++) {
462bc88bb2bSRuslan Bukin phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
463bc88bb2bSRuslan Bukin PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i);
464bc88bb2bSRuslan Bukin phw->phw_pmc = NULL;
465bc88bb2bSRuslan Bukin pc->pc_hwpmcs[i + first_ri] = phw;
466bc88bb2bSRuslan Bukin }
467bc88bb2bSRuslan Bukin
468b826cc3cSMitchell Horne /*
469b826cc3cSMitchell Horne * Disable all counters and overflow interrupts. Upon reset they are in
470b826cc3cSMitchell Horne * an undefined state.
471b826cc3cSMitchell Horne *
472b826cc3cSMitchell Horne * Don't issue an isb here, just wait for the one in arm64_pmcr_write()
473b826cc3cSMitchell Horne * to make the writes visible.
474b826cc3cSMitchell Horne */
475b826cc3cSMitchell Horne WRITE_SPECIALREG(pmcntenclr_el0, 0xffffffff);
476b826cc3cSMitchell Horne WRITE_SPECIALREG(pmintenclr_el1, 0xffffffff);
477b826cc3cSMitchell Horne
478bc88bb2bSRuslan Bukin /* Enable unit */
479bc88bb2bSRuslan Bukin pmcr = arm64_pmcr_read();
480bc88bb2bSRuslan Bukin pmcr |= PMCR_E;
481bc88bb2bSRuslan Bukin arm64_pmcr_write(pmcr);
482bc88bb2bSRuslan Bukin
483bc88bb2bSRuslan Bukin return (0);
484bc88bb2bSRuslan Bukin }
485bc88bb2bSRuslan Bukin
486bc88bb2bSRuslan Bukin static int
arm64_pcpu_fini(struct pmc_mdep * md,int cpu)487bc88bb2bSRuslan Bukin arm64_pcpu_fini(struct pmc_mdep *md, int cpu)
488bc88bb2bSRuslan Bukin {
489bc88bb2bSRuslan Bukin uint32_t pmcr;
490bc88bb2bSRuslan Bukin
4918bdceaecSMitchell Horne PMCDBG0(MDP, INI, 1, "arm64-pcpu-fini");
4928bdceaecSMitchell Horne
493bc88bb2bSRuslan Bukin pmcr = arm64_pmcr_read();
494bc88bb2bSRuslan Bukin pmcr &= ~PMCR_E;
495bc88bb2bSRuslan Bukin arm64_pmcr_write(pmcr);
496bc88bb2bSRuslan Bukin
4978bdceaecSMitchell Horne free(arm64_pcpu[cpu]->pc_arm64pmcs, M_PMC);
4988bdceaecSMitchell Horne free(arm64_pcpu[cpu], M_PMC);
4998bdceaecSMitchell Horne arm64_pcpu[cpu] = NULL;
5008bdceaecSMitchell Horne
501bc88bb2bSRuslan Bukin return (0);
502bc88bb2bSRuslan Bukin }
503bc88bb2bSRuslan Bukin
504bc88bb2bSRuslan Bukin struct pmc_mdep *
pmc_arm64_initialize(void)50505cef747SAndrew Turner pmc_arm64_initialize(void)
506bc88bb2bSRuslan Bukin {
507bc88bb2bSRuslan Bukin struct pmc_mdep *pmc_mdep;
508bc88bb2bSRuslan Bukin struct pmc_classdep *pcd;
509fdfeaa66SAleksandr Rybalko int classes, idcode, impcode;
510bc88bb2bSRuslan Bukin int reg;
5115867cccdSMitchell Horne uint64_t midr;
512bc88bb2bSRuslan Bukin
513bc88bb2bSRuslan Bukin reg = arm64_pmcr_read();
514bc88bb2bSRuslan Bukin arm64_npmcs = (reg & PMCR_N_MASK) >> PMCR_N_SHIFT;
515da11e1f9SAndrew Turner impcode = (reg & PMCR_IMP_MASK) >> PMCR_IMP_SHIFT;
516bc88bb2bSRuslan Bukin idcode = (reg & PMCR_IDCODE_MASK) >> PMCR_IDCODE_SHIFT;
517bc88bb2bSRuslan Bukin
518bc88bb2bSRuslan Bukin PMCDBG1(MDP, INI, 1, "arm64-init npmcs=%d", arm64_npmcs);
519bc88bb2bSRuslan Bukin
520bc88bb2bSRuslan Bukin /*
5215867cccdSMitchell Horne * Write the CPU model to kern.hwpmc.cpuid.
5225867cccdSMitchell Horne *
5235867cccdSMitchell Horne * We zero the variant and revision fields.
5245867cccdSMitchell Horne *
5255867cccdSMitchell Horne * TODO: how to handle differences between cores due to big.LITTLE?
5265867cccdSMitchell Horne * For now, just use MIDR from CPU 0.
5275867cccdSMitchell Horne */
5285867cccdSMitchell Horne midr = (uint64_t)(pcpu_find(0)->pc_midr);
5295867cccdSMitchell Horne midr &= ~(CPU_VAR_MASK | CPU_REV_MASK);
5305867cccdSMitchell Horne snprintf(pmc_cpuid, sizeof(pmc_cpuid), "0x%016lx", midr);
5315867cccdSMitchell Horne
5325867cccdSMitchell Horne /*
533bc88bb2bSRuslan Bukin * Allocate space for pointers to PMC HW descriptors and for
534bc88bb2bSRuslan Bukin * the MDEP structure used by MI code.
535bc88bb2bSRuslan Bukin */
536bc88bb2bSRuslan Bukin arm64_pcpu = malloc(sizeof(struct arm64_cpu *) * pmc_cpu_max(),
537bc88bb2bSRuslan Bukin M_PMC, M_WAITOK | M_ZERO);
538bc88bb2bSRuslan Bukin
539fdfeaa66SAleksandr Rybalko /* One AArch64 CPU class */
540fdfeaa66SAleksandr Rybalko classes = 1;
541fdfeaa66SAleksandr Rybalko
5420a9a4d2cSEmmanuel Vadot #ifdef DEV_ACPI
543fdfeaa66SAleksandr Rybalko /* Query presence of optional classes and set max class. */
544fdfeaa66SAleksandr Rybalko if (pmc_cmn600_nclasses() > 0)
545fdfeaa66SAleksandr Rybalko classes = MAX(classes, PMC_MDEP_CLASS_INDEX_CMN600);
546fdfeaa66SAleksandr Rybalko if (pmc_dmc620_nclasses() > 0)
547fdfeaa66SAleksandr Rybalko classes = MAX(classes, PMC_MDEP_CLASS_INDEX_DMC620_C);
5480a9a4d2cSEmmanuel Vadot #endif
549fdfeaa66SAleksandr Rybalko
550fdfeaa66SAleksandr Rybalko pmc_mdep = pmc_mdep_alloc(classes);
551bc88bb2bSRuslan Bukin
552da11e1f9SAndrew Turner switch(impcode) {
553da11e1f9SAndrew Turner case PMCR_IMP_ARM:
554bc88bb2bSRuslan Bukin switch (idcode) {
555da11e1f9SAndrew Turner case PMCR_IDCODE_CORTEX_A76:
556da11e1f9SAndrew Turner case PMCR_IDCODE_NEOVERSE_N1:
557da11e1f9SAndrew Turner pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A76;
558da11e1f9SAndrew Turner break;
559bc88bb2bSRuslan Bukin case PMCR_IDCODE_CORTEX_A57:
560bc88bb2bSRuslan Bukin case PMCR_IDCODE_CORTEX_A72:
561bc88bb2bSRuslan Bukin pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A57;
562bc88bb2bSRuslan Bukin break;
563bc88bb2bSRuslan Bukin default:
564bc88bb2bSRuslan Bukin case PMCR_IDCODE_CORTEX_A53:
565bc88bb2bSRuslan Bukin pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53;
566bc88bb2bSRuslan Bukin break;
567bc88bb2bSRuslan Bukin }
568da11e1f9SAndrew Turner break;
569da11e1f9SAndrew Turner default:
570da11e1f9SAndrew Turner pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53;
571da11e1f9SAndrew Turner break;
572da11e1f9SAndrew Turner }
573bc88bb2bSRuslan Bukin
574bc88bb2bSRuslan Bukin pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8];
575bc88bb2bSRuslan Bukin pcd->pcd_caps = ARMV8_PMC_CAPS;
576bc88bb2bSRuslan Bukin pcd->pcd_class = PMC_CLASS_ARMV8;
577bc88bb2bSRuslan Bukin pcd->pcd_num = arm64_npmcs;
578bc88bb2bSRuslan Bukin pcd->pcd_ri = pmc_mdep->pmd_npmc;
579bc88bb2bSRuslan Bukin pcd->pcd_width = 32;
580bc88bb2bSRuslan Bukin
581bc88bb2bSRuslan Bukin pcd->pcd_allocate_pmc = arm64_allocate_pmc;
582bc88bb2bSRuslan Bukin pcd->pcd_config_pmc = arm64_config_pmc;
583bc88bb2bSRuslan Bukin pcd->pcd_pcpu_fini = arm64_pcpu_fini;
584bc88bb2bSRuslan Bukin pcd->pcd_pcpu_init = arm64_pcpu_init;
585bc88bb2bSRuslan Bukin pcd->pcd_describe = arm64_describe;
586bc88bb2bSRuslan Bukin pcd->pcd_get_config = arm64_get_config;
587bc88bb2bSRuslan Bukin pcd->pcd_read_pmc = arm64_read_pmc;
588bc88bb2bSRuslan Bukin pcd->pcd_release_pmc = arm64_release_pmc;
589bc88bb2bSRuslan Bukin pcd->pcd_start_pmc = arm64_start_pmc;
590bc88bb2bSRuslan Bukin pcd->pcd_stop_pmc = arm64_stop_pmc;
591bc88bb2bSRuslan Bukin pcd->pcd_write_pmc = arm64_write_pmc;
592bc88bb2bSRuslan Bukin
593bc88bb2bSRuslan Bukin pmc_mdep->pmd_intr = arm64_intr;
594bc88bb2bSRuslan Bukin pmc_mdep->pmd_npmc += arm64_npmcs;
595bc88bb2bSRuslan Bukin
5960a9a4d2cSEmmanuel Vadot #ifdef DEV_ACPI
597fdfeaa66SAleksandr Rybalko if (pmc_cmn600_nclasses() > 0)
598fdfeaa66SAleksandr Rybalko pmc_cmn600_initialize(pmc_mdep);
599fdfeaa66SAleksandr Rybalko if (pmc_dmc620_nclasses() > 0) {
600fdfeaa66SAleksandr Rybalko pmc_dmc620_initialize_cd2(pmc_mdep);
601fdfeaa66SAleksandr Rybalko pmc_dmc620_initialize_c(pmc_mdep);
602fdfeaa66SAleksandr Rybalko }
6030a9a4d2cSEmmanuel Vadot #endif
604fdfeaa66SAleksandr Rybalko
605bc88bb2bSRuslan Bukin return (pmc_mdep);
606bc88bb2bSRuslan Bukin }
607bc88bb2bSRuslan Bukin
608bc88bb2bSRuslan Bukin void
pmc_arm64_finalize(struct pmc_mdep * md)609bc88bb2bSRuslan Bukin pmc_arm64_finalize(struct pmc_mdep *md)
610bc88bb2bSRuslan Bukin {
6118bdceaecSMitchell Horne PMCDBG0(MDP, INI, 1, "arm64-finalize");
612bc88bb2bSRuslan Bukin
613*90a6ea5cSMitchell Horne for (int i = 0; i < pmc_cpu_max(); i++)
614*90a6ea5cSMitchell Horne KASSERT(arm64_pcpu[i] == NULL,
615*90a6ea5cSMitchell Horne ("[arm64,%d] non-null pcpu cpu %d", __LINE__, i));
616*90a6ea5cSMitchell Horne
6178bdceaecSMitchell Horne free(arm64_pcpu, M_PMC);
618bc88bb2bSRuslan Bukin }
619