1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2003-2008 Joseph Koshy 5 * Copyright (c) 2007 The FreeBSD Foundation 6 * Copyright (c) 2021 ARM Ltd 7 * 8 * Portions of this software were developed by A. Joseph Koshy under 9 * sponsorship from the FreeBSD Foundation and Google, Inc. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 /* Arm CoreLink CMN-600 Coherent Mesh Network PMU Driver */ 34 35 #include <sys/param.h> 36 #include <sys/lock.h> 37 #include <sys/malloc.h> 38 #include <sys/module.h> 39 #include <sys/mutex.h> 40 #include <sys/pmc.h> 41 #include <sys/pmckern.h> 42 #include <sys/systm.h> 43 44 #include <machine/cmn600_reg.h> 45 46 struct cmn600_descr { 47 struct pmc_descr pd_descr; /* "base class" */ 48 void *pd_rw_arg; /* Argument to use with read/write */ 49 struct pmc *pd_pmc; 50 struct pmc_hw *pd_phw; 51 uint32_t pd_nodeid; 52 int32_t pd_node_type; 53 int pd_local_counter; 54 55 }; 56 57 static struct cmn600_descr **cmn600_pmcdesc; 58 59 static struct cmn600_pmc cmn600_pmcs[CMN600_UNIT_MAX]; 60 static int cmn600_units = 0; 61 62 static inline struct cmn600_descr * 63 cmn600desc(int ri) 64 { 65 66 return (cmn600_pmcdesc[ri]); 67 } 68 69 static inline int 70 class_ri2unit(int ri) 71 { 72 73 return (ri / CMN600_COUNTERS_N); 74 } 75 76 #define EVENCNTR(x) (((x) >> POR_DT_PMEVCNT_EVENCNT_SHIFT) << \ 77 POR_DTM_PMEVCNT_CNTR_WIDTH) 78 #define ODDCNTR(x) (((x) >> POR_DT_PMEVCNT_ODDCNT_SHIFT) << \ 79 POR_DTM_PMEVCNT_CNTR_WIDTH) 80 81 static uint64_t 82 cmn600_pmu_readcntr(void *arg, u_int nodeid, u_int xpcntr, u_int dtccntr, 83 u_int width) 84 { 85 uint64_t dtcval, xpval; 86 87 KASSERT(xpcntr < 4, ("[cmn600,%d] XP counter number %d is too big." 88 " Max: 3", __LINE__, xpcntr)); 89 KASSERT(dtccntr < 8, ("[cmn600,%d] Global counter number %d is too" 90 " big. Max: 7", __LINE__, dtccntr)); 91 92 dtcval = pmu_cmn600_rd8(arg, nodeid, NODE_TYPE_DTC, 93 POR_DT_PMEVCNT(dtccntr >> 1)); 94 if (width == 4) { 95 dtcval = (dtccntr & 1) ? ODDCNTR(dtcval) : EVENCNTR(dtcval); 96 dtcval &= 0xffffffff0000UL; 97 } else 98 dtcval <<= POR_DTM_PMEVCNT_CNTR_WIDTH; 99 100 xpval = pmu_cmn600_rd8(arg, nodeid, NODE_TYPE_XP, POR_DTM_PMEVCNT); 101 xpval >>= xpcntr * POR_DTM_PMEVCNT_CNTR_WIDTH; 102 xpval &= 0xffffUL; 103 return (dtcval | xpval); 104 } 105 106 static void 107 cmn600_pmu_writecntr(void *arg, u_int nodeid, u_int xpcntr, u_int dtccntr, 108 u_int width, uint64_t val) 109 { 110 int shift; 111 112 KASSERT(xpcntr < 4, ("[cmn600,%d] XP counter number %d is too big." 113 " Max: 3", __LINE__, xpcntr)); 114 KASSERT(dtccntr < 8, ("[cmn600,%d] Global counter number %d is too" 115 " big. Max: 7", __LINE__, dtccntr)); 116 117 if (width == 4) { 118 shift = (dtccntr & 1) ? POR_DT_PMEVCNT_ODDCNT_SHIFT : 119 POR_DT_PMEVCNT_EVENCNT_SHIFT; 120 pmu_cmn600_md8(arg, nodeid, NODE_TYPE_DTC, 121 POR_DT_PMEVCNT(dtccntr >> 1), 0xffffffffUL << shift, 122 ((val >> POR_DTM_PMEVCNT_CNTR_WIDTH) & 0xffffffff) << shift); 123 } else 124 pmu_cmn600_wr8(arg, nodeid, NODE_TYPE_DTC, 125 POR_DT_PMEVCNT(dtccntr & ~0x1), val >> 126 POR_DTM_PMEVCNT_CNTR_WIDTH); 127 128 shift = xpcntr * POR_DTM_PMEVCNT_CNTR_WIDTH; 129 val &= 0xffffUL; 130 pmu_cmn600_md8(arg, nodeid, NODE_TYPE_XP, POR_DTM_PMEVCNT, 131 0xffffUL << shift, val << shift); 132 } 133 134 #undef EVENCNTR 135 #undef ODDCNTR 136 137 /* 138 * read a pmc register 139 */ 140 static int 141 cmn600_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) 142 { 143 int counter, local_counter, nodeid; 144 struct cmn600_descr *desc; 145 void *arg; 146 147 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 148 ("[cmn600,%d] illegal CPU value %d", __LINE__, cpu)); 149 KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, 150 ri)); 151 152 counter = ri % CMN600_COUNTERS_N; 153 desc = cmn600desc(ri); 154 arg = desc->pd_rw_arg; 155 nodeid = pm->pm_md.pm_cmn600.pm_cmn600_nodeid; 156 local_counter = pm->pm_md.pm_cmn600.pm_cmn600_local_counter; 157 158 *v = cmn600_pmu_readcntr(arg, nodeid, local_counter, counter, 4); 159 PMCDBG3(MDP, REA, 2, "%s id=%d -> %jd", __func__, ri, *v); 160 161 return (0); 162 } 163 164 /* 165 * Write a pmc register. 166 */ 167 static int 168 cmn600_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v) 169 { 170 int counter, local_counter, nodeid; 171 struct cmn600_descr *desc; 172 void *arg; 173 174 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 175 ("[cmn600,%d] illegal CPU value %d", __LINE__, cpu)); 176 KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, 177 ri)); 178 179 counter = ri % CMN600_COUNTERS_N; 180 desc = cmn600desc(ri); 181 arg = desc->pd_rw_arg; 182 nodeid = pm->pm_md.pm_cmn600.pm_cmn600_nodeid; 183 local_counter = pm->pm_md.pm_cmn600.pm_cmn600_local_counter; 184 185 KASSERT(pm != NULL, 186 ("[cmn600,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, 187 cpu, ri)); 188 189 PMCDBG4(MDP, WRI, 1, "%s cpu=%d ri=%d v=%jx", __func__, cpu, ri, v); 190 191 cmn600_pmu_writecntr(arg, nodeid, local_counter, counter, 4, v); 192 return (0); 193 } 194 195 /* 196 * configure hardware pmc according to the configuration recorded in 197 * pmc 'pm'. 198 */ 199 static int 200 cmn600_config_pmc(int cpu, int ri, struct pmc *pm) 201 { 202 struct pmc_hw *phw; 203 204 PMCDBG4(MDP, CFG, 1, "%s cpu=%d ri=%d pm=%p", __func__, cpu, ri, pm); 205 206 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 207 ("[cmn600,%d] illegal CPU value %d", __LINE__, cpu)); 208 KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, 209 ri)); 210 211 phw = cmn600desc(ri)->pd_phw; 212 213 KASSERT(pm == NULL || phw->phw_pmc == NULL, 214 ("[cmn600,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 215 __LINE__, pm, phw->phw_pmc)); 216 217 phw->phw_pmc = pm; 218 return (0); 219 } 220 221 /* 222 * Retrieve a configured PMC pointer from hardware state. 223 */ 224 static int 225 cmn600_get_config(int cpu, int ri, struct pmc **ppm) 226 { 227 228 *ppm = cmn600desc(ri)->pd_phw->phw_pmc; 229 230 return (0); 231 } 232 233 #define CASE_DN_VER_EVT(n, id) case PMC_EV_CMN600_PMU_ ## n: { *event = id; \ 234 return (0); } 235 static int 236 cmn600_map_ev2event(int ev, int rev, int *node_type, uint8_t *event) 237 { 238 if (ev < PMC_EV_CMN600_PMU_dn_rxreq_dvmop || 239 ev > PMC_EV_CMN600_PMU_rni_rdb_ord) 240 return (EINVAL); 241 if (ev <= PMC_EV_CMN600_PMU_dn_rxreq_trk_full) { 242 *node_type = NODE_TYPE_DVM; 243 if (rev < 0x200) { 244 switch (ev) { 245 CASE_DN_VER_EVT(dn_rxreq_dvmop, 1); 246 CASE_DN_VER_EVT(dn_rxreq_dvmsync, 2); 247 CASE_DN_VER_EVT(dn_rxreq_dvmop_vmid_filtered, 3); 248 CASE_DN_VER_EVT(dn_rxreq_retried, 4); 249 CASE_DN_VER_EVT(dn_rxreq_trk_occupancy, 5); 250 } 251 } else { 252 switch (ev) { 253 CASE_DN_VER_EVT(dn_rxreq_tlbi_dvmop, 0x01); 254 CASE_DN_VER_EVT(dn_rxreq_bpi_dvmop, 0x02); 255 CASE_DN_VER_EVT(dn_rxreq_pici_dvmop, 0x03); 256 CASE_DN_VER_EVT(dn_rxreq_vivi_dvmop, 0x04); 257 CASE_DN_VER_EVT(dn_rxreq_dvmsync, 0x05); 258 CASE_DN_VER_EVT(dn_rxreq_dvmop_vmid_filtered, 0x06); 259 CASE_DN_VER_EVT(dn_rxreq_dvmop_other_filtered, 0x07); 260 CASE_DN_VER_EVT(dn_rxreq_retried, 0x08); 261 CASE_DN_VER_EVT(dn_rxreq_snp_sent, 0x09); 262 CASE_DN_VER_EVT(dn_rxreq_snp_stalled, 0x0a); 263 CASE_DN_VER_EVT(dn_rxreq_trk_full, 0x0b); 264 CASE_DN_VER_EVT(dn_rxreq_trk_occupancy, 0x0c); 265 } 266 } 267 return (EINVAL); 268 } else if (ev <= PMC_EV_CMN600_PMU_hnf_snp_fwded) { 269 *node_type = NODE_TYPE_HN_F; 270 *event = ev - PMC_EV_CMN600_PMU_hnf_cache_miss; 271 return (0); 272 } else if (ev <= PMC_EV_CMN600_PMU_hni_pcie_serialization) { 273 *node_type = NODE_TYPE_HN_I; 274 *event = ev - PMC_EV_CMN600_PMU_hni_rrt_rd_occ_cnt_ovfl; 275 return (0); 276 } else if (ev <= PMC_EV_CMN600_PMU_xp_partial_dat_flit) { 277 *node_type = NODE_TYPE_XP; 278 *event = ev - PMC_EV_CMN600_PMU_xp_txflit_valid; 279 return (0); 280 } else if (ev <= PMC_EV_CMN600_PMU_sbsx_txrsp_stall) { 281 *node_type = NODE_TYPE_SBSX; 282 *event = ev - PMC_EV_CMN600_PMU_sbsx_rd_req; 283 return (0); 284 } else if (ev <= PMC_EV_CMN600_PMU_rnd_rdb_ord) { 285 *node_type = NODE_TYPE_RN_D; 286 *event = ev - PMC_EV_CMN600_PMU_rnd_s0_rdata_beats; 287 return (0); 288 } else if (ev <= PMC_EV_CMN600_PMU_rni_rdb_ord) { 289 *node_type = NODE_TYPE_RN_I; 290 *event = ev - PMC_EV_CMN600_PMU_rni_s0_rdata_beats; 291 return (0); 292 } else if (ev <= PMC_EV_CMN600_PMU_cxha_snphaz_occ) { 293 *node_type = NODE_TYPE_CXHA; 294 *event = ev - PMC_EV_CMN600_PMU_cxha_rddatbyp; 295 return (0); 296 } else if (ev <= PMC_EV_CMN600_PMU_cxra_ext_dat_stall) { 297 *node_type = NODE_TYPE_CXRA; 298 *event = ev - PMC_EV_CMN600_PMU_cxra_req_trk_occ; 299 return (0); 300 } else if (ev <= PMC_EV_CMN600_PMU_cxla_avg_latency_form_tx_tlp) { 301 *node_type = NODE_TYPE_CXLA; 302 *event = ev - PMC_EV_CMN600_PMU_cxla_rx_tlp_link0; 303 return (0); 304 } 305 return (EINVAL); 306 } 307 308 /* 309 * Check if a given allocation is feasible. 310 */ 311 312 static int 313 cmn600_allocate_pmc(int cpu, int ri, struct pmc *pm, 314 const struct pmc_op_pmcallocate *a) 315 { 316 struct cmn600_descr *desc; 317 const struct pmc_descr *pd; 318 uint64_t caps __unused; 319 int local_counter, node_type; 320 enum pmc_event pe; 321 void *arg; 322 uint8_t e; 323 int err; 324 325 (void) cpu; 326 327 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 328 ("[cmn600,%d] illegal CPU value %d", __LINE__, cpu)); 329 KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, 330 ri)); 331 332 desc = cmn600desc(ri); 333 arg = desc->pd_rw_arg; 334 pd = &desc->pd_descr; 335 if (cmn600_pmcs[class_ri2unit(ri)].domain != pcpu_find(cpu)->pc_domain) 336 return (EINVAL); 337 338 /* check class match */ 339 if (pd->pd_class != a->pm_class) 340 return (EINVAL); 341 342 caps = pm->pm_caps; 343 344 PMCDBG3(MDP, ALL, 1, "%s ri=%d caps=0x%x", __func__, ri, caps); 345 346 pe = a->pm_ev; 347 err = cmn600_map_ev2event(pe, pmu_cmn600_rev(arg), &node_type, &e); 348 if (err != 0) 349 return (err); 350 err = pmu_cmn600_alloc_localpmc(arg, 351 a->pm_md.pm_cmn600.pma_cmn600_nodeid, node_type, &local_counter); 352 if (err != 0) 353 return (err); 354 355 pm->pm_md.pm_cmn600.pm_cmn600_config = 356 a->pm_md.pm_cmn600.pma_cmn600_config; 357 pm->pm_md.pm_cmn600.pm_cmn600_occupancy = 358 a->pm_md.pm_cmn600.pma_cmn600_occupancy; 359 desc->pd_nodeid = pm->pm_md.pm_cmn600.pm_cmn600_nodeid = 360 a->pm_md.pm_cmn600.pma_cmn600_nodeid; 361 desc->pd_node_type = pm->pm_md.pm_cmn600.pm_cmn600_node_type = 362 node_type; 363 pm->pm_md.pm_cmn600.pm_cmn600_event = e; 364 desc->pd_local_counter = pm->pm_md.pm_cmn600.pm_cmn600_local_counter = 365 local_counter; 366 367 return (0); 368 } 369 370 /* Release machine dependent state associated with a PMC. */ 371 372 static int 373 cmn600_release_pmc(int cpu, int ri, struct pmc *pmc) 374 { 375 struct cmn600_descr *desc; 376 struct pmc_hw *phw; 377 struct pmc *pm __diagused; 378 int err; 379 380 (void) pmc; 381 382 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 383 ("[cmn600,%d] illegal CPU value %d", __LINE__, cpu)); 384 KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, 385 ri)); 386 387 desc = cmn600desc(ri); 388 phw = desc->pd_phw; 389 pm = phw->phw_pmc; 390 err = pmu_cmn600_free_localpmc(desc->pd_rw_arg, desc->pd_nodeid, 391 desc->pd_node_type, desc->pd_local_counter); 392 if (err != 0) 393 return (err); 394 395 KASSERT(pm == NULL, ("[cmn600,%d] PHW pmc %p non-NULL", __LINE__, pm)); 396 397 return (0); 398 } 399 400 static inline uint64_t 401 cmn600_encode_source(int node_type, int counter, int port, int sub) 402 { 403 404 /* Calculate pmevcnt0_input_sel based on list in Table 3-794. */ 405 if (node_type == NODE_TYPE_XP) 406 return (0x4 | counter); 407 408 return (((port + 1) << 4) | (sub << 2) | counter); 409 } 410 411 /* 412 * start a PMC. 413 */ 414 415 static int 416 cmn600_start_pmc(int cpu, int ri, struct pmc *pm) 417 { 418 int counter, local_counter, node_type, shift; 419 uint64_t config, occupancy, source, xp_pmucfg; 420 struct cmn600_descr *desc; 421 uint8_t event, port, sub; 422 uint16_t nodeid; 423 void *arg; 424 425 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 426 ("[cmn600,%d] illegal CPU value %d", __LINE__, cpu)); 427 KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, 428 ri)); 429 430 counter = ri % CMN600_COUNTERS_N; 431 desc = cmn600desc(ri); 432 arg = desc->pd_rw_arg; 433 434 PMCDBG3(MDP, STA, 1, "%s cpu=%d ri=%d", __func__, cpu, ri); 435 436 config = pm->pm_md.pm_cmn600.pm_cmn600_config; 437 occupancy = pm->pm_md.pm_cmn600.pm_cmn600_occupancy; 438 node_type = pm->pm_md.pm_cmn600.pm_cmn600_node_type; 439 event = pm->pm_md.pm_cmn600.pm_cmn600_event; 440 nodeid = pm->pm_md.pm_cmn600.pm_cmn600_nodeid; 441 local_counter = pm->pm_md.pm_cmn600.pm_cmn600_local_counter; 442 port = (nodeid >> 2) & 1; 443 sub = nodeid & 3; 444 445 switch (node_type) { 446 case NODE_TYPE_DVM: 447 case NODE_TYPE_HN_F: 448 case NODE_TYPE_CXHA: 449 case NODE_TYPE_CXRA: 450 pmu_cmn600_md8(arg, nodeid, node_type, 451 CMN600_COMMON_PMU_EVENT_SEL, 452 CMN600_COMMON_PMU_EVENT_SEL_OCC_MASK, 453 occupancy << CMN600_COMMON_PMU_EVENT_SEL_OCC_SHIFT); 454 break; 455 case NODE_TYPE_XP: 456 /* Set PC and Interface.*/ 457 event |= config; 458 } 459 460 /* 461 * 5.5.1 Set up PMU counters 462 * 1. Ensure that the NIDEN input is asserted. HW side. */ 463 /* 2. Select event of target node for one of four outputs. */ 464 pmu_cmn600_md8(arg, nodeid, node_type, CMN600_COMMON_PMU_EVENT_SEL, 465 0xff << (local_counter * 8), 466 event << (local_counter * 8)); 467 468 xp_pmucfg = pmu_cmn600_rd8(arg, nodeid, NODE_TYPE_XP, 469 POR_DTM_PMU_CONFIG); 470 /* 471 * 3. configure XP to connect one of four target node outputs to local 472 * counter. 473 */ 474 source = cmn600_encode_source(node_type, local_counter, port, sub); 475 shift = (local_counter * POR_DTM_PMU_CONFIG_VCNT_INPUT_SEL_WIDTH) + 476 POR_DTM_PMU_CONFIG_VCNT_INPUT_SEL_SHIFT; 477 xp_pmucfg &= ~(0xffUL << shift); 478 xp_pmucfg |= source << shift; 479 480 /* 4. Pair with global counters A, B, C, ..., H. */ 481 shift = (local_counter * 4) + 16; 482 xp_pmucfg &= ~(0xfUL << shift); 483 xp_pmucfg |= counter << shift; 484 /* Enable pairing.*/ 485 xp_pmucfg |= 1 << (local_counter + 4); 486 487 /* 5. Combine local counters 0 with 1, 2 with 3 or all four. */ 488 xp_pmucfg &= ~0xeUL; 489 490 /* 6. Enable XP's PMU function. */ 491 xp_pmucfg |= POR_DTM_PMU_CONFIG_PMU_EN; 492 pmu_cmn600_wr8(arg, nodeid, NODE_TYPE_XP, POR_DTM_PMU_CONFIG, xp_pmucfg); 493 if (node_type == NODE_TYPE_CXLA) 494 pmu_cmn600_set8(arg, nodeid, NODE_TYPE_CXLA, 495 POR_CXG_RA_CFG_CTL, EN_CXLA_PMUCMD_PROP); 496 497 /* 7. Enable DTM. */ 498 pmu_cmn600_set8(arg, nodeid, NODE_TYPE_XP, POR_DTM_CONTROL, 499 POR_DTM_CONTROL_DTM_ENABLE); 500 501 /* 8. Reset grouping of global counters. Use 32 bits. */ 502 pmu_cmn600_clr8(arg, nodeid, NODE_TYPE_DTC, POR_DT_PMCR, 503 POR_DT_PMCR_CNTCFG_MASK); 504 505 /* 9. Enable DTC. */ 506 pmu_cmn600_set8(arg, nodeid, NODE_TYPE_DTC, POR_DT_DTC_CTL, 507 POR_DT_DTC_CTL_DT_EN); 508 509 /* 10. Enable Overflow Interrupt. */ 510 pmu_cmn600_set8(arg, nodeid, NODE_TYPE_DTC, POR_DT_PMCR, 511 POR_DT_PMCR_OVFL_INTR_EN); 512 513 /* 11. Run PMC. */ 514 pmu_cmn600_set8(arg, nodeid, NODE_TYPE_DTC, POR_DT_PMCR, 515 POR_DT_PMCR_PMU_EN); 516 517 return (0); 518 } 519 520 /* 521 * Stop a PMC. 522 */ 523 524 static int 525 cmn600_stop_pmc(int cpu, int ri, struct pmc *pm) 526 { 527 struct cmn600_descr *desc; 528 int local_counter; 529 uint64_t val; 530 531 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 532 ("[cmn600,%d] illegal CPU value %d", __LINE__, cpu)); 533 KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, 534 ri)); 535 536 desc = cmn600desc(ri); 537 538 PMCDBG2(MDP, STO, 1, "%s ri=%d", __func__, ri); 539 540 /* Disable pairing. */ 541 local_counter = pm->pm_md.pm_cmn600.pm_cmn600_local_counter; 542 pmu_cmn600_clr8(desc->pd_rw_arg, pm->pm_md.pm_cmn600.pm_cmn600_nodeid, 543 NODE_TYPE_XP, POR_DTM_PMU_CONFIG, (1 << (local_counter + 4))); 544 545 /* Shutdown XP's DTM function if no paired counters. */ 546 val = pmu_cmn600_rd8(desc->pd_rw_arg, 547 pm->pm_md.pm_cmn600.pm_cmn600_nodeid, NODE_TYPE_XP, 548 POR_DTM_PMU_CONFIG); 549 if ((val & 0xf0) == 0) 550 pmu_cmn600_clr8(desc->pd_rw_arg, 551 pm->pm_md.pm_cmn600.pm_cmn600_nodeid, NODE_TYPE_XP, 552 POR_DTM_PMU_CONFIG, POR_DTM_CONTROL_DTM_ENABLE); 553 554 return (0); 555 } 556 557 /* 558 * describe a PMC 559 */ 560 static int 561 cmn600_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 562 { 563 struct pmc_descr *pd; 564 struct pmc_hw *phw; 565 566 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 567 ("[cmn600,%d] illegal CPU %d", __LINE__, cpu)); 568 KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, 569 ri)); 570 571 phw = cmn600desc(ri)->pd_phw; 572 pd = &cmn600desc(ri)->pd_descr; 573 574 strlcpy(pi->pm_name, pd->pd_name, sizeof(pi->pm_name)); 575 pi->pm_class = pd->pd_class; 576 577 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 578 pi->pm_enabled = TRUE; 579 *ppmc = phw->phw_pmc; 580 } else { 581 pi->pm_enabled = FALSE; 582 *ppmc = NULL; 583 } 584 585 return (0); 586 } 587 588 /* 589 * processor dependent initialization. 590 */ 591 592 static int 593 cmn600_pcpu_init(struct pmc_mdep *md, int cpu) 594 { 595 int first_ri, n, npmc; 596 struct pmc_hw *phw; 597 struct pmc_cpu *pc; 598 int mdep_class; 599 600 mdep_class = PMC_MDEP_CLASS_INDEX_CMN600; 601 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 602 ("[cmn600,%d] insane cpu number %d", __LINE__, cpu)); 603 604 PMCDBG1(MDP, INI, 1, "cmn600-init cpu=%d", cpu); 605 606 /* 607 * Set the content of the hardware descriptors to a known 608 * state and initialize pointers in the MI per-cpu descriptor. 609 */ 610 611 pc = pmc_pcpu[cpu]; 612 first_ri = md->pmd_classdep[mdep_class].pcd_ri; 613 npmc = md->pmd_classdep[mdep_class].pcd_num; 614 615 for (n = 0; n < npmc; n++, phw++) { 616 phw = cmn600desc(n)->pd_phw; 617 phw->phw_state = PMC_PHW_CPU_TO_STATE(cpu) | 618 PMC_PHW_INDEX_TO_STATE(n); 619 /* Set enabled only if unit present. */ 620 if (cmn600_pmcs[class_ri2unit(n)].arg != NULL) 621 phw->phw_state |= PMC_PHW_FLAG_IS_ENABLED; 622 phw->phw_pmc = NULL; 623 pc->pc_hwpmcs[n + first_ri] = phw; 624 } 625 return (0); 626 } 627 628 /* 629 * processor dependent cleanup prior to the KLD 630 * being unloaded 631 */ 632 633 static int 634 cmn600_pcpu_fini(struct pmc_mdep *md, int cpu) 635 { 636 637 return (0); 638 } 639 640 static int 641 cmn600_pmu_intr(struct trapframe *tf, int unit, int i) 642 { 643 struct pmc_cpu *pc __diagused; 644 struct pmc_hw *phw; 645 struct pmc *pm; 646 int error, cpu, ri; 647 648 ri = i + unit * CMN600_COUNTERS_N; 649 cpu = curcpu; 650 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 651 ("[cmn600,%d] CPU %d out of range", __LINE__, cpu)); 652 pc = pmc_pcpu[cpu]; 653 KASSERT(pc != NULL, ("pc != NULL")); 654 655 phw = cmn600desc(ri)->pd_phw; 656 KASSERT(phw != NULL, ("phw != NULL")); 657 pm = phw->phw_pmc; 658 if (pm == NULL) 659 return (0); 660 661 if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 662 /* Always CPU0. */ 663 pm->pm_pcpu_state[0].pps_overflowcnt += 1; 664 return (0); 665 } 666 667 if (pm->pm_state != PMC_STATE_RUNNING) 668 return (0); 669 670 error = pmc_process_interrupt(PMC_HR, pm, tf); 671 if (error) 672 cmn600_stop_pmc(cpu, ri, pm); 673 674 /* Reload sampling count */ 675 cmn600_write_pmc(cpu, ri, pm, pm->pm_sc.pm_reloadcount); 676 677 return (0); 678 } 679 680 /* 681 * Initialize ourselves. 682 */ 683 static int 684 cmn600_init_pmc_units(void) 685 { 686 int i; 687 688 if (cmn600_units > 0) { /* Already initialized. */ 689 return (0); 690 } 691 692 cmn600_units = cmn600_pmc_nunits(); 693 if (cmn600_units == 0) 694 return (ENOENT); 695 696 for (i = 0; i < cmn600_units; i++) { 697 if (cmn600_pmc_getunit(i, &cmn600_pmcs[i].arg, 698 &cmn600_pmcs[i].domain) != 0) 699 cmn600_pmcs[i].arg = NULL; 700 } 701 return (0); 702 } 703 704 int 705 pmc_cmn600_nclasses(void) 706 { 707 708 if (cmn600_pmc_nunits() > 0) 709 return (1); 710 return (0); 711 } 712 713 int 714 pmc_cmn600_initialize(struct pmc_mdep *md) 715 { 716 struct pmc_classdep *pcd; 717 int i, npmc, unit; 718 719 cmn600_init_pmc_units(); 720 KASSERT(md != NULL, ("[cmn600,%d] md is NULL", __LINE__)); 721 KASSERT(cmn600_units < CMN600_UNIT_MAX, 722 ("[cmn600,%d] cmn600_units too big", __LINE__)); 723 724 PMCDBG0(MDP,INI,1, "cmn600-initialize"); 725 726 npmc = CMN600_COUNTERS_N * cmn600_units; 727 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_CMN600]; 728 729 pcd->pcd_caps = PMC_CAP_SYSTEM | PMC_CAP_READ | 730 PMC_CAP_WRITE | PMC_CAP_QUALIFIER | PMC_CAP_INTERRUPT | 731 PMC_CAP_DOMWIDE; 732 pcd->pcd_class = PMC_CLASS_CMN600_PMU; 733 pcd->pcd_num = npmc; 734 pcd->pcd_ri = md->pmd_npmc; 735 pcd->pcd_width = 48; 736 737 pcd->pcd_allocate_pmc = cmn600_allocate_pmc; 738 pcd->pcd_config_pmc = cmn600_config_pmc; 739 pcd->pcd_describe = cmn600_describe; 740 pcd->pcd_get_config = cmn600_get_config; 741 pcd->pcd_get_msr = NULL; 742 pcd->pcd_pcpu_fini = cmn600_pcpu_fini; 743 pcd->pcd_pcpu_init = cmn600_pcpu_init; 744 pcd->pcd_read_pmc = cmn600_read_pmc; 745 pcd->pcd_release_pmc = cmn600_release_pmc; 746 pcd->pcd_start_pmc = cmn600_start_pmc; 747 pcd->pcd_stop_pmc = cmn600_stop_pmc; 748 pcd->pcd_write_pmc = cmn600_write_pmc; 749 750 md->pmd_npmc += npmc; 751 cmn600_pmcdesc = malloc(sizeof(struct cmn600_descr *) * npmc * 752 CMN600_PMU_DEFAULT_UNITS_N, M_PMC, M_WAITOK|M_ZERO); 753 for (i = 0; i < npmc; i++) { 754 cmn600_pmcdesc[i] = malloc(sizeof(struct cmn600_descr), M_PMC, 755 M_WAITOK|M_ZERO); 756 757 unit = i / CMN600_COUNTERS_N; 758 KASSERT(unit >= 0, ("unit >= 0")); 759 KASSERT(cmn600_pmcs[unit].arg != NULL, ("arg != NULL")); 760 761 cmn600_pmcdesc[i]->pd_rw_arg = cmn600_pmcs[unit].arg; 762 cmn600_pmcdesc[i]->pd_descr.pd_class = 763 PMC_CLASS_CMN600_PMU; 764 cmn600_pmcdesc[i]->pd_descr.pd_caps = pcd->pcd_caps; 765 cmn600_pmcdesc[i]->pd_phw = (struct pmc_hw *)malloc( 766 sizeof(struct pmc_hw), M_PMC, M_WAITOK|M_ZERO); 767 snprintf(cmn600_pmcdesc[i]->pd_descr.pd_name, 63, 768 "CMN600_%d", i); 769 cmn600_pmu_intr_cb(cmn600_pmcs[unit].arg, cmn600_pmu_intr); 770 } 771 772 return (0); 773 } 774 775 void 776 pmc_cmn600_finalize(struct pmc_mdep *md) 777 { 778 struct pmc_classdep *pcd; 779 int i, npmc; 780 781 KASSERT(md->pmd_classdep[PMC_MDEP_CLASS_INDEX_CMN600].pcd_class == 782 PMC_CLASS_CMN600_PMU, ("[cmn600,%d] pmc class mismatch", 783 __LINE__)); 784 785 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_CMN600]; 786 787 npmc = pcd->pcd_num; 788 for (i = 0; i < npmc; i++) { 789 free(cmn600_pmcdesc[i]->pd_phw, M_PMC); 790 free(cmn600_pmcdesc[i], M_PMC); 791 } 792 free(cmn600_pmcdesc, M_PMC); 793 cmn600_pmcdesc = NULL; 794 } 795 796 MODULE_DEPEND(pmc, cmn600, 1, 1, 1); 797