1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2010 Fabien Thomas 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * Intel Uncore PMCs. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include <sys/param.h> 37 #include <sys/bus.h> 38 #include <sys/pmc.h> 39 #include <sys/pmckern.h> 40 #include <sys/systm.h> 41 42 #include <machine/intr_machdep.h> 43 #if (__FreeBSD_version >= 1100000) 44 #include <x86/apicvar.h> 45 #else 46 #include <machine/apicvar.h> 47 #endif 48 #include <machine/cpu.h> 49 #include <machine/cpufunc.h> 50 #include <machine/specialreg.h> 51 52 #define UCF_PMC_CAPS \ 53 (PMC_CAP_READ | PMC_CAP_WRITE) 54 55 #define UCP_PMC_CAPS \ 56 (PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ 57 PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE) 58 59 #define SELECTSEL(x) \ 60 (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ 61 UCP_CB0_EVSEL0 : UCP_EVSEL0) 62 63 #define SELECTOFF(x) \ 64 (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ 65 UCF_OFFSET_SB : UCF_OFFSET) 66 67 static enum pmc_cputype uncore_cputype; 68 69 struct uncore_cpu { 70 volatile uint32_t pc_resync; 71 volatile uint32_t pc_ucfctrl; /* Fixed function control. */ 72 volatile uint64_t pc_globalctrl; /* Global control register. */ 73 struct pmc_hw pc_uncorepmcs[]; 74 }; 75 76 static struct uncore_cpu **uncore_pcpu; 77 78 static uint64_t uncore_pmcmask; 79 80 static int uncore_ucf_ri; /* relative index of fixed counters */ 81 static int uncore_ucf_width; 82 static int uncore_ucf_npmc; 83 84 static int uncore_ucp_width; 85 static int uncore_ucp_npmc; 86 87 static int 88 uncore_pcpu_noop(struct pmc_mdep *md, int cpu) 89 { 90 (void) md; 91 (void) cpu; 92 return (0); 93 } 94 95 static int 96 uncore_pcpu_init(struct pmc_mdep *md, int cpu) 97 { 98 struct pmc_cpu *pc; 99 struct uncore_cpu *cc; 100 struct pmc_hw *phw; 101 int uncore_ri, n, npmc; 102 103 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 104 ("[ucf,%d] insane cpu number %d", __LINE__, cpu)); 105 106 PMCDBG1(MDP,INI,1,"uncore-init cpu=%d", cpu); 107 108 uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; 109 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; 110 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; 111 112 cc = malloc(sizeof(struct uncore_cpu) + npmc * sizeof(struct pmc_hw), 113 M_PMC, M_WAITOK | M_ZERO); 114 115 uncore_pcpu[cpu] = cc; 116 pc = pmc_pcpu[cpu]; 117 118 KASSERT(pc != NULL && cc != NULL, 119 ("[uncore,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu)); 120 121 for (n = 0, phw = cc->pc_uncorepmcs; n < npmc; n++, phw++) { 122 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 123 PMC_PHW_CPU_TO_STATE(cpu) | 124 PMC_PHW_INDEX_TO_STATE(n + uncore_ri); 125 phw->phw_pmc = NULL; 126 pc->pc_hwpmcs[n + uncore_ri] = phw; 127 } 128 129 return (0); 130 } 131 132 static int 133 uncore_pcpu_fini(struct pmc_mdep *md, int cpu) 134 { 135 int uncore_ri, n, npmc; 136 struct pmc_cpu *pc; 137 struct uncore_cpu *cc; 138 139 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 140 ("[uncore,%d] insane cpu number (%d)", __LINE__, cpu)); 141 142 PMCDBG1(MDP,INI,1,"uncore-pcpu-fini cpu=%d", cpu); 143 144 if ((cc = uncore_pcpu[cpu]) == NULL) 145 return (0); 146 147 uncore_pcpu[cpu] = NULL; 148 149 pc = pmc_pcpu[cpu]; 150 151 KASSERT(pc != NULL, ("[uncore,%d] NULL per-cpu %d state", __LINE__, 152 cpu)); 153 154 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; 155 uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; 156 157 for (n = 0; n < npmc; n++) 158 wrmsr(SELECTSEL(uncore_cputype) + n, 0); 159 160 wrmsr(UCF_CTRL, 0); 161 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; 162 163 for (n = 0; n < npmc; n++) 164 pc->pc_hwpmcs[n + uncore_ri] = NULL; 165 166 free(cc, M_PMC); 167 168 return (0); 169 } 170 171 /* 172 * Fixed function counters. 173 */ 174 175 static pmc_value_t 176 ucf_perfctr_value_to_reload_count(pmc_value_t v) 177 { 178 v &= (1ULL << uncore_ucf_width) - 1; 179 return (1ULL << uncore_ucf_width) - v; 180 } 181 182 static pmc_value_t 183 ucf_reload_count_to_perfctr_value(pmc_value_t rlc) 184 { 185 return (1ULL << uncore_ucf_width) - rlc; 186 } 187 188 static int 189 ucf_allocate_pmc(int cpu, int ri, struct pmc *pm, 190 const struct pmc_op_pmcallocate *a) 191 { 192 enum pmc_event ev; 193 uint32_t caps, flags; 194 195 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 196 ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); 197 198 PMCDBG2(MDP,ALL,1, "ucf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); 199 200 if (ri < 0 || ri > uncore_ucf_npmc) 201 return (EINVAL); 202 203 caps = a->pm_caps; 204 205 if (a->pm_class != PMC_CLASS_UCF || 206 (caps & UCF_PMC_CAPS) != caps) 207 return (EINVAL); 208 209 ev = pm->pm_event; 210 flags = UCF_EN; 211 212 pm->pm_md.pm_ucf.pm_ucf_ctrl = (flags << (ri * 4)); 213 214 PMCDBG1(MDP,ALL,2, "ucf-allocate config=0x%jx", 215 (uintmax_t) pm->pm_md.pm_ucf.pm_ucf_ctrl); 216 217 return (0); 218 } 219 220 static int 221 ucf_config_pmc(int cpu, int ri, struct pmc *pm) 222 { 223 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 224 ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); 225 226 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 227 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 228 229 PMCDBG3(MDP,CFG,1, "ucf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); 230 231 KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, 232 cpu)); 233 234 uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc = pm; 235 236 return (0); 237 } 238 239 static int 240 ucf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 241 { 242 int error; 243 struct pmc_hw *phw; 244 char ucf_name[PMC_NAME_MAX]; 245 246 phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri]; 247 248 (void) snprintf(ucf_name, sizeof(ucf_name), "UCF-%d", ri); 249 if ((error = copystr(ucf_name, pi->pm_name, PMC_NAME_MAX, 250 NULL)) != 0) 251 return (error); 252 253 pi->pm_class = PMC_CLASS_UCF; 254 255 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 256 pi->pm_enabled = TRUE; 257 *ppmc = phw->phw_pmc; 258 } else { 259 pi->pm_enabled = FALSE; 260 *ppmc = NULL; 261 } 262 263 return (0); 264 } 265 266 static int 267 ucf_get_config(int cpu, int ri, struct pmc **ppm) 268 { 269 *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; 270 271 return (0); 272 } 273 274 static int 275 ucf_read_pmc(int cpu, int ri, pmc_value_t *v) 276 { 277 struct pmc *pm; 278 pmc_value_t tmp; 279 280 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 281 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 282 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 283 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 284 285 pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; 286 287 KASSERT(pm, 288 ("[uncore,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu, 289 ri, ri + uncore_ucf_ri)); 290 291 tmp = rdmsr(UCF_CTR0 + ri); 292 293 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 294 *v = ucf_perfctr_value_to_reload_count(tmp); 295 else 296 *v = tmp; 297 298 PMCDBG3(MDP,REA,1, "ucf-read cpu=%d ri=%d -> v=%jx", cpu, ri, *v); 299 300 return (0); 301 } 302 303 static int 304 ucf_release_pmc(int cpu, int ri, struct pmc *pmc) 305 { 306 PMCDBG3(MDP,REL,1, "ucf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); 307 308 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 309 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 310 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 311 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 312 313 KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc == NULL, 314 ("[uncore,%d] PHW pmc non-NULL", __LINE__)); 315 316 return (0); 317 } 318 319 static int 320 ucf_start_pmc(int cpu, int ri) 321 { 322 struct pmc *pm; 323 struct uncore_cpu *ucfc; 324 325 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 326 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 327 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 328 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 329 330 PMCDBG2(MDP,STA,1,"ucf-start cpu=%d ri=%d", cpu, ri); 331 332 ucfc = uncore_pcpu[cpu]; 333 pm = ucfc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; 334 335 ucfc->pc_ucfctrl |= pm->pm_md.pm_ucf.pm_ucf_ctrl; 336 337 wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); 338 339 do { 340 ucfc->pc_resync = 0; 341 ucfc->pc_globalctrl |= (1ULL << (ri + SELECTOFF(uncore_cputype))); 342 wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); 343 } while (ucfc->pc_resync != 0); 344 345 PMCDBG4(MDP,STA,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", 346 ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), 347 ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); 348 349 return (0); 350 } 351 352 static int 353 ucf_stop_pmc(int cpu, int ri) 354 { 355 uint32_t fc; 356 struct uncore_cpu *ucfc; 357 358 PMCDBG2(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri); 359 360 ucfc = uncore_pcpu[cpu]; 361 362 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 363 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 364 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 365 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 366 367 fc = (UCF_MASK << (ri * 4)); 368 369 ucfc->pc_ucfctrl &= ~fc; 370 371 PMCDBG1(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl); 372 wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); 373 374 do { 375 ucfc->pc_resync = 0; 376 ucfc->pc_globalctrl &= ~(1ULL << (ri + SELECTOFF(uncore_cputype))); 377 wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); 378 } while (ucfc->pc_resync != 0); 379 380 PMCDBG4(MDP,STO,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", 381 ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), 382 ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); 383 384 return (0); 385 } 386 387 static int 388 ucf_write_pmc(int cpu, int ri, pmc_value_t v) 389 { 390 struct uncore_cpu *cc; 391 struct pmc *pm; 392 393 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 394 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 395 KASSERT(ri >= 0 && ri < uncore_ucf_npmc, 396 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 397 398 cc = uncore_pcpu[cpu]; 399 pm = cc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; 400 401 KASSERT(pm, 402 ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 403 404 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 405 v = ucf_reload_count_to_perfctr_value(v); 406 407 wrmsr(UCF_CTRL, 0); /* Turn off fixed counters */ 408 wrmsr(UCF_CTR0 + ri, v); 409 wrmsr(UCF_CTRL, cc->pc_ucfctrl); 410 411 PMCDBG4(MDP,WRI,1, "ucf-write cpu=%d ri=%d v=%jx ucfctrl=%jx ", 412 cpu, ri, v, (uintmax_t) rdmsr(UCF_CTRL)); 413 414 return (0); 415 } 416 417 418 static void 419 ucf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) 420 { 421 struct pmc_classdep *pcd; 422 423 KASSERT(md != NULL, ("[ucf,%d] md is NULL", __LINE__)); 424 425 PMCDBG0(MDP,INI,1, "ucf-initialize"); 426 427 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF]; 428 429 pcd->pcd_caps = UCF_PMC_CAPS; 430 pcd->pcd_class = PMC_CLASS_UCF; 431 pcd->pcd_num = npmc; 432 pcd->pcd_ri = md->pmd_npmc; 433 pcd->pcd_width = pmcwidth; 434 435 pcd->pcd_allocate_pmc = ucf_allocate_pmc; 436 pcd->pcd_config_pmc = ucf_config_pmc; 437 pcd->pcd_describe = ucf_describe; 438 pcd->pcd_get_config = ucf_get_config; 439 pcd->pcd_get_msr = NULL; 440 pcd->pcd_pcpu_fini = uncore_pcpu_noop; 441 pcd->pcd_pcpu_init = uncore_pcpu_noop; 442 pcd->pcd_read_pmc = ucf_read_pmc; 443 pcd->pcd_release_pmc = ucf_release_pmc; 444 pcd->pcd_start_pmc = ucf_start_pmc; 445 pcd->pcd_stop_pmc = ucf_stop_pmc; 446 pcd->pcd_write_pmc = ucf_write_pmc; 447 448 md->pmd_npmc += npmc; 449 } 450 451 /* 452 * Intel programmable PMCs. 453 */ 454 455 /* 456 * Event descriptor tables. 457 * 458 * For each event id, we track: 459 * 460 * 1. The CPUs that the event is valid for. 461 * 462 * 2. If the event uses a fixed UMASK, the value of the umask field. 463 * If the event doesn't use a fixed UMASK, a mask of legal bits 464 * to check against. 465 */ 466 467 struct ucp_event_descr { 468 enum pmc_event ucp_ev; 469 unsigned char ucp_evcode; 470 unsigned char ucp_umask; 471 unsigned char ucp_flags; 472 }; 473 474 #define UCP_F_I7 (1 << 0) /* CPU: Core i7 */ 475 #define UCP_F_WM (1 << 1) /* CPU: Westmere */ 476 #define UCP_F_SB (1 << 2) /* CPU: Sandy Bridge */ 477 #define UCP_F_HW (1 << 3) /* CPU: Haswell */ 478 #define UCP_F_FM (1 << 4) /* Fixed mask */ 479 480 #define UCP_F_ALLCPUS \ 481 (UCP_F_I7 | UCP_F_WM) 482 483 #define UCP_F_CMASK 0xFF000000 484 485 static pmc_value_t 486 ucp_perfctr_value_to_reload_count(pmc_value_t v) 487 { 488 v &= (1ULL << uncore_ucp_width) - 1; 489 return (1ULL << uncore_ucp_width) - v; 490 } 491 492 static pmc_value_t 493 ucp_reload_count_to_perfctr_value(pmc_value_t rlc) 494 { 495 return (1ULL << uncore_ucp_width) - rlc; 496 } 497 498 /* 499 * Counter specific event information for Sandybridge and Haswell 500 */ 501 static int 502 ucp_event_sb_hw_ok_on_counter(uint8_t ev, int ri) 503 { 504 uint32_t mask; 505 506 switch (ev) { 507 /* 508 * Events valid only on counter 0. 509 */ 510 case 0x80: 511 case 0x83: 512 mask = (1 << 0); 513 break; 514 515 default: 516 mask = ~0; /* Any row index is ok. */ 517 } 518 519 return (mask & (1 << ri)); 520 } 521 522 static int 523 ucp_allocate_pmc(int cpu, int ri, struct pmc *pm, 524 const struct pmc_op_pmcallocate *a) 525 { 526 uint8_t ev; 527 uint32_t caps; 528 const struct pmc_md_ucp_op_pmcallocate *ucp; 529 530 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 531 ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); 532 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 533 ("[uncore,%d] illegal row-index value %d", __LINE__, ri)); 534 535 /* check requested capabilities */ 536 caps = a->pm_caps; 537 if ((UCP_PMC_CAPS & caps) != caps) 538 return (EPERM); 539 540 ucp = &a->pm_md.pm_ucp; 541 ev = UCP_EVSEL(ucp->pm_ucp_config); 542 switch (uncore_cputype) { 543 case PMC_CPU_INTEL_HASWELL: 544 case PMC_CPU_INTEL_SANDYBRIDGE: 545 if (ucp_event_sb_hw_ok_on_counter(ev, ri) == 0) 546 return (EINVAL); 547 break; 548 default: 549 break; 550 } 551 552 pm->pm_md.pm_ucp.pm_ucp_evsel = ucp->pm_ucp_config | UCP_EN; 553 554 return (0); 555 } 556 557 static int 558 ucp_config_pmc(int cpu, int ri, struct pmc *pm) 559 { 560 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 561 ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); 562 563 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 564 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 565 566 PMCDBG3(MDP,CFG,1, "ucp-config cpu=%d ri=%d pm=%p", cpu, ri, pm); 567 568 KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, 569 cpu)); 570 571 uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc = pm; 572 573 return (0); 574 } 575 576 static int 577 ucp_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 578 { 579 int error; 580 struct pmc_hw *phw; 581 char ucp_name[PMC_NAME_MAX]; 582 583 phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri]; 584 585 (void) snprintf(ucp_name, sizeof(ucp_name), "UCP-%d", ri); 586 if ((error = copystr(ucp_name, pi->pm_name, PMC_NAME_MAX, 587 NULL)) != 0) 588 return (error); 589 590 pi->pm_class = PMC_CLASS_UCP; 591 592 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 593 pi->pm_enabled = TRUE; 594 *ppmc = phw->phw_pmc; 595 } else { 596 pi->pm_enabled = FALSE; 597 *ppmc = NULL; 598 } 599 600 return (0); 601 } 602 603 static int 604 ucp_get_config(int cpu, int ri, struct pmc **ppm) 605 { 606 *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; 607 608 return (0); 609 } 610 611 static int 612 ucp_read_pmc(int cpu, int ri, pmc_value_t *v) 613 { 614 struct pmc *pm; 615 pmc_value_t tmp; 616 617 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 618 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 619 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 620 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 621 622 pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; 623 624 KASSERT(pm, 625 ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, 626 ri)); 627 628 tmp = rdmsr(UCP_PMC0 + ri); 629 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 630 *v = ucp_perfctr_value_to_reload_count(tmp); 631 else 632 *v = tmp; 633 634 PMCDBG4(MDP,REA,1, "ucp-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, 635 ri, *v); 636 637 return (0); 638 } 639 640 static int 641 ucp_release_pmc(int cpu, int ri, struct pmc *pm) 642 { 643 (void) pm; 644 645 PMCDBG3(MDP,REL,1, "ucp-release cpu=%d ri=%d pm=%p", cpu, ri, 646 pm); 647 648 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 649 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 650 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 651 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 652 653 KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc 654 == NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__)); 655 656 return (0); 657 } 658 659 static int 660 ucp_start_pmc(int cpu, int ri) 661 { 662 struct pmc *pm; 663 uint32_t evsel; 664 struct uncore_cpu *cc; 665 666 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 667 ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); 668 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 669 ("[uncore,%d] illegal row-index %d", __LINE__, ri)); 670 671 cc = uncore_pcpu[cpu]; 672 pm = cc->pc_uncorepmcs[ri].phw_pmc; 673 674 KASSERT(pm, 675 ("[uncore,%d] starting cpu%d,ri%d with no pmc configured", 676 __LINE__, cpu, ri)); 677 678 PMCDBG2(MDP,STA,1, "ucp-start cpu=%d ri=%d", cpu, ri); 679 680 evsel = pm->pm_md.pm_ucp.pm_ucp_evsel; 681 682 PMCDBG4(MDP,STA,2, 683 "ucp-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", 684 cpu, ri, SELECTSEL(uncore_cputype) + ri, evsel); 685 686 /* Event specific configuration. */ 687 switch (pm->pm_event) { 688 case PMC_EV_UCP_EVENT_0CH_04H_E: 689 case PMC_EV_UCP_EVENT_0CH_08H_E: 690 wrmsr(MSR_GQ_SNOOP_MESF,0x2); 691 break; 692 case PMC_EV_UCP_EVENT_0CH_04H_F: 693 case PMC_EV_UCP_EVENT_0CH_08H_F: 694 wrmsr(MSR_GQ_SNOOP_MESF,0x8); 695 break; 696 case PMC_EV_UCP_EVENT_0CH_04H_M: 697 case PMC_EV_UCP_EVENT_0CH_08H_M: 698 wrmsr(MSR_GQ_SNOOP_MESF,0x1); 699 break; 700 case PMC_EV_UCP_EVENT_0CH_04H_S: 701 case PMC_EV_UCP_EVENT_0CH_08H_S: 702 wrmsr(MSR_GQ_SNOOP_MESF,0x4); 703 break; 704 default: 705 break; 706 } 707 wrmsr(SELECTSEL(uncore_cputype) + ri, evsel); 708 709 do { 710 cc->pc_resync = 0; 711 cc->pc_globalctrl |= (1ULL << ri); 712 wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl); 713 } while (cc->pc_resync != 0); 714 715 return (0); 716 } 717 718 static int 719 ucp_stop_pmc(int cpu, int ri) 720 { 721 struct pmc *pm; 722 struct uncore_cpu *cc; 723 724 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 725 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 726 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 727 ("[uncore,%d] illegal row index %d", __LINE__, ri)); 728 729 cc = uncore_pcpu[cpu]; 730 pm = cc->pc_uncorepmcs[ri].phw_pmc; 731 732 KASSERT(pm, 733 ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, 734 cpu, ri)); 735 736 PMCDBG2(MDP,STO,1, "ucp-stop cpu=%d ri=%d", cpu, ri); 737 738 /* stop hw. */ 739 wrmsr(SELECTSEL(uncore_cputype) + ri, 0); 740 741 do { 742 cc->pc_resync = 0; 743 cc->pc_globalctrl &= ~(1ULL << ri); 744 wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl); 745 } while (cc->pc_resync != 0); 746 747 return (0); 748 } 749 750 static int 751 ucp_write_pmc(int cpu, int ri, pmc_value_t v) 752 { 753 struct pmc *pm; 754 struct uncore_cpu *cc; 755 756 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 757 ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); 758 KASSERT(ri >= 0 && ri < uncore_ucp_npmc, 759 ("[uncore,%d] illegal row index %d", __LINE__, ri)); 760 761 cc = uncore_pcpu[cpu]; 762 pm = cc->pc_uncorepmcs[ri].phw_pmc; 763 764 KASSERT(pm, 765 ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, 766 cpu, ri)); 767 768 PMCDBG4(MDP,WRI,1, "ucp-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, 769 UCP_PMC0 + ri, v); 770 771 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 772 v = ucp_reload_count_to_perfctr_value(v); 773 774 /* 775 * Write the new value to the counter. The counter will be in 776 * a stopped state when the pcd_write() entry point is called. 777 */ 778 779 wrmsr(UCP_PMC0 + ri, v); 780 781 return (0); 782 } 783 784 785 static void 786 ucp_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) 787 { 788 struct pmc_classdep *pcd; 789 790 KASSERT(md != NULL, ("[ucp,%d] md is NULL", __LINE__)); 791 792 PMCDBG0(MDP,INI,1, "ucp-initialize"); 793 794 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP]; 795 796 pcd->pcd_caps = UCP_PMC_CAPS; 797 pcd->pcd_class = PMC_CLASS_UCP; 798 pcd->pcd_num = npmc; 799 pcd->pcd_ri = md->pmd_npmc; 800 pcd->pcd_width = pmcwidth; 801 802 pcd->pcd_allocate_pmc = ucp_allocate_pmc; 803 pcd->pcd_config_pmc = ucp_config_pmc; 804 pcd->pcd_describe = ucp_describe; 805 pcd->pcd_get_config = ucp_get_config; 806 pcd->pcd_get_msr = NULL; 807 pcd->pcd_pcpu_fini = uncore_pcpu_fini; 808 pcd->pcd_pcpu_init = uncore_pcpu_init; 809 pcd->pcd_read_pmc = ucp_read_pmc; 810 pcd->pcd_release_pmc = ucp_release_pmc; 811 pcd->pcd_start_pmc = ucp_start_pmc; 812 pcd->pcd_stop_pmc = ucp_stop_pmc; 813 pcd->pcd_write_pmc = ucp_write_pmc; 814 815 md->pmd_npmc += npmc; 816 } 817 818 int 819 pmc_uncore_initialize(struct pmc_mdep *md, int maxcpu) 820 { 821 uncore_cputype = md->pmd_cputype; 822 uncore_pmcmask = 0; 823 824 /* 825 * Initialize programmable counters. 826 */ 827 828 uncore_ucp_npmc = 8; 829 uncore_ucp_width = 48; 830 831 uncore_pmcmask |= ((1ULL << uncore_ucp_npmc) - 1); 832 833 ucp_initialize(md, maxcpu, uncore_ucp_npmc, uncore_ucp_width); 834 835 /* 836 * Initialize fixed function counters, if present. 837 */ 838 uncore_ucf_ri = uncore_ucp_npmc; 839 uncore_ucf_npmc = 1; 840 uncore_ucf_width = 48; 841 842 ucf_initialize(md, maxcpu, uncore_ucf_npmc, uncore_ucf_width); 843 uncore_pmcmask |= ((1ULL << uncore_ucf_npmc) - 1) << SELECTOFF(uncore_cputype); 844 845 PMCDBG2(MDP,INI,1,"uncore-init pmcmask=0x%jx ucfri=%d", uncore_pmcmask, 846 uncore_ucf_ri); 847 848 uncore_pcpu = malloc(sizeof(*uncore_pcpu) * maxcpu, M_PMC, 849 M_ZERO | M_WAITOK); 850 851 return (0); 852 } 853 854 void 855 pmc_uncore_finalize(struct pmc_mdep *md) 856 { 857 PMCDBG0(MDP,INI,1, "uncore-finalize"); 858 859 free(uncore_pcpu, M_PMC); 860 uncore_pcpu = NULL; 861 } 862