1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2012 Fabien Thomas 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/pmc.h> 34 #include <sys/pmckern.h> 35 #include <sys/systm.h> 36 #include <sys/mutex.h> 37 38 #include <machine/cpu.h> 39 #include <machine/cpufunc.h> 40 41 #include "hwpmc_soft.h" 42 43 /* 44 * Software PMC support. 45 */ 46 47 #define SOFT_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \ 48 PMC_CAP_USER | PMC_CAP_SYSTEM) 49 50 struct soft_descr { 51 struct pmc_descr pm_descr; /* "base class" */ 52 }; 53 54 static struct soft_descr soft_pmcdesc[SOFT_NPMCS] = 55 { 56 #define SOFT_PMCDESCR(N) \ 57 { \ 58 .pm_descr = \ 59 { \ 60 .pd_name = #N, \ 61 .pd_class = PMC_CLASS_SOFT, \ 62 .pd_caps = SOFT_CAPS, \ 63 .pd_width = 64 \ 64 }, \ 65 } 66 67 SOFT_PMCDESCR(SOFT0), 68 SOFT_PMCDESCR(SOFT1), 69 SOFT_PMCDESCR(SOFT2), 70 SOFT_PMCDESCR(SOFT3), 71 SOFT_PMCDESCR(SOFT4), 72 SOFT_PMCDESCR(SOFT5), 73 SOFT_PMCDESCR(SOFT6), 74 SOFT_PMCDESCR(SOFT7), 75 SOFT_PMCDESCR(SOFT8), 76 SOFT_PMCDESCR(SOFT9), 77 SOFT_PMCDESCR(SOFT10), 78 SOFT_PMCDESCR(SOFT11), 79 SOFT_PMCDESCR(SOFT12), 80 SOFT_PMCDESCR(SOFT13), 81 SOFT_PMCDESCR(SOFT14), 82 SOFT_PMCDESCR(SOFT15) 83 }; 84 85 /* 86 * Per-CPU data structure. 87 */ 88 89 struct soft_cpu { 90 struct pmc_hw soft_hw[SOFT_NPMCS]; 91 pmc_value_t soft_values[SOFT_NPMCS]; 92 }; 93 94 95 static struct soft_cpu **soft_pcpu; 96 97 static int 98 soft_allocate_pmc(int cpu, int ri, struct pmc *pm, 99 const struct pmc_op_pmcallocate *a) 100 { 101 enum pmc_event ev; 102 struct pmc_soft *ps; 103 104 (void) cpu; 105 106 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 107 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 108 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 109 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 110 111 if (a->pm_class != PMC_CLASS_SOFT) 112 return (EINVAL); 113 114 if ((pm->pm_caps & SOFT_CAPS) == 0) 115 return (EINVAL); 116 117 if ((pm->pm_caps & ~SOFT_CAPS) != 0) 118 return (EPERM); 119 120 ev = pm->pm_event; 121 if ((int)ev < PMC_EV_SOFT_FIRST || (int)ev > PMC_EV_SOFT_LAST) 122 return (EINVAL); 123 124 /* Check if event is registered. */ 125 ps = pmc_soft_ev_acquire(ev); 126 if (ps == NULL) 127 return (EINVAL); 128 pmc_soft_ev_release(ps); 129 /* Module unload is protected by pmc SX lock. */ 130 if (ps->ps_alloc != NULL) 131 ps->ps_alloc(); 132 133 return (0); 134 } 135 136 static int 137 soft_config_pmc(int cpu, int ri, struct pmc *pm) 138 { 139 struct pmc_hw *phw; 140 141 PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 142 143 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 144 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 145 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 146 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 147 148 phw = &soft_pcpu[cpu]->soft_hw[ri]; 149 150 KASSERT(pm == NULL || phw->phw_pmc == NULL, 151 ("[soft,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, 152 pm, phw->phw_pmc)); 153 154 phw->phw_pmc = pm; 155 156 return (0); 157 } 158 159 static int 160 soft_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 161 { 162 int error; 163 size_t copied; 164 const struct soft_descr *pd; 165 struct pmc_hw *phw; 166 167 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 168 ("[soft,%d] illegal CPU %d", __LINE__, cpu)); 169 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 170 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 171 172 phw = &soft_pcpu[cpu]->soft_hw[ri]; 173 pd = &soft_pmcdesc[ri]; 174 175 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 176 PMC_NAME_MAX, &copied)) != 0) 177 return (error); 178 179 pi->pm_class = pd->pm_descr.pd_class; 180 181 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 182 pi->pm_enabled = TRUE; 183 *ppmc = phw->phw_pmc; 184 } else { 185 pi->pm_enabled = FALSE; 186 *ppmc = NULL; 187 } 188 189 return (0); 190 } 191 192 static int 193 soft_get_config(int cpu, int ri, struct pmc **ppm) 194 { 195 (void) ri; 196 197 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 198 ("[soft,%d] illegal CPU %d", __LINE__, cpu)); 199 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 200 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 201 202 *ppm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; 203 return (0); 204 } 205 206 static int 207 soft_pcpu_fini(struct pmc_mdep *md, int cpu) 208 { 209 int ri; 210 struct pmc_cpu *pc; 211 212 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 213 ("[soft,%d] illegal cpu %d", __LINE__, cpu)); 214 KASSERT(soft_pcpu[cpu] != NULL, ("[soft,%d] null pcpu", __LINE__)); 215 216 free(soft_pcpu[cpu], M_PMC); 217 soft_pcpu[cpu] = NULL; 218 219 ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; 220 221 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 222 ("[soft,%d] ri=%d", __LINE__, ri)); 223 224 pc = pmc_pcpu[cpu]; 225 pc->pc_hwpmcs[ri] = NULL; 226 227 return (0); 228 } 229 230 static int 231 soft_pcpu_init(struct pmc_mdep *md, int cpu) 232 { 233 int first_ri, n; 234 struct pmc_cpu *pc; 235 struct soft_cpu *soft_pc; 236 struct pmc_hw *phw; 237 238 239 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 240 ("[soft,%d] illegal cpu %d", __LINE__, cpu)); 241 KASSERT(soft_pcpu, ("[soft,%d] null pcpu", __LINE__)); 242 KASSERT(soft_pcpu[cpu] == NULL, ("[soft,%d] non-null per-cpu", 243 __LINE__)); 244 245 soft_pc = malloc(sizeof(struct soft_cpu), M_PMC, M_WAITOK|M_ZERO); 246 pc = pmc_pcpu[cpu]; 247 248 KASSERT(pc != NULL, ("[soft,%d] cpu %d null per-cpu", __LINE__, cpu)); 249 250 soft_pcpu[cpu] = soft_pc; 251 phw = soft_pc->soft_hw; 252 first_ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; 253 254 for (n = 0; n < SOFT_NPMCS; n++, phw++) { 255 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 256 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 257 phw->phw_pmc = NULL; 258 pc->pc_hwpmcs[n + first_ri] = phw; 259 } 260 261 return (0); 262 } 263 264 static int 265 soft_read_pmc(int cpu, int ri, pmc_value_t *v) 266 { 267 struct pmc *pm __diagused; 268 const struct pmc_hw *phw; 269 270 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 271 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 272 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 273 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 274 275 phw = &soft_pcpu[cpu]->soft_hw[ri]; 276 pm = phw->phw_pmc; 277 278 KASSERT(pm != NULL, 279 ("[soft,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri)); 280 281 PMCDBG1(MDP,REA,1,"soft-read id=%d", ri); 282 283 *v = soft_pcpu[cpu]->soft_values[ri]; 284 285 return (0); 286 } 287 288 static int 289 soft_write_pmc(int cpu, int ri, pmc_value_t v) 290 { 291 struct pmc *pm __diagused; 292 293 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 294 ("[soft,%d] illegal cpu value %d", __LINE__, cpu)); 295 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 296 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 297 298 pm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; 299 300 KASSERT(pm, 301 ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 302 303 PMCDBG3(MDP,WRI,1, "soft-write cpu=%d ri=%d v=%jx", cpu, ri, v); 304 305 soft_pcpu[cpu]->soft_values[ri] = v; 306 307 return (0); 308 } 309 310 static int 311 soft_release_pmc(int cpu, int ri, struct pmc *pmc) 312 { 313 struct pmc_hw *phw __diagused; 314 enum pmc_event ev; 315 struct pmc_soft *ps; 316 317 (void) pmc; 318 319 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 320 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 321 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 322 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 323 324 phw = &soft_pcpu[cpu]->soft_hw[ri]; 325 326 KASSERT(phw->phw_pmc == NULL, 327 ("[soft,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 328 329 ev = pmc->pm_event; 330 331 /* Check if event is registered. */ 332 ps = pmc_soft_ev_acquire(ev); 333 KASSERT(ps != NULL, 334 ("[soft,%d] unregistered event %d", __LINE__, ev)); 335 pmc_soft_ev_release(ps); 336 /* Module unload is protected by pmc SX lock. */ 337 if (ps->ps_release != NULL) 338 ps->ps_release(); 339 return (0); 340 } 341 342 static int 343 soft_start_pmc(int cpu, int ri) 344 { 345 struct pmc *pm; 346 struct soft_cpu *pc; 347 struct pmc_soft *ps; 348 349 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 350 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 351 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 352 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 353 354 pc = soft_pcpu[cpu]; 355 pm = pc->soft_hw[ri].phw_pmc; 356 357 KASSERT(pm, 358 ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 359 360 ps = pmc_soft_ev_acquire(pm->pm_event); 361 if (ps == NULL) 362 return (EINVAL); 363 atomic_add_int(&ps->ps_running, 1); 364 pmc_soft_ev_release(ps); 365 366 return (0); 367 } 368 369 static int 370 soft_stop_pmc(int cpu, int ri) 371 { 372 struct pmc *pm; 373 struct soft_cpu *pc; 374 struct pmc_soft *ps; 375 376 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 377 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 378 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 379 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 380 381 pc = soft_pcpu[cpu]; 382 pm = pc->soft_hw[ri].phw_pmc; 383 384 KASSERT(pm, 385 ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 386 387 ps = pmc_soft_ev_acquire(pm->pm_event); 388 /* event unregistered ? */ 389 if (ps != NULL) { 390 atomic_subtract_int(&ps->ps_running, 1); 391 pmc_soft_ev_release(ps); 392 } 393 394 return (0); 395 } 396 397 int 398 pmc_soft_intr(struct pmckern_soft *ks) 399 { 400 struct pmc *pm; 401 struct soft_cpu *pc; 402 int ri, processed, error, user_mode; 403 404 KASSERT(ks->pm_cpu >= 0 && ks->pm_cpu < pmc_cpu_max(), 405 ("[soft,%d] CPU %d out of range", __LINE__, ks->pm_cpu)); 406 407 processed = 0; 408 pc = soft_pcpu[ks->pm_cpu]; 409 410 for (ri = 0; ri < SOFT_NPMCS; ri++) { 411 412 pm = pc->soft_hw[ri].phw_pmc; 413 if (pm == NULL || 414 pm->pm_state != PMC_STATE_RUNNING || 415 pm->pm_event != ks->pm_ev) { 416 continue; 417 } 418 419 processed = 1; 420 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 421 if ((pc->soft_values[ri]--) <= 0) 422 pc->soft_values[ri] += pm->pm_sc.pm_reloadcount; 423 else 424 continue; 425 user_mode = TRAPF_USERMODE(ks->pm_tf); 426 error = pmc_process_interrupt(PMC_SR, pm, ks->pm_tf); 427 if (error) { 428 soft_stop_pmc(ks->pm_cpu, ri); 429 continue; 430 } 431 432 if (user_mode) { 433 /* 434 * If in user mode setup AST to process 435 * callchain out of interrupt context. 436 */ 437 ast_sched(curthread, TDA_HWPMC); 438 } 439 } else 440 pc->soft_values[ri]++; 441 } 442 if (processed) 443 counter_u64_add(pmc_stats.pm_intr_processed, 1); 444 else 445 counter_u64_add(pmc_stats.pm_intr_ignored, 1); 446 447 return (processed); 448 } 449 450 static void 451 ast_hwpmc(struct thread *td, int tda __unused) 452 { 453 /* Handle Software PMC callchain capture. */ 454 if (PMC_IS_PENDING_CALLCHAIN(td)) 455 PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_USER_CALLCHAIN_SOFT, 456 (void *)td->td_frame); 457 } 458 459 void 460 pmc_soft_initialize(struct pmc_mdep *md) 461 { 462 struct pmc_classdep *pcd; 463 464 /* Add SOFT PMCs. */ 465 soft_pcpu = malloc(sizeof(struct soft_cpu *) * pmc_cpu_max(), M_PMC, 466 M_ZERO|M_WAITOK); 467 468 pcd = &md->pmd_classdep[PMC_CLASS_INDEX_SOFT]; 469 470 pcd->pcd_caps = SOFT_CAPS; 471 pcd->pcd_class = PMC_CLASS_SOFT; 472 pcd->pcd_num = SOFT_NPMCS; 473 pcd->pcd_ri = md->pmd_npmc; 474 pcd->pcd_width = 64; 475 476 pcd->pcd_allocate_pmc = soft_allocate_pmc; 477 pcd->pcd_config_pmc = soft_config_pmc; 478 pcd->pcd_describe = soft_describe; 479 pcd->pcd_get_config = soft_get_config; 480 pcd->pcd_get_msr = NULL; 481 pcd->pcd_pcpu_init = soft_pcpu_init; 482 pcd->pcd_pcpu_fini = soft_pcpu_fini; 483 pcd->pcd_read_pmc = soft_read_pmc; 484 pcd->pcd_write_pmc = soft_write_pmc; 485 pcd->pcd_release_pmc = soft_release_pmc; 486 pcd->pcd_start_pmc = soft_start_pmc; 487 pcd->pcd_stop_pmc = soft_stop_pmc; 488 489 md->pmd_npmc += SOFT_NPMCS; 490 491 ast_register(TDA_HWPMC, ASTR_UNCOND, 0, ast_hwpmc); 492 } 493 494 void 495 pmc_soft_finalize(struct pmc_mdep *md) 496 { 497 #ifdef INVARIANTS 498 int i, ncpus; 499 500 ncpus = pmc_cpu_max(); 501 for (i = 0; i < ncpus; i++) 502 KASSERT(soft_pcpu[i] == NULL, ("[soft,%d] non-null pcpu cpu %d", 503 __LINE__, i)); 504 505 KASSERT(md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_class == 506 PMC_CLASS_SOFT, ("[soft,%d] class mismatch", __LINE__)); 507 #endif 508 ast_deregister(TDA_HWPMC); 509 free(soft_pcpu, M_PMC); 510 soft_pcpu = NULL; 511 } 512