1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2012 Fabien Thomas 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/pmc.h> 34 #include <sys/pmckern.h> 35 #include <sys/systm.h> 36 #include <sys/mutex.h> 37 38 #include <machine/cpu.h> 39 #include <machine/cpufunc.h> 40 41 #include "hwpmc_soft.h" 42 43 /* 44 * Software PMC support. 45 */ 46 47 #define SOFT_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \ 48 PMC_CAP_USER | PMC_CAP_SYSTEM) 49 50 struct soft_descr { 51 struct pmc_descr pm_descr; /* "base class" */ 52 }; 53 54 static struct soft_descr soft_pmcdesc[SOFT_NPMCS] = 55 { 56 #define SOFT_PMCDESCR(N) \ 57 { \ 58 .pm_descr = \ 59 { \ 60 .pd_name = #N, \ 61 .pd_class = PMC_CLASS_SOFT, \ 62 .pd_caps = SOFT_CAPS, \ 63 .pd_width = 64 \ 64 }, \ 65 } 66 67 SOFT_PMCDESCR(SOFT0), 68 SOFT_PMCDESCR(SOFT1), 69 SOFT_PMCDESCR(SOFT2), 70 SOFT_PMCDESCR(SOFT3), 71 SOFT_PMCDESCR(SOFT4), 72 SOFT_PMCDESCR(SOFT5), 73 SOFT_PMCDESCR(SOFT6), 74 SOFT_PMCDESCR(SOFT7), 75 SOFT_PMCDESCR(SOFT8), 76 SOFT_PMCDESCR(SOFT9), 77 SOFT_PMCDESCR(SOFT10), 78 SOFT_PMCDESCR(SOFT11), 79 SOFT_PMCDESCR(SOFT12), 80 SOFT_PMCDESCR(SOFT13), 81 SOFT_PMCDESCR(SOFT14), 82 SOFT_PMCDESCR(SOFT15) 83 }; 84 85 /* 86 * Per-CPU data structure. 87 */ 88 89 struct soft_cpu { 90 struct pmc_hw soft_hw[SOFT_NPMCS]; 91 pmc_value_t soft_values[SOFT_NPMCS]; 92 }; 93 94 95 static struct soft_cpu **soft_pcpu; 96 97 static int 98 soft_allocate_pmc(int cpu, int ri, struct pmc *pm, 99 const struct pmc_op_pmcallocate *a) 100 { 101 enum pmc_event ev; 102 struct pmc_soft *ps; 103 104 (void) cpu; 105 106 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 107 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 108 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 109 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 110 111 if (a->pm_class != PMC_CLASS_SOFT) 112 return (EINVAL); 113 114 if ((pm->pm_caps & SOFT_CAPS) == 0) 115 return (EINVAL); 116 117 if ((pm->pm_caps & ~SOFT_CAPS) != 0) 118 return (EPERM); 119 120 ev = pm->pm_event; 121 if ((int)ev < PMC_EV_SOFT_FIRST || (int)ev > PMC_EV_SOFT_LAST) 122 return (EINVAL); 123 124 /* Check if event is registered. */ 125 ps = pmc_soft_ev_acquire(ev); 126 if (ps == NULL) 127 return (EINVAL); 128 pmc_soft_ev_release(ps); 129 /* Module unload is protected by pmc SX lock. */ 130 if (ps->ps_alloc != NULL) 131 ps->ps_alloc(); 132 133 return (0); 134 } 135 136 static int 137 soft_config_pmc(int cpu, int ri, struct pmc *pm) 138 { 139 struct pmc_hw *phw; 140 141 PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 142 143 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 144 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 145 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 146 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 147 148 phw = &soft_pcpu[cpu]->soft_hw[ri]; 149 150 KASSERT(pm == NULL || phw->phw_pmc == NULL, 151 ("[soft,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, 152 pm, phw->phw_pmc)); 153 154 phw->phw_pmc = pm; 155 156 return (0); 157 } 158 159 static int 160 soft_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 161 { 162 int error; 163 size_t copied; 164 const struct soft_descr *pd; 165 struct pmc_hw *phw; 166 167 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 168 ("[soft,%d] illegal CPU %d", __LINE__, cpu)); 169 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 170 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 171 172 phw = &soft_pcpu[cpu]->soft_hw[ri]; 173 pd = &soft_pmcdesc[ri]; 174 175 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 176 PMC_NAME_MAX, &copied)) != 0) 177 return (error); 178 179 pi->pm_class = pd->pm_descr.pd_class; 180 181 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 182 pi->pm_enabled = TRUE; 183 *ppmc = phw->phw_pmc; 184 } else { 185 pi->pm_enabled = FALSE; 186 *ppmc = NULL; 187 } 188 189 return (0); 190 } 191 192 static int 193 soft_get_config(int cpu, int ri, struct pmc **ppm) 194 { 195 (void) ri; 196 197 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 198 ("[soft,%d] illegal CPU %d", __LINE__, cpu)); 199 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 200 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 201 202 *ppm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; 203 return (0); 204 } 205 206 static int 207 soft_pcpu_fini(struct pmc_mdep *md, int cpu) 208 { 209 int ri; 210 struct pmc_cpu *pc; 211 212 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 213 ("[soft,%d] illegal cpu %d", __LINE__, cpu)); 214 KASSERT(soft_pcpu[cpu] != NULL, ("[soft,%d] null pcpu", __LINE__)); 215 216 free(soft_pcpu[cpu], M_PMC); 217 soft_pcpu[cpu] = NULL; 218 219 ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; 220 221 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 222 ("[soft,%d] ri=%d", __LINE__, ri)); 223 224 pc = pmc_pcpu[cpu]; 225 pc->pc_hwpmcs[ri] = NULL; 226 227 return (0); 228 } 229 230 static int 231 soft_pcpu_init(struct pmc_mdep *md, int cpu) 232 { 233 int first_ri, n; 234 struct pmc_cpu *pc; 235 struct soft_cpu *soft_pc; 236 struct pmc_hw *phw; 237 238 239 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 240 ("[soft,%d] illegal cpu %d", __LINE__, cpu)); 241 KASSERT(soft_pcpu, ("[soft,%d] null pcpu", __LINE__)); 242 KASSERT(soft_pcpu[cpu] == NULL, ("[soft,%d] non-null per-cpu", 243 __LINE__)); 244 245 soft_pc = malloc(sizeof(struct soft_cpu), M_PMC, M_WAITOK|M_ZERO); 246 pc = pmc_pcpu[cpu]; 247 248 KASSERT(pc != NULL, ("[soft,%d] cpu %d null per-cpu", __LINE__, cpu)); 249 250 soft_pcpu[cpu] = soft_pc; 251 phw = soft_pc->soft_hw; 252 first_ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; 253 254 for (n = 0; n < SOFT_NPMCS; n++, phw++) { 255 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 256 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 257 phw->phw_pmc = NULL; 258 pc->pc_hwpmcs[n + first_ri] = phw; 259 } 260 261 return (0); 262 } 263 264 static int 265 soft_read_pmc(int cpu, int ri, pmc_value_t *v) 266 { 267 struct pmc *pm; 268 const struct pmc_hw *phw; 269 270 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 271 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 272 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 273 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 274 275 phw = &soft_pcpu[cpu]->soft_hw[ri]; 276 pm = phw->phw_pmc; 277 278 KASSERT(pm != NULL, 279 ("[soft,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri)); 280 281 PMCDBG1(MDP,REA,1,"soft-read id=%d", ri); 282 283 *v = soft_pcpu[cpu]->soft_values[ri]; 284 285 return (0); 286 } 287 288 static int 289 soft_write_pmc(int cpu, int ri, pmc_value_t v) 290 { 291 struct pmc *pm; 292 const struct soft_descr *pd; 293 294 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 295 ("[soft,%d] illegal cpu value %d", __LINE__, cpu)); 296 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 297 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 298 299 pm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; 300 pd = &soft_pmcdesc[ri]; 301 302 KASSERT(pm, 303 ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 304 305 PMCDBG3(MDP,WRI,1, "soft-write cpu=%d ri=%d v=%jx", cpu, ri, v); 306 307 soft_pcpu[cpu]->soft_values[ri] = v; 308 309 return (0); 310 } 311 312 static int 313 soft_release_pmc(int cpu, int ri, struct pmc *pmc) 314 { 315 struct pmc_hw *phw; 316 enum pmc_event ev; 317 struct pmc_soft *ps; 318 319 (void) pmc; 320 321 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 322 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 323 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 324 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 325 326 phw = &soft_pcpu[cpu]->soft_hw[ri]; 327 328 KASSERT(phw->phw_pmc == NULL, 329 ("[soft,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 330 331 ev = pmc->pm_event; 332 333 /* Check if event is registered. */ 334 ps = pmc_soft_ev_acquire(ev); 335 KASSERT(ps != NULL, 336 ("[soft,%d] unregistered event %d", __LINE__, ev)); 337 pmc_soft_ev_release(ps); 338 /* Module unload is protected by pmc SX lock. */ 339 if (ps->ps_release != NULL) 340 ps->ps_release(); 341 return (0); 342 } 343 344 static int 345 soft_start_pmc(int cpu, int ri) 346 { 347 struct pmc *pm; 348 struct soft_cpu *pc; 349 struct pmc_soft *ps; 350 351 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 352 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 353 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 354 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 355 356 pc = soft_pcpu[cpu]; 357 pm = pc->soft_hw[ri].phw_pmc; 358 359 KASSERT(pm, 360 ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 361 362 ps = pmc_soft_ev_acquire(pm->pm_event); 363 if (ps == NULL) 364 return (EINVAL); 365 atomic_add_int(&ps->ps_running, 1); 366 pmc_soft_ev_release(ps); 367 368 return (0); 369 } 370 371 static int 372 soft_stop_pmc(int cpu, int ri) 373 { 374 struct pmc *pm; 375 struct soft_cpu *pc; 376 struct pmc_soft *ps; 377 378 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 379 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 380 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 381 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 382 383 pc = soft_pcpu[cpu]; 384 pm = pc->soft_hw[ri].phw_pmc; 385 386 KASSERT(pm, 387 ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 388 389 ps = pmc_soft_ev_acquire(pm->pm_event); 390 /* event unregistered ? */ 391 if (ps != NULL) { 392 atomic_subtract_int(&ps->ps_running, 1); 393 pmc_soft_ev_release(ps); 394 } 395 396 return (0); 397 } 398 399 int 400 pmc_soft_intr(struct pmckern_soft *ks) 401 { 402 struct pmc *pm; 403 struct soft_cpu *pc; 404 int ri, processed, error, user_mode; 405 406 KASSERT(ks->pm_cpu >= 0 && ks->pm_cpu < pmc_cpu_max(), 407 ("[soft,%d] CPU %d out of range", __LINE__, ks->pm_cpu)); 408 409 processed = 0; 410 pc = soft_pcpu[ks->pm_cpu]; 411 412 for (ri = 0; ri < SOFT_NPMCS; ri++) { 413 414 pm = pc->soft_hw[ri].phw_pmc; 415 if (pm == NULL || 416 pm->pm_state != PMC_STATE_RUNNING || 417 pm->pm_event != ks->pm_ev) { 418 continue; 419 } 420 421 processed = 1; 422 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 423 if ((pc->soft_values[ri]--) <= 0) 424 pc->soft_values[ri] += pm->pm_sc.pm_reloadcount; 425 else 426 continue; 427 user_mode = TRAPF_USERMODE(ks->pm_tf); 428 error = pmc_process_interrupt(PMC_SR, pm, ks->pm_tf); 429 if (error) { 430 soft_stop_pmc(ks->pm_cpu, ri); 431 continue; 432 } 433 434 if (user_mode) { 435 /* If in user mode setup AST to process 436 * callchain out of interrupt context. 437 */ 438 curthread->td_flags |= TDF_ASTPENDING; 439 } 440 } else 441 pc->soft_values[ri]++; 442 } 443 if (processed) 444 counter_u64_add(pmc_stats.pm_intr_processed, 1); 445 else 446 counter_u64_add(pmc_stats.pm_intr_ignored, 1); 447 448 return (processed); 449 } 450 451 void 452 pmc_soft_initialize(struct pmc_mdep *md) 453 { 454 struct pmc_classdep *pcd; 455 456 /* Add SOFT PMCs. */ 457 soft_pcpu = malloc(sizeof(struct soft_cpu *) * pmc_cpu_max(), M_PMC, 458 M_ZERO|M_WAITOK); 459 460 pcd = &md->pmd_classdep[PMC_CLASS_INDEX_SOFT]; 461 462 pcd->pcd_caps = SOFT_CAPS; 463 pcd->pcd_class = PMC_CLASS_SOFT; 464 pcd->pcd_num = SOFT_NPMCS; 465 pcd->pcd_ri = md->pmd_npmc; 466 pcd->pcd_width = 64; 467 468 pcd->pcd_allocate_pmc = soft_allocate_pmc; 469 pcd->pcd_config_pmc = soft_config_pmc; 470 pcd->pcd_describe = soft_describe; 471 pcd->pcd_get_config = soft_get_config; 472 pcd->pcd_get_msr = NULL; 473 pcd->pcd_pcpu_init = soft_pcpu_init; 474 pcd->pcd_pcpu_fini = soft_pcpu_fini; 475 pcd->pcd_read_pmc = soft_read_pmc; 476 pcd->pcd_write_pmc = soft_write_pmc; 477 pcd->pcd_release_pmc = soft_release_pmc; 478 pcd->pcd_start_pmc = soft_start_pmc; 479 pcd->pcd_stop_pmc = soft_stop_pmc; 480 481 md->pmd_npmc += SOFT_NPMCS; 482 } 483 484 void 485 pmc_soft_finalize(struct pmc_mdep *md) 486 { 487 #ifdef INVARIANTS 488 int i, ncpus; 489 490 ncpus = pmc_cpu_max(); 491 for (i = 0; i < ncpus; i++) 492 KASSERT(soft_pcpu[i] == NULL, ("[soft,%d] non-null pcpu cpu %d", 493 __LINE__, i)); 494 495 KASSERT(md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_class == 496 PMC_CLASS_SOFT, ("[soft,%d] class mismatch", __LINE__)); 497 #endif 498 free(soft_pcpu, M_PMC); 499 soft_pcpu = NULL; 500 } 501