1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2014, The Linux Foundation. All rights reserved. 4 */ 5 6 #include <linux/bitops.h> 7 #include <linux/kernel.h> 8 #include <linux/moduleparam.h> 9 #include <linux/init.h> 10 #include <linux/types.h> 11 #include <linux/device.h> 12 #include <linux/io.h> 13 #include <linux/err.h> 14 #include <linux/fs.h> 15 #include <linux/slab.h> 16 #include <linux/delay.h> 17 #include <linux/smp.h> 18 #include <linux/sysfs.h> 19 #include <linux/stat.h> 20 #include <linux/clk.h> 21 #include <linux/cpu.h> 22 #include <linux/cpu_pm.h> 23 #include <linux/coresight.h> 24 #include <linux/coresight-pmu.h> 25 #include <linux/pm_wakeup.h> 26 #include <linux/amba/bus.h> 27 #include <linux/seq_file.h> 28 #include <linux/uaccess.h> 29 #include <linux/perf_event.h> 30 #include <linux/platform_device.h> 31 #include <linux/pm_runtime.h> 32 #include <linux/property.h> 33 34 #include <asm/barrier.h> 35 #include <asm/sections.h> 36 #include <asm/sysreg.h> 37 #include <asm/local.h> 38 #include <asm/virt.h> 39 40 #include "coresight-etm4x.h" 41 #include "coresight-etm-perf.h" 42 #include "coresight-etm4x-cfg.h" 43 #include "coresight-self-hosted-trace.h" 44 #include "coresight-syscfg.h" 45 #include "coresight-trace-id.h" 46 47 static int boot_enable; 48 module_param(boot_enable, int, 0444); 49 MODULE_PARM_DESC(boot_enable, "Enable tracing on boot"); 50 51 #define PARAM_PM_SAVE_FIRMWARE 0 /* save self-hosted state as per firmware */ 52 #define PARAM_PM_SAVE_NEVER 1 /* never save any state */ 53 #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */ 54 55 static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE; 56 module_param(pm_save_enable, int, 0444); 57 MODULE_PARM_DESC(pm_save_enable, 58 "Save/restore state on power down: 1 = never, 2 = self-hosted"); 59 60 static struct etmv4_drvdata *etmdrvdata[NR_CPUS]; 61 static void etm4_set_default_config(struct etmv4_config *config); 62 static int etm4_set_event_filters(struct etmv4_drvdata *drvdata, 63 struct perf_event *event); 64 static u64 etm4_get_access_type(struct etmv4_config *config); 65 66 static enum cpuhp_state hp_online; 67 68 struct etm4_init_arg { 69 unsigned int pid; 70 struct device *dev; 71 struct csdev_access *csa; 72 }; 73 74 static DEFINE_PER_CPU(struct etm4_init_arg *, delayed_probe); 75 static int etm4_probe_cpu(unsigned int cpu); 76 77 /* 78 * Check if TRCSSPCICRn(i) is implemented for a given instance. 79 * 80 * TRCSSPCICRn is implemented only if : 81 * TRCSSPCICR<n> is present only if all of the following are true: 82 * TRCIDR4.NUMSSCC > n. 83 * TRCIDR4.NUMPC > 0b0000 . 84 * TRCSSCSR<n>.PC == 0b1 85 */ 86 static inline bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n) 87 { 88 return (n < drvdata->nr_ss_cmp) && 89 drvdata->nr_pe && 90 (drvdata->config.ss_status[n] & TRCSSCSRn_PC); 91 } 92 93 u64 etm4x_sysreg_read(u32 offset, bool _relaxed, bool _64bit) 94 { 95 u64 res = 0; 96 97 switch (offset) { 98 ETM4x_READ_SYSREG_CASES(res) 99 default : 100 pr_warn_ratelimited("etm4x: trying to read unsupported register @%x\n", 101 offset); 102 } 103 104 if (!_relaxed) 105 __io_ar(res); /* Imitate the !relaxed I/O helpers */ 106 107 return res; 108 } 109 110 void etm4x_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit) 111 { 112 if (!_relaxed) 113 __io_bw(); /* Imitate the !relaxed I/O helpers */ 114 if (!_64bit) 115 val &= GENMASK(31, 0); 116 117 switch (offset) { 118 ETM4x_WRITE_SYSREG_CASES(val) 119 default : 120 pr_warn_ratelimited("etm4x: trying to write to unsupported register @%x\n", 121 offset); 122 } 123 } 124 125 static u64 ete_sysreg_read(u32 offset, bool _relaxed, bool _64bit) 126 { 127 u64 res = 0; 128 129 switch (offset) { 130 ETE_READ_CASES(res) 131 default : 132 pr_warn_ratelimited("ete: trying to read unsupported register @%x\n", 133 offset); 134 } 135 136 if (!_relaxed) 137 __io_ar(res); /* Imitate the !relaxed I/O helpers */ 138 139 return res; 140 } 141 142 static void ete_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit) 143 { 144 if (!_relaxed) 145 __io_bw(); /* Imitate the !relaxed I/O helpers */ 146 if (!_64bit) 147 val &= GENMASK(31, 0); 148 149 switch (offset) { 150 ETE_WRITE_CASES(val) 151 default : 152 pr_warn_ratelimited("ete: trying to write to unsupported register @%x\n", 153 offset); 154 } 155 } 156 157 static void etm_detect_os_lock(struct etmv4_drvdata *drvdata, 158 struct csdev_access *csa) 159 { 160 u32 oslsr = etm4x_relaxed_read32(csa, TRCOSLSR); 161 162 drvdata->os_lock_model = ETM_OSLSR_OSLM(oslsr); 163 } 164 165 static void etm_write_os_lock(struct etmv4_drvdata *drvdata, 166 struct csdev_access *csa, u32 val) 167 { 168 val = !!val; 169 170 switch (drvdata->os_lock_model) { 171 case ETM_OSLOCK_PRESENT: 172 etm4x_relaxed_write32(csa, val, TRCOSLAR); 173 break; 174 case ETM_OSLOCK_PE: 175 write_sysreg_s(val, SYS_OSLAR_EL1); 176 break; 177 default: 178 pr_warn_once("CPU%d: Unsupported Trace OSLock model: %x\n", 179 smp_processor_id(), drvdata->os_lock_model); 180 fallthrough; 181 case ETM_OSLOCK_NI: 182 return; 183 } 184 isb(); 185 } 186 187 static inline void etm4_os_unlock_csa(struct etmv4_drvdata *drvdata, 188 struct csdev_access *csa) 189 { 190 WARN_ON(drvdata->cpu != smp_processor_id()); 191 192 /* Writing 0 to OS Lock unlocks the trace unit registers */ 193 etm_write_os_lock(drvdata, csa, 0x0); 194 drvdata->os_unlock = true; 195 } 196 197 static void etm4_os_unlock(struct etmv4_drvdata *drvdata) 198 { 199 if (!WARN_ON(!drvdata->csdev)) 200 etm4_os_unlock_csa(drvdata, &drvdata->csdev->access); 201 } 202 203 static void etm4_os_lock(struct etmv4_drvdata *drvdata) 204 { 205 if (WARN_ON(!drvdata->csdev)) 206 return; 207 /* Writing 0x1 to OS Lock locks the trace registers */ 208 etm_write_os_lock(drvdata, &drvdata->csdev->access, 0x1); 209 drvdata->os_unlock = false; 210 } 211 212 static void etm4_cs_lock(struct etmv4_drvdata *drvdata, 213 struct csdev_access *csa) 214 { 215 /* Software Lock is only accessible via memory mapped interface */ 216 if (csa->io_mem) 217 CS_LOCK(csa->base); 218 } 219 220 static void etm4_cs_unlock(struct etmv4_drvdata *drvdata, 221 struct csdev_access *csa) 222 { 223 if (csa->io_mem) 224 CS_UNLOCK(csa->base); 225 } 226 227 static int etm4_cpu_id(struct coresight_device *csdev) 228 { 229 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 230 231 return drvdata->cpu; 232 } 233 234 int etm4_read_alloc_trace_id(struct etmv4_drvdata *drvdata) 235 { 236 int trace_id; 237 238 /* 239 * This will allocate a trace ID to the cpu, 240 * or return the one currently allocated. 241 * The trace id function has its own lock 242 */ 243 trace_id = coresight_trace_id_get_cpu_id(drvdata->cpu); 244 if (IS_VALID_CS_TRACE_ID(trace_id)) 245 drvdata->trcid = (u8)trace_id; 246 else 247 dev_err(&drvdata->csdev->dev, 248 "Failed to allocate trace ID for %s on CPU%d\n", 249 dev_name(&drvdata->csdev->dev), drvdata->cpu); 250 return trace_id; 251 } 252 253 void etm4_release_trace_id(struct etmv4_drvdata *drvdata) 254 { 255 coresight_trace_id_put_cpu_id(drvdata->cpu); 256 } 257 258 struct etm4_enable_arg { 259 struct etmv4_drvdata *drvdata; 260 int rc; 261 }; 262 263 /* 264 * etm4x_prohibit_trace - Prohibit the CPU from tracing at all ELs. 265 * When the CPU supports FEAT_TRF, we could move the ETM to a trace 266 * prohibited state by filtering the Exception levels via TRFCR_EL1. 267 */ 268 static void etm4x_prohibit_trace(struct etmv4_drvdata *drvdata) 269 { 270 /* If the CPU doesn't support FEAT_TRF, nothing to do */ 271 if (!drvdata->trfcr) 272 return; 273 cpu_prohibit_trace(); 274 } 275 276 /* 277 * etm4x_allow_trace - Allow CPU tracing in the respective ELs, 278 * as configured by the drvdata->config.mode for the current 279 * session. Even though we have TRCVICTLR bits to filter the 280 * trace in the ELs, it doesn't prevent the ETM from generating 281 * a packet (e.g, TraceInfo) that might contain the addresses from 282 * the excluded levels. Thus we use the additional controls provided 283 * via the Trace Filtering controls (FEAT_TRF) to make sure no trace 284 * is generated for the excluded ELs. 285 */ 286 static void etm4x_allow_trace(struct etmv4_drvdata *drvdata) 287 { 288 u64 trfcr = drvdata->trfcr; 289 290 /* If the CPU doesn't support FEAT_TRF, nothing to do */ 291 if (!trfcr) 292 return; 293 294 if (drvdata->config.mode & ETM_MODE_EXCL_KERN) 295 trfcr &= ~TRFCR_ELx_ExTRE; 296 if (drvdata->config.mode & ETM_MODE_EXCL_USER) 297 trfcr &= ~TRFCR_ELx_E0TRE; 298 299 write_trfcr(trfcr); 300 } 301 302 #ifdef CONFIG_ETM4X_IMPDEF_FEATURE 303 304 #define HISI_HIP08_AMBA_ID 0x000b6d01 305 #define ETM4_AMBA_MASK 0xfffff 306 #define HISI_HIP08_CORE_COMMIT_MASK 0x3000 307 #define HISI_HIP08_CORE_COMMIT_SHIFT 12 308 #define HISI_HIP08_CORE_COMMIT_FULL 0b00 309 #define HISI_HIP08_CORE_COMMIT_LVL_1 0b01 310 #define HISI_HIP08_CORE_COMMIT_REG sys_reg(3, 1, 15, 2, 5) 311 312 struct etm4_arch_features { 313 void (*arch_callback)(bool enable); 314 }; 315 316 static bool etm4_hisi_match_pid(unsigned int id) 317 { 318 return (id & ETM4_AMBA_MASK) == HISI_HIP08_AMBA_ID; 319 } 320 321 static void etm4_hisi_config_core_commit(bool enable) 322 { 323 u8 commit = enable ? HISI_HIP08_CORE_COMMIT_LVL_1 : 324 HISI_HIP08_CORE_COMMIT_FULL; 325 u64 val; 326 327 /* 328 * bit 12 and 13 of HISI_HIP08_CORE_COMMIT_REG are used together 329 * to set core-commit, 2'b00 means cpu is at full speed, 2'b01, 330 * 2'b10, 2'b11 mean reduce pipeline speed, and 2'b01 means level-1 331 * speed(minimun value). So bit 12 and 13 should be cleared together. 332 */ 333 val = read_sysreg_s(HISI_HIP08_CORE_COMMIT_REG); 334 val &= ~HISI_HIP08_CORE_COMMIT_MASK; 335 val |= commit << HISI_HIP08_CORE_COMMIT_SHIFT; 336 write_sysreg_s(val, HISI_HIP08_CORE_COMMIT_REG); 337 } 338 339 static struct etm4_arch_features etm4_features[] = { 340 [ETM4_IMPDEF_HISI_CORE_COMMIT] = { 341 .arch_callback = etm4_hisi_config_core_commit, 342 }, 343 {}, 344 }; 345 346 static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata) 347 { 348 struct etm4_arch_features *ftr; 349 int bit; 350 351 for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) { 352 ftr = &etm4_features[bit]; 353 354 if (ftr->arch_callback) 355 ftr->arch_callback(true); 356 } 357 } 358 359 static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata) 360 { 361 struct etm4_arch_features *ftr; 362 int bit; 363 364 for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) { 365 ftr = &etm4_features[bit]; 366 367 if (ftr->arch_callback) 368 ftr->arch_callback(false); 369 } 370 } 371 372 static void etm4_check_arch_features(struct etmv4_drvdata *drvdata, 373 unsigned int id) 374 { 375 if (etm4_hisi_match_pid(id)) 376 set_bit(ETM4_IMPDEF_HISI_CORE_COMMIT, drvdata->arch_features); 377 } 378 #else 379 static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata) 380 { 381 } 382 383 static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata) 384 { 385 } 386 387 static void etm4_check_arch_features(struct etmv4_drvdata *drvdata, 388 unsigned int id) 389 { 390 } 391 #endif /* CONFIG_ETM4X_IMPDEF_FEATURE */ 392 393 static int etm4_enable_hw(struct etmv4_drvdata *drvdata) 394 { 395 int i, rc; 396 struct etmv4_config *config = &drvdata->config; 397 struct coresight_device *csdev = drvdata->csdev; 398 struct device *etm_dev = &csdev->dev; 399 struct csdev_access *csa = &csdev->access; 400 401 402 etm4_cs_unlock(drvdata, csa); 403 etm4_enable_arch_specific(drvdata); 404 405 etm4_os_unlock(drvdata); 406 407 rc = coresight_claim_device_unlocked(csdev); 408 if (rc) 409 goto done; 410 411 /* Disable the trace unit before programming trace registers */ 412 etm4x_relaxed_write32(csa, 0, TRCPRGCTLR); 413 414 /* 415 * If we use system instructions, we need to synchronize the 416 * write to the TRCPRGCTLR, before accessing the TRCSTATR. 417 * See ARM IHI0064F, section 418 * "4.3.7 Synchronization of register updates" 419 */ 420 if (!csa->io_mem) 421 isb(); 422 423 /* wait for TRCSTATR.IDLE to go up */ 424 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) 425 dev_err(etm_dev, 426 "timeout while waiting for Idle Trace Status\n"); 427 if (drvdata->nr_pe) 428 etm4x_relaxed_write32(csa, config->pe_sel, TRCPROCSELR); 429 etm4x_relaxed_write32(csa, config->cfg, TRCCONFIGR); 430 /* nothing specific implemented */ 431 etm4x_relaxed_write32(csa, 0x0, TRCAUXCTLR); 432 etm4x_relaxed_write32(csa, config->eventctrl0, TRCEVENTCTL0R); 433 etm4x_relaxed_write32(csa, config->eventctrl1, TRCEVENTCTL1R); 434 if (drvdata->stallctl) 435 etm4x_relaxed_write32(csa, config->stall_ctrl, TRCSTALLCTLR); 436 etm4x_relaxed_write32(csa, config->ts_ctrl, TRCTSCTLR); 437 etm4x_relaxed_write32(csa, config->syncfreq, TRCSYNCPR); 438 etm4x_relaxed_write32(csa, config->ccctlr, TRCCCCTLR); 439 etm4x_relaxed_write32(csa, config->bb_ctrl, TRCBBCTLR); 440 etm4x_relaxed_write32(csa, drvdata->trcid, TRCTRACEIDR); 441 etm4x_relaxed_write32(csa, config->vinst_ctrl, TRCVICTLR); 442 etm4x_relaxed_write32(csa, config->viiectlr, TRCVIIECTLR); 443 etm4x_relaxed_write32(csa, config->vissctlr, TRCVISSCTLR); 444 if (drvdata->nr_pe_cmp) 445 etm4x_relaxed_write32(csa, config->vipcssctlr, TRCVIPCSSCTLR); 446 for (i = 0; i < drvdata->nrseqstate - 1; i++) 447 etm4x_relaxed_write32(csa, config->seq_ctrl[i], TRCSEQEVRn(i)); 448 if (drvdata->nrseqstate) { 449 etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR); 450 etm4x_relaxed_write32(csa, config->seq_state, TRCSEQSTR); 451 } 452 etm4x_relaxed_write32(csa, config->ext_inp, TRCEXTINSELR); 453 for (i = 0; i < drvdata->nr_cntr; i++) { 454 etm4x_relaxed_write32(csa, config->cntrldvr[i], TRCCNTRLDVRn(i)); 455 etm4x_relaxed_write32(csa, config->cntr_ctrl[i], TRCCNTCTLRn(i)); 456 etm4x_relaxed_write32(csa, config->cntr_val[i], TRCCNTVRn(i)); 457 } 458 459 /* 460 * Resource selector pair 0 is always implemented and reserved. As 461 * such start at 2. 462 */ 463 for (i = 2; i < drvdata->nr_resource * 2; i++) 464 etm4x_relaxed_write32(csa, config->res_ctrl[i], TRCRSCTLRn(i)); 465 466 for (i = 0; i < drvdata->nr_ss_cmp; i++) { 467 /* always clear status bit on restart if using single-shot */ 468 if (config->ss_ctrl[i] || config->ss_pe_cmp[i]) 469 config->ss_status[i] &= ~TRCSSCSRn_STATUS; 470 etm4x_relaxed_write32(csa, config->ss_ctrl[i], TRCSSCCRn(i)); 471 etm4x_relaxed_write32(csa, config->ss_status[i], TRCSSCSRn(i)); 472 if (etm4x_sspcicrn_present(drvdata, i)) 473 etm4x_relaxed_write32(csa, config->ss_pe_cmp[i], TRCSSPCICRn(i)); 474 } 475 for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { 476 etm4x_relaxed_write64(csa, config->addr_val[i], TRCACVRn(i)); 477 etm4x_relaxed_write64(csa, config->addr_acc[i], TRCACATRn(i)); 478 } 479 for (i = 0; i < drvdata->numcidc; i++) 480 etm4x_relaxed_write64(csa, config->ctxid_pid[i], TRCCIDCVRn(i)); 481 etm4x_relaxed_write32(csa, config->ctxid_mask0, TRCCIDCCTLR0); 482 if (drvdata->numcidc > 4) 483 etm4x_relaxed_write32(csa, config->ctxid_mask1, TRCCIDCCTLR1); 484 485 for (i = 0; i < drvdata->numvmidc; i++) 486 etm4x_relaxed_write64(csa, config->vmid_val[i], TRCVMIDCVRn(i)); 487 etm4x_relaxed_write32(csa, config->vmid_mask0, TRCVMIDCCTLR0); 488 if (drvdata->numvmidc > 4) 489 etm4x_relaxed_write32(csa, config->vmid_mask1, TRCVMIDCCTLR1); 490 491 if (!drvdata->skip_power_up) { 492 u32 trcpdcr = etm4x_relaxed_read32(csa, TRCPDCR); 493 494 /* 495 * Request to keep the trace unit powered and also 496 * emulation of powerdown 497 */ 498 etm4x_relaxed_write32(csa, trcpdcr | TRCPDCR_PU, TRCPDCR); 499 } 500 501 /* 502 * ETE mandates that the TRCRSR is written to before 503 * enabling it. 504 */ 505 if (etm4x_is_ete(drvdata)) 506 etm4x_relaxed_write32(csa, TRCRSR_TA, TRCRSR); 507 508 etm4x_allow_trace(drvdata); 509 /* Enable the trace unit */ 510 etm4x_relaxed_write32(csa, 1, TRCPRGCTLR); 511 512 /* Synchronize the register updates for sysreg access */ 513 if (!csa->io_mem) 514 isb(); 515 516 /* wait for TRCSTATR.IDLE to go back down to '0' */ 517 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 0)) 518 dev_err(etm_dev, 519 "timeout while waiting for Idle Trace Status\n"); 520 521 /* 522 * As recommended by section 4.3.7 ("Synchronization when using the 523 * memory-mapped interface") of ARM IHI 0064D 524 */ 525 dsb(sy); 526 isb(); 527 528 done: 529 etm4_cs_lock(drvdata, csa); 530 531 dev_dbg(etm_dev, "cpu: %d enable smp call done: %d\n", 532 drvdata->cpu, rc); 533 return rc; 534 } 535 536 static void etm4_enable_hw_smp_call(void *info) 537 { 538 struct etm4_enable_arg *arg = info; 539 540 if (WARN_ON(!arg)) 541 return; 542 arg->rc = etm4_enable_hw(arg->drvdata); 543 } 544 545 /* 546 * The goal of function etm4_config_timestamp_event() is to configure a 547 * counter that will tell the tracer to emit a timestamp packet when it 548 * reaches zero. This is done in order to get a more fine grained idea 549 * of when instructions are executed so that they can be correlated 550 * with execution on other CPUs. 551 * 552 * To do this the counter itself is configured to self reload and 553 * TRCRSCTLR1 (always true) used to get the counter to decrement. From 554 * there a resource selector is configured with the counter and the 555 * timestamp control register to use the resource selector to trigger the 556 * event that will insert a timestamp packet in the stream. 557 */ 558 static int etm4_config_timestamp_event(struct etmv4_drvdata *drvdata) 559 { 560 int ctridx, ret = -EINVAL; 561 int counter, rselector; 562 u32 val = 0; 563 struct etmv4_config *config = &drvdata->config; 564 565 /* No point in trying if we don't have at least one counter */ 566 if (!drvdata->nr_cntr) 567 goto out; 568 569 /* Find a counter that hasn't been initialised */ 570 for (ctridx = 0; ctridx < drvdata->nr_cntr; ctridx++) 571 if (config->cntr_val[ctridx] == 0) 572 break; 573 574 /* All the counters have been configured already, bail out */ 575 if (ctridx == drvdata->nr_cntr) { 576 pr_debug("%s: no available counter found\n", __func__); 577 ret = -ENOSPC; 578 goto out; 579 } 580 581 /* 582 * Searching for an available resource selector to use, starting at 583 * '2' since every implementation has at least 2 resource selector. 584 * ETMIDR4 gives the number of resource selector _pairs_, 585 * hence multiply by 2. 586 */ 587 for (rselector = 2; rselector < drvdata->nr_resource * 2; rselector++) 588 if (!config->res_ctrl[rselector]) 589 break; 590 591 if (rselector == drvdata->nr_resource * 2) { 592 pr_debug("%s: no available resource selector found\n", 593 __func__); 594 ret = -ENOSPC; 595 goto out; 596 } 597 598 /* Remember what counter we used */ 599 counter = 1 << ctridx; 600 601 /* 602 * Initialise original and reload counter value to the smallest 603 * possible value in order to get as much precision as we can. 604 */ 605 config->cntr_val[ctridx] = 1; 606 config->cntrldvr[ctridx] = 1; 607 608 /* Set the trace counter control register */ 609 val = 0x1 << 16 | /* Bit 16, reload counter automatically */ 610 0x0 << 7 | /* Select single resource selector */ 611 0x1; /* Resource selector 1, i.e always true */ 612 613 config->cntr_ctrl[ctridx] = val; 614 615 val = 0x2 << 16 | /* Group 0b0010 - Counter and sequencers */ 616 counter << 0; /* Counter to use */ 617 618 config->res_ctrl[rselector] = val; 619 620 val = 0x0 << 7 | /* Select single resource selector */ 621 rselector; /* Resource selector */ 622 623 config->ts_ctrl = val; 624 625 ret = 0; 626 out: 627 return ret; 628 } 629 630 static int etm4_parse_event_config(struct coresight_device *csdev, 631 struct perf_event *event) 632 { 633 int ret = 0; 634 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 635 struct etmv4_config *config = &drvdata->config; 636 struct perf_event_attr *attr = &event->attr; 637 unsigned long cfg_hash; 638 int preset; 639 640 /* Clear configuration from previous run */ 641 memset(config, 0, sizeof(struct etmv4_config)); 642 643 if (attr->exclude_kernel) 644 config->mode = ETM_MODE_EXCL_KERN; 645 646 if (attr->exclude_user) 647 config->mode = ETM_MODE_EXCL_USER; 648 649 /* Always start from the default config */ 650 etm4_set_default_config(config); 651 652 /* Configure filters specified on the perf cmd line, if any. */ 653 ret = etm4_set_event_filters(drvdata, event); 654 if (ret) 655 goto out; 656 657 /* Go from generic option to ETMv4 specifics */ 658 if (attr->config & BIT(ETM_OPT_CYCACC)) { 659 config->cfg |= TRCCONFIGR_CCI; 660 /* TRM: Must program this for cycacc to work */ 661 config->ccctlr = ETM_CYC_THRESHOLD_DEFAULT; 662 } 663 if (attr->config & BIT(ETM_OPT_TS)) { 664 /* 665 * Configure timestamps to be emitted at regular intervals in 666 * order to correlate instructions executed on different CPUs 667 * (CPU-wide trace scenarios). 668 */ 669 ret = etm4_config_timestamp_event(drvdata); 670 671 /* 672 * No need to go further if timestamp intervals can't 673 * be configured. 674 */ 675 if (ret) 676 goto out; 677 678 /* bit[11], Global timestamp tracing bit */ 679 config->cfg |= TRCCONFIGR_TS; 680 } 681 682 /* Only trace contextID when runs in root PID namespace */ 683 if ((attr->config & BIT(ETM_OPT_CTXTID)) && 684 task_is_in_init_pid_ns(current)) 685 /* bit[6], Context ID tracing bit */ 686 config->cfg |= TRCCONFIGR_CID; 687 688 /* 689 * If set bit ETM_OPT_CTXTID2 in perf config, this asks to trace VMID 690 * for recording CONTEXTIDR_EL2. Do not enable VMID tracing if the 691 * kernel is not running in EL2. 692 */ 693 if (attr->config & BIT(ETM_OPT_CTXTID2)) { 694 if (!is_kernel_in_hyp_mode()) { 695 ret = -EINVAL; 696 goto out; 697 } 698 /* Only trace virtual contextID when runs in root PID namespace */ 699 if (task_is_in_init_pid_ns(current)) 700 config->cfg |= TRCCONFIGR_VMID | TRCCONFIGR_VMIDOPT; 701 } 702 703 /* return stack - enable if selected and supported */ 704 if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack) 705 /* bit[12], Return stack enable bit */ 706 config->cfg |= TRCCONFIGR_RS; 707 708 /* 709 * Set any selected configuration and preset. 710 * 711 * This extracts the values of PMU_FORMAT_ATTR(configid) and PMU_FORMAT_ATTR(preset) 712 * in the perf attributes defined in coresight-etm-perf.c. 713 * configid uses bits 63:32 of attr->config2, preset uses bits 3:0 of attr->config. 714 * A zero configid means no configuration active, preset = 0 means no preset selected. 715 */ 716 if (attr->config2 & GENMASK_ULL(63, 32)) { 717 cfg_hash = (u32)(attr->config2 >> 32); 718 preset = attr->config & 0xF; 719 ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); 720 } 721 722 /* branch broadcast - enable if selected and supported */ 723 if (attr->config & BIT(ETM_OPT_BRANCH_BROADCAST)) { 724 if (!drvdata->trcbb) { 725 /* 726 * Missing BB support could cause silent decode errors 727 * so fail to open if it's not supported. 728 */ 729 ret = -EINVAL; 730 goto out; 731 } else { 732 config->cfg |= BIT(ETM4_CFG_BIT_BB); 733 } 734 } 735 736 out: 737 return ret; 738 } 739 740 static int etm4_enable_perf(struct coresight_device *csdev, 741 struct perf_event *event) 742 { 743 int ret = 0, trace_id; 744 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 745 746 if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) { 747 ret = -EINVAL; 748 goto out; 749 } 750 751 /* Configure the tracer based on the session's specifics */ 752 ret = etm4_parse_event_config(csdev, event); 753 if (ret) 754 goto out; 755 756 /* 757 * perf allocates cpu ids as part of _setup_aux() - device needs to use 758 * the allocated ID. This reads the current version without allocation. 759 * 760 * This does not use the trace id lock to prevent lock_dep issues 761 * with perf locks - we know the ID cannot change until perf shuts down 762 * the session 763 */ 764 trace_id = coresight_trace_id_read_cpu_id(drvdata->cpu); 765 if (!IS_VALID_CS_TRACE_ID(trace_id)) { 766 dev_err(&drvdata->csdev->dev, "Failed to set trace ID for %s on CPU%d\n", 767 dev_name(&drvdata->csdev->dev), drvdata->cpu); 768 ret = -EINVAL; 769 goto out; 770 } 771 drvdata->trcid = (u8)trace_id; 772 773 /* And enable it */ 774 ret = etm4_enable_hw(drvdata); 775 776 out: 777 return ret; 778 } 779 780 static int etm4_enable_sysfs(struct coresight_device *csdev) 781 { 782 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 783 struct etm4_enable_arg arg = { }; 784 unsigned long cfg_hash; 785 int ret, preset; 786 787 /* enable any config activated by configfs */ 788 cscfg_config_sysfs_get_active_cfg(&cfg_hash, &preset); 789 if (cfg_hash) { 790 ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); 791 if (ret) 792 return ret; 793 } 794 795 spin_lock(&drvdata->spinlock); 796 797 /* sysfs needs to read and allocate a trace ID */ 798 ret = etm4_read_alloc_trace_id(drvdata); 799 if (ret < 0) 800 goto unlock_sysfs_enable; 801 802 /* 803 * Executing etm4_enable_hw on the cpu whose ETM is being enabled 804 * ensures that register writes occur when cpu is powered. 805 */ 806 arg.drvdata = drvdata; 807 ret = smp_call_function_single(drvdata->cpu, 808 etm4_enable_hw_smp_call, &arg, 1); 809 if (!ret) 810 ret = arg.rc; 811 if (!ret) 812 drvdata->sticky_enable = true; 813 814 if (ret) 815 etm4_release_trace_id(drvdata); 816 817 unlock_sysfs_enable: 818 spin_unlock(&drvdata->spinlock); 819 820 if (!ret) 821 dev_dbg(&csdev->dev, "ETM tracing enabled\n"); 822 return ret; 823 } 824 825 static int etm4_enable(struct coresight_device *csdev, 826 struct perf_event *event, u32 mode) 827 { 828 int ret; 829 u32 val; 830 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 831 832 val = local_cmpxchg(&drvdata->mode, CS_MODE_DISABLED, mode); 833 834 /* Someone is already using the tracer */ 835 if (val) 836 return -EBUSY; 837 838 switch (mode) { 839 case CS_MODE_SYSFS: 840 ret = etm4_enable_sysfs(csdev); 841 break; 842 case CS_MODE_PERF: 843 ret = etm4_enable_perf(csdev, event); 844 break; 845 default: 846 ret = -EINVAL; 847 } 848 849 /* The tracer didn't start */ 850 if (ret) 851 local_set(&drvdata->mode, CS_MODE_DISABLED); 852 853 return ret; 854 } 855 856 static void etm4_disable_hw(void *info) 857 { 858 u32 control; 859 struct etmv4_drvdata *drvdata = info; 860 struct etmv4_config *config = &drvdata->config; 861 struct coresight_device *csdev = drvdata->csdev; 862 struct device *etm_dev = &csdev->dev; 863 struct csdev_access *csa = &csdev->access; 864 int i; 865 866 etm4_cs_unlock(drvdata, csa); 867 etm4_disable_arch_specific(drvdata); 868 869 if (!drvdata->skip_power_up) { 870 /* power can be removed from the trace unit now */ 871 control = etm4x_relaxed_read32(csa, TRCPDCR); 872 control &= ~TRCPDCR_PU; 873 etm4x_relaxed_write32(csa, control, TRCPDCR); 874 } 875 876 control = etm4x_relaxed_read32(csa, TRCPRGCTLR); 877 878 /* EN, bit[0] Trace unit enable bit */ 879 control &= ~0x1; 880 881 /* 882 * If the CPU supports v8.4 Trace filter Control, 883 * set the ETM to trace prohibited region. 884 */ 885 etm4x_prohibit_trace(drvdata); 886 /* 887 * Make sure everything completes before disabling, as recommended 888 * by section 7.3.77 ("TRCVICTLR, ViewInst Main Control Register, 889 * SSTATUS") of ARM IHI 0064D 890 */ 891 dsb(sy); 892 isb(); 893 /* Trace synchronization barrier, is a nop if not supported */ 894 tsb_csync(); 895 etm4x_relaxed_write32(csa, control, TRCPRGCTLR); 896 897 /* wait for TRCSTATR.PMSTABLE to go to '1' */ 898 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) 899 dev_err(etm_dev, 900 "timeout while waiting for PM stable Trace Status\n"); 901 /* read the status of the single shot comparators */ 902 for (i = 0; i < drvdata->nr_ss_cmp; i++) { 903 config->ss_status[i] = 904 etm4x_relaxed_read32(csa, TRCSSCSRn(i)); 905 } 906 907 /* read back the current counter values */ 908 for (i = 0; i < drvdata->nr_cntr; i++) { 909 config->cntr_val[i] = 910 etm4x_relaxed_read32(csa, TRCCNTVRn(i)); 911 } 912 913 coresight_disclaim_device_unlocked(csdev); 914 etm4_cs_lock(drvdata, csa); 915 916 dev_dbg(&drvdata->csdev->dev, 917 "cpu: %d disable smp call done\n", drvdata->cpu); 918 } 919 920 static int etm4_disable_perf(struct coresight_device *csdev, 921 struct perf_event *event) 922 { 923 u32 control; 924 struct etm_filters *filters = event->hw.addr_filters; 925 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 926 struct perf_event_attr *attr = &event->attr; 927 928 if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) 929 return -EINVAL; 930 931 etm4_disable_hw(drvdata); 932 /* 933 * The config_id occupies bits 63:32 of the config2 perf event attr 934 * field. If this is non-zero then we will have enabled a config. 935 */ 936 if (attr->config2 & GENMASK_ULL(63, 32)) 937 cscfg_csdev_disable_active_config(csdev); 938 939 /* 940 * Check if the start/stop logic was active when the unit was stopped. 941 * That way we can re-enable the start/stop logic when the process is 942 * scheduled again. Configuration of the start/stop logic happens in 943 * function etm4_set_event_filters(). 944 */ 945 control = etm4x_relaxed_read32(&csdev->access, TRCVICTLR); 946 /* TRCVICTLR::SSSTATUS, bit[9] */ 947 filters->ssstatus = (control & BIT(9)); 948 949 /* 950 * perf will release trace ids when _free_aux() is 951 * called at the end of the session. 952 */ 953 954 return 0; 955 } 956 957 static void etm4_disable_sysfs(struct coresight_device *csdev) 958 { 959 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 960 961 /* 962 * Taking hotplug lock here protects from clocks getting disabled 963 * with tracing being left on (crash scenario) if user disable occurs 964 * after cpu online mask indicates the cpu is offline but before the 965 * DYING hotplug callback is serviced by the ETM driver. 966 */ 967 cpus_read_lock(); 968 spin_lock(&drvdata->spinlock); 969 970 /* 971 * Executing etm4_disable_hw on the cpu whose ETM is being disabled 972 * ensures that register writes occur when cpu is powered. 973 */ 974 smp_call_function_single(drvdata->cpu, etm4_disable_hw, drvdata, 1); 975 976 spin_unlock(&drvdata->spinlock); 977 cpus_read_unlock(); 978 979 /* 980 * we only release trace IDs when resetting sysfs. 981 * This permits sysfs users to read the trace ID after the trace 982 * session has completed. This maintains operational behaviour with 983 * prior trace id allocation method 984 */ 985 986 dev_dbg(&csdev->dev, "ETM tracing disabled\n"); 987 } 988 989 static void etm4_disable(struct coresight_device *csdev, 990 struct perf_event *event) 991 { 992 u32 mode; 993 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 994 995 /* 996 * For as long as the tracer isn't disabled another entity can't 997 * change its status. As such we can read the status here without 998 * fearing it will change under us. 999 */ 1000 mode = local_read(&drvdata->mode); 1001 1002 switch (mode) { 1003 case CS_MODE_DISABLED: 1004 break; 1005 case CS_MODE_SYSFS: 1006 etm4_disable_sysfs(csdev); 1007 break; 1008 case CS_MODE_PERF: 1009 etm4_disable_perf(csdev, event); 1010 break; 1011 } 1012 1013 if (mode) 1014 local_set(&drvdata->mode, CS_MODE_DISABLED); 1015 } 1016 1017 static const struct coresight_ops_source etm4_source_ops = { 1018 .cpu_id = etm4_cpu_id, 1019 .enable = etm4_enable, 1020 .disable = etm4_disable, 1021 }; 1022 1023 static const struct coresight_ops etm4_cs_ops = { 1024 .source_ops = &etm4_source_ops, 1025 }; 1026 1027 static inline bool cpu_supports_sysreg_trace(void) 1028 { 1029 u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); 1030 1031 return ((dfr0 >> ID_AA64DFR0_EL1_TraceVer_SHIFT) & 0xfUL) > 0; 1032 } 1033 1034 static bool etm4_init_sysreg_access(struct etmv4_drvdata *drvdata, 1035 struct csdev_access *csa) 1036 { 1037 u32 devarch; 1038 1039 if (!cpu_supports_sysreg_trace()) 1040 return false; 1041 1042 /* 1043 * ETMs implementing sysreg access must implement TRCDEVARCH. 1044 */ 1045 devarch = read_etm4x_sysreg_const_offset(TRCDEVARCH); 1046 switch (devarch & ETM_DEVARCH_ID_MASK) { 1047 case ETM_DEVARCH_ETMv4x_ARCH: 1048 *csa = (struct csdev_access) { 1049 .io_mem = false, 1050 .read = etm4x_sysreg_read, 1051 .write = etm4x_sysreg_write, 1052 }; 1053 break; 1054 case ETM_DEVARCH_ETE_ARCH: 1055 *csa = (struct csdev_access) { 1056 .io_mem = false, 1057 .read = ete_sysreg_read, 1058 .write = ete_sysreg_write, 1059 }; 1060 break; 1061 default: 1062 return false; 1063 } 1064 1065 drvdata->arch = etm_devarch_to_arch(devarch); 1066 return true; 1067 } 1068 1069 static bool etm4_init_iomem_access(struct etmv4_drvdata *drvdata, 1070 struct csdev_access *csa) 1071 { 1072 u32 devarch = readl_relaxed(drvdata->base + TRCDEVARCH); 1073 1074 /* 1075 * All ETMs must implement TRCDEVARCH to indicate that 1076 * the component is an ETMv4. Even though TRCIDR1 also 1077 * contains the information, it is part of the "Trace" 1078 * register and must be accessed with the OSLK cleared, 1079 * with MMIO. But we cannot touch the OSLK until we are 1080 * sure this is an ETM. So rely only on the TRCDEVARCH. 1081 */ 1082 if ((devarch & ETM_DEVARCH_ID_MASK) != ETM_DEVARCH_ETMv4x_ARCH) { 1083 pr_warn_once("TRCDEVARCH doesn't match ETMv4 architecture\n"); 1084 return false; 1085 } 1086 1087 drvdata->arch = etm_devarch_to_arch(devarch); 1088 *csa = CSDEV_ACCESS_IOMEM(drvdata->base); 1089 return true; 1090 } 1091 1092 static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata, 1093 struct csdev_access *csa) 1094 { 1095 /* 1096 * Always choose the memory mapped io, if there is 1097 * a memory map to prevent sysreg access on broken 1098 * systems. 1099 */ 1100 if (drvdata->base) 1101 return etm4_init_iomem_access(drvdata, csa); 1102 1103 if (etm4_init_sysreg_access(drvdata, csa)) 1104 return true; 1105 1106 return false; 1107 } 1108 1109 static void cpu_detect_trace_filtering(struct etmv4_drvdata *drvdata) 1110 { 1111 u64 dfr0 = read_sysreg(id_aa64dfr0_el1); 1112 u64 trfcr; 1113 1114 drvdata->trfcr = 0; 1115 if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_TraceFilt_SHIFT)) 1116 return; 1117 1118 /* 1119 * If the CPU supports v8.4 SelfHosted Tracing, enable 1120 * tracing at the kernel EL and EL0, forcing to use the 1121 * virtual time as the timestamp. 1122 */ 1123 trfcr = (TRFCR_ELx_TS_VIRTUAL | 1124 TRFCR_ELx_ExTRE | 1125 TRFCR_ELx_E0TRE); 1126 1127 /* If we are running at EL2, allow tracing the CONTEXTIDR_EL2. */ 1128 if (is_kernel_in_hyp_mode()) 1129 trfcr |= TRFCR_EL2_CX; 1130 1131 drvdata->trfcr = trfcr; 1132 } 1133 1134 static void etm4_init_arch_data(void *info) 1135 { 1136 u32 etmidr0; 1137 u32 etmidr2; 1138 u32 etmidr3; 1139 u32 etmidr4; 1140 u32 etmidr5; 1141 struct etm4_init_arg *init_arg = info; 1142 struct etmv4_drvdata *drvdata; 1143 struct csdev_access *csa; 1144 int i; 1145 1146 drvdata = dev_get_drvdata(init_arg->dev); 1147 csa = init_arg->csa; 1148 1149 /* 1150 * If we are unable to detect the access mechanism, 1151 * or unable to detect the trace unit type, fail 1152 * early. 1153 */ 1154 if (!etm4_init_csdev_access(drvdata, csa)) 1155 return; 1156 1157 /* Detect the support for OS Lock before we actually use it */ 1158 etm_detect_os_lock(drvdata, csa); 1159 1160 /* Make sure all registers are accessible */ 1161 etm4_os_unlock_csa(drvdata, csa); 1162 etm4_cs_unlock(drvdata, csa); 1163 1164 etm4_check_arch_features(drvdata, init_arg->pid); 1165 1166 /* find all capabilities of the tracing unit */ 1167 etmidr0 = etm4x_relaxed_read32(csa, TRCIDR0); 1168 1169 /* INSTP0, bits[2:1] P0 tracing support field */ 1170 drvdata->instrp0 = !!(FIELD_GET(TRCIDR0_INSTP0_MASK, etmidr0) == 0b11); 1171 /* TRCBB, bit[5] Branch broadcast tracing support bit */ 1172 drvdata->trcbb = !!(etmidr0 & TRCIDR0_TRCBB); 1173 /* TRCCOND, bit[6] Conditional instruction tracing support bit */ 1174 drvdata->trccond = !!(etmidr0 & TRCIDR0_TRCCOND); 1175 /* TRCCCI, bit[7] Cycle counting instruction bit */ 1176 drvdata->trccci = !!(etmidr0 & TRCIDR0_TRCCCI); 1177 /* RETSTACK, bit[9] Return stack bit */ 1178 drvdata->retstack = !!(etmidr0 & TRCIDR0_RETSTACK); 1179 /* NUMEVENT, bits[11:10] Number of events field */ 1180 drvdata->nr_event = FIELD_GET(TRCIDR0_NUMEVENT_MASK, etmidr0); 1181 /* QSUPP, bits[16:15] Q element support field */ 1182 drvdata->q_support = FIELD_GET(TRCIDR0_QSUPP_MASK, etmidr0); 1183 /* TSSIZE, bits[28:24] Global timestamp size field */ 1184 drvdata->ts_size = FIELD_GET(TRCIDR0_TSSIZE_MASK, etmidr0); 1185 1186 /* maximum size of resources */ 1187 etmidr2 = etm4x_relaxed_read32(csa, TRCIDR2); 1188 /* CIDSIZE, bits[9:5] Indicates the Context ID size */ 1189 drvdata->ctxid_size = FIELD_GET(TRCIDR2_CIDSIZE_MASK, etmidr2); 1190 /* VMIDSIZE, bits[14:10] Indicates the VMID size */ 1191 drvdata->vmid_size = FIELD_GET(TRCIDR2_VMIDSIZE_MASK, etmidr2); 1192 /* CCSIZE, bits[28:25] size of the cycle counter in bits minus 12 */ 1193 drvdata->ccsize = FIELD_GET(TRCIDR2_CCSIZE_MASK, etmidr2); 1194 1195 etmidr3 = etm4x_relaxed_read32(csa, TRCIDR3); 1196 /* CCITMIN, bits[11:0] minimum threshold value that can be programmed */ 1197 drvdata->ccitmin = FIELD_GET(TRCIDR3_CCITMIN_MASK, etmidr3); 1198 /* EXLEVEL_S, bits[19:16] Secure state instruction tracing */ 1199 drvdata->s_ex_level = FIELD_GET(TRCIDR3_EXLEVEL_S_MASK, etmidr3); 1200 drvdata->config.s_ex_level = drvdata->s_ex_level; 1201 /* EXLEVEL_NS, bits[23:20] Non-secure state instruction tracing */ 1202 drvdata->ns_ex_level = FIELD_GET(TRCIDR3_EXLEVEL_NS_MASK, etmidr3); 1203 /* 1204 * TRCERR, bit[24] whether a trace unit can trace a 1205 * system error exception. 1206 */ 1207 drvdata->trc_error = !!(etmidr3 & TRCIDR3_TRCERR); 1208 /* SYNCPR, bit[25] implementation has a fixed synchronization period? */ 1209 drvdata->syncpr = !!(etmidr3 & TRCIDR3_SYNCPR); 1210 /* STALLCTL, bit[26] is stall control implemented? */ 1211 drvdata->stallctl = !!(etmidr3 & TRCIDR3_STALLCTL); 1212 /* SYSSTALL, bit[27] implementation can support stall control? */ 1213 drvdata->sysstall = !!(etmidr3 & TRCIDR3_SYSSTALL); 1214 /* 1215 * NUMPROC - the number of PEs available for tracing, 5bits 1216 * = TRCIDR3.bits[13:12]bits[30:28] 1217 * bits[4:3] = TRCIDR3.bits[13:12] (since etm-v4.2, otherwise RES0) 1218 * bits[3:0] = TRCIDR3.bits[30:28] 1219 */ 1220 drvdata->nr_pe = (FIELD_GET(TRCIDR3_NUMPROC_HI_MASK, etmidr3) << 3) | 1221 FIELD_GET(TRCIDR3_NUMPROC_LO_MASK, etmidr3); 1222 /* NOOVERFLOW, bit[31] is trace overflow prevention supported */ 1223 drvdata->nooverflow = !!(etmidr3 & TRCIDR3_NOOVERFLOW); 1224 1225 /* number of resources trace unit supports */ 1226 etmidr4 = etm4x_relaxed_read32(csa, TRCIDR4); 1227 /* NUMACPAIRS, bits[0:3] number of addr comparator pairs for tracing */ 1228 drvdata->nr_addr_cmp = FIELD_GET(TRCIDR4_NUMACPAIRS_MASK, etmidr4); 1229 /* NUMPC, bits[15:12] number of PE comparator inputs for tracing */ 1230 drvdata->nr_pe_cmp = FIELD_GET(TRCIDR4_NUMPC_MASK, etmidr4); 1231 /* 1232 * NUMRSPAIR, bits[19:16] 1233 * The number of resource pairs conveyed by the HW starts at 0, i.e a 1234 * value of 0x0 indicate 1 resource pair, 0x1 indicate two and so on. 1235 * As such add 1 to the value of NUMRSPAIR for a better representation. 1236 * 1237 * For ETM v4.3 and later, 0x0 means 0, and no pairs are available - 1238 * the default TRUE and FALSE resource selectors are omitted. 1239 * Otherwise for values 0x1 and above the number is N + 1 as per v4.2. 1240 */ 1241 drvdata->nr_resource = FIELD_GET(TRCIDR4_NUMRSPAIR_MASK, etmidr4); 1242 if ((drvdata->arch < ETM_ARCH_V4_3) || (drvdata->nr_resource > 0)) 1243 drvdata->nr_resource += 1; 1244 /* 1245 * NUMSSCC, bits[23:20] the number of single-shot 1246 * comparator control for tracing. Read any status regs as these 1247 * also contain RO capability data. 1248 */ 1249 drvdata->nr_ss_cmp = FIELD_GET(TRCIDR4_NUMSSCC_MASK, etmidr4); 1250 for (i = 0; i < drvdata->nr_ss_cmp; i++) { 1251 drvdata->config.ss_status[i] = 1252 etm4x_relaxed_read32(csa, TRCSSCSRn(i)); 1253 } 1254 /* NUMCIDC, bits[27:24] number of Context ID comparators for tracing */ 1255 drvdata->numcidc = FIELD_GET(TRCIDR4_NUMCIDC_MASK, etmidr4); 1256 /* NUMVMIDC, bits[31:28] number of VMID comparators for tracing */ 1257 drvdata->numvmidc = FIELD_GET(TRCIDR4_NUMVMIDC_MASK, etmidr4); 1258 1259 etmidr5 = etm4x_relaxed_read32(csa, TRCIDR5); 1260 /* NUMEXTIN, bits[8:0] number of external inputs implemented */ 1261 drvdata->nr_ext_inp = FIELD_GET(TRCIDR5_NUMEXTIN_MASK, etmidr5); 1262 /* TRACEIDSIZE, bits[21:16] indicates the trace ID width */ 1263 drvdata->trcid_size = FIELD_GET(TRCIDR5_TRACEIDSIZE_MASK, etmidr5); 1264 /* ATBTRIG, bit[22] implementation can support ATB triggers? */ 1265 drvdata->atbtrig = !!(etmidr5 & TRCIDR5_ATBTRIG); 1266 /* 1267 * LPOVERRIDE, bit[23] implementation supports 1268 * low-power state override 1269 */ 1270 drvdata->lpoverride = (etmidr5 & TRCIDR5_LPOVERRIDE) && (!drvdata->skip_power_up); 1271 /* NUMSEQSTATE, bits[27:25] number of sequencer states implemented */ 1272 drvdata->nrseqstate = FIELD_GET(TRCIDR5_NUMSEQSTATE_MASK, etmidr5); 1273 /* NUMCNTR, bits[30:28] number of counters available for tracing */ 1274 drvdata->nr_cntr = FIELD_GET(TRCIDR5_NUMCNTR_MASK, etmidr5); 1275 etm4_cs_lock(drvdata, csa); 1276 cpu_detect_trace_filtering(drvdata); 1277 } 1278 1279 static inline u32 etm4_get_victlr_access_type(struct etmv4_config *config) 1280 { 1281 return etm4_get_access_type(config) << __bf_shf(TRCVICTLR_EXLEVEL_MASK); 1282 } 1283 1284 /* Set ELx trace filter access in the TRCVICTLR register */ 1285 static void etm4_set_victlr_access(struct etmv4_config *config) 1286 { 1287 config->vinst_ctrl &= ~TRCVICTLR_EXLEVEL_MASK; 1288 config->vinst_ctrl |= etm4_get_victlr_access_type(config); 1289 } 1290 1291 static void etm4_set_default_config(struct etmv4_config *config) 1292 { 1293 /* disable all events tracing */ 1294 config->eventctrl0 = 0x0; 1295 config->eventctrl1 = 0x0; 1296 1297 /* disable stalling */ 1298 config->stall_ctrl = 0x0; 1299 1300 /* enable trace synchronization every 4096 bytes, if available */ 1301 config->syncfreq = 0xC; 1302 1303 /* disable timestamp event */ 1304 config->ts_ctrl = 0x0; 1305 1306 /* TRCVICTLR::EVENT = 0x01, select the always on logic */ 1307 config->vinst_ctrl = FIELD_PREP(TRCVICTLR_EVENT_MASK, 0x01); 1308 1309 /* TRCVICTLR::EXLEVEL_NS:EXLEVELS: Set kernel / user filtering */ 1310 etm4_set_victlr_access(config); 1311 } 1312 1313 static u64 etm4_get_ns_access_type(struct etmv4_config *config) 1314 { 1315 u64 access_type = 0; 1316 1317 /* 1318 * EXLEVEL_NS, for NonSecure Exception levels. 1319 * The mask here is a generic value and must be 1320 * shifted to the corresponding field for the registers 1321 */ 1322 if (!is_kernel_in_hyp_mode()) { 1323 /* Stay away from hypervisor mode for non-VHE */ 1324 access_type = ETM_EXLEVEL_NS_HYP; 1325 if (config->mode & ETM_MODE_EXCL_KERN) 1326 access_type |= ETM_EXLEVEL_NS_OS; 1327 } else if (config->mode & ETM_MODE_EXCL_KERN) { 1328 access_type = ETM_EXLEVEL_NS_HYP; 1329 } 1330 1331 if (config->mode & ETM_MODE_EXCL_USER) 1332 access_type |= ETM_EXLEVEL_NS_APP; 1333 1334 return access_type; 1335 } 1336 1337 /* 1338 * Construct the exception level masks for a given config. 1339 * This must be shifted to the corresponding register field 1340 * for usage. 1341 */ 1342 static u64 etm4_get_access_type(struct etmv4_config *config) 1343 { 1344 /* All Secure exception levels are excluded from the trace */ 1345 return etm4_get_ns_access_type(config) | (u64)config->s_ex_level; 1346 } 1347 1348 static u64 etm4_get_comparator_access_type(struct etmv4_config *config) 1349 { 1350 return etm4_get_access_type(config) << TRCACATR_EXLEVEL_SHIFT; 1351 } 1352 1353 static void etm4_set_comparator_filter(struct etmv4_config *config, 1354 u64 start, u64 stop, int comparator) 1355 { 1356 u64 access_type = etm4_get_comparator_access_type(config); 1357 1358 /* First half of default address comparator */ 1359 config->addr_val[comparator] = start; 1360 config->addr_acc[comparator] = access_type; 1361 config->addr_type[comparator] = ETM_ADDR_TYPE_RANGE; 1362 1363 /* Second half of default address comparator */ 1364 config->addr_val[comparator + 1] = stop; 1365 config->addr_acc[comparator + 1] = access_type; 1366 config->addr_type[comparator + 1] = ETM_ADDR_TYPE_RANGE; 1367 1368 /* 1369 * Configure the ViewInst function to include this address range 1370 * comparator. 1371 * 1372 * @comparator is divided by two since it is the index in the 1373 * etmv4_config::addr_val array but register TRCVIIECTLR deals with 1374 * address range comparator _pairs_. 1375 * 1376 * Therefore: 1377 * index 0 -> compatator pair 0 1378 * index 2 -> comparator pair 1 1379 * index 4 -> comparator pair 2 1380 * ... 1381 * index 14 -> comparator pair 7 1382 */ 1383 config->viiectlr |= BIT(comparator / 2); 1384 } 1385 1386 static void etm4_set_start_stop_filter(struct etmv4_config *config, 1387 u64 address, int comparator, 1388 enum etm_addr_type type) 1389 { 1390 int shift; 1391 u64 access_type = etm4_get_comparator_access_type(config); 1392 1393 /* Configure the comparator */ 1394 config->addr_val[comparator] = address; 1395 config->addr_acc[comparator] = access_type; 1396 config->addr_type[comparator] = type; 1397 1398 /* 1399 * Configure ViewInst Start-Stop control register. 1400 * Addresses configured to start tracing go from bit 0 to n-1, 1401 * while those configured to stop tracing from 16 to 16 + n-1. 1402 */ 1403 shift = (type == ETM_ADDR_TYPE_START ? 0 : 16); 1404 config->vissctlr |= BIT(shift + comparator); 1405 } 1406 1407 static void etm4_set_default_filter(struct etmv4_config *config) 1408 { 1409 /* Trace everything 'default' filter achieved by no filtering */ 1410 config->viiectlr = 0x0; 1411 1412 /* 1413 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is 1414 * in the started state 1415 */ 1416 config->vinst_ctrl |= TRCVICTLR_SSSTATUS; 1417 config->mode |= ETM_MODE_VIEWINST_STARTSTOP; 1418 1419 /* No start-stop filtering for ViewInst */ 1420 config->vissctlr = 0x0; 1421 } 1422 1423 static void etm4_set_default(struct etmv4_config *config) 1424 { 1425 if (WARN_ON_ONCE(!config)) 1426 return; 1427 1428 /* 1429 * Make default initialisation trace everything 1430 * 1431 * This is done by a minimum default config sufficient to enable 1432 * full instruction trace - with a default filter for trace all 1433 * achieved by having no filtering. 1434 */ 1435 etm4_set_default_config(config); 1436 etm4_set_default_filter(config); 1437 } 1438 1439 static int etm4_get_next_comparator(struct etmv4_drvdata *drvdata, u32 type) 1440 { 1441 int nr_comparator, index = 0; 1442 struct etmv4_config *config = &drvdata->config; 1443 1444 /* 1445 * nr_addr_cmp holds the number of comparator _pair_, so time 2 1446 * for the total number of comparators. 1447 */ 1448 nr_comparator = drvdata->nr_addr_cmp * 2; 1449 1450 /* Go through the tally of comparators looking for a free one. */ 1451 while (index < nr_comparator) { 1452 switch (type) { 1453 case ETM_ADDR_TYPE_RANGE: 1454 if (config->addr_type[index] == ETM_ADDR_TYPE_NONE && 1455 config->addr_type[index + 1] == ETM_ADDR_TYPE_NONE) 1456 return index; 1457 1458 /* Address range comparators go in pairs */ 1459 index += 2; 1460 break; 1461 case ETM_ADDR_TYPE_START: 1462 case ETM_ADDR_TYPE_STOP: 1463 if (config->addr_type[index] == ETM_ADDR_TYPE_NONE) 1464 return index; 1465 1466 /* Start/stop address can have odd indexes */ 1467 index += 1; 1468 break; 1469 default: 1470 return -EINVAL; 1471 } 1472 } 1473 1474 /* If we are here all the comparators have been used. */ 1475 return -ENOSPC; 1476 } 1477 1478 static int etm4_set_event_filters(struct etmv4_drvdata *drvdata, 1479 struct perf_event *event) 1480 { 1481 int i, comparator, ret = 0; 1482 u64 address; 1483 struct etmv4_config *config = &drvdata->config; 1484 struct etm_filters *filters = event->hw.addr_filters; 1485 1486 if (!filters) 1487 goto default_filter; 1488 1489 /* Sync events with what Perf got */ 1490 perf_event_addr_filters_sync(event); 1491 1492 /* 1493 * If there are no filters to deal with simply go ahead with 1494 * the default filter, i.e the entire address range. 1495 */ 1496 if (!filters->nr_filters) 1497 goto default_filter; 1498 1499 for (i = 0; i < filters->nr_filters; i++) { 1500 struct etm_filter *filter = &filters->etm_filter[i]; 1501 enum etm_addr_type type = filter->type; 1502 1503 /* See if a comparator is free. */ 1504 comparator = etm4_get_next_comparator(drvdata, type); 1505 if (comparator < 0) { 1506 ret = comparator; 1507 goto out; 1508 } 1509 1510 switch (type) { 1511 case ETM_ADDR_TYPE_RANGE: 1512 etm4_set_comparator_filter(config, 1513 filter->start_addr, 1514 filter->stop_addr, 1515 comparator); 1516 /* 1517 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is 1518 * in the started state 1519 */ 1520 config->vinst_ctrl |= TRCVICTLR_SSSTATUS; 1521 1522 /* No start-stop filtering for ViewInst */ 1523 config->vissctlr = 0x0; 1524 break; 1525 case ETM_ADDR_TYPE_START: 1526 case ETM_ADDR_TYPE_STOP: 1527 /* Get the right start or stop address */ 1528 address = (type == ETM_ADDR_TYPE_START ? 1529 filter->start_addr : 1530 filter->stop_addr); 1531 1532 /* Configure comparator */ 1533 etm4_set_start_stop_filter(config, address, 1534 comparator, type); 1535 1536 /* 1537 * If filters::ssstatus == 1, trace acquisition was 1538 * started but the process was yanked away before the 1539 * stop address was hit. As such the start/stop 1540 * logic needs to be re-started so that tracing can 1541 * resume where it left. 1542 * 1543 * The start/stop logic status when a process is 1544 * scheduled out is checked in function 1545 * etm4_disable_perf(). 1546 */ 1547 if (filters->ssstatus) 1548 config->vinst_ctrl |= TRCVICTLR_SSSTATUS; 1549 1550 /* No include/exclude filtering for ViewInst */ 1551 config->viiectlr = 0x0; 1552 break; 1553 default: 1554 ret = -EINVAL; 1555 goto out; 1556 } 1557 } 1558 1559 goto out; 1560 1561 1562 default_filter: 1563 etm4_set_default_filter(config); 1564 1565 out: 1566 return ret; 1567 } 1568 1569 void etm4_config_trace_mode(struct etmv4_config *config) 1570 { 1571 u32 mode; 1572 1573 mode = config->mode; 1574 mode &= (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER); 1575 1576 /* excluding kernel AND user space doesn't make sense */ 1577 WARN_ON_ONCE(mode == (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER)); 1578 1579 /* nothing to do if neither flags are set */ 1580 if (!(mode & ETM_MODE_EXCL_KERN) && !(mode & ETM_MODE_EXCL_USER)) 1581 return; 1582 1583 etm4_set_victlr_access(config); 1584 } 1585 1586 static int etm4_online_cpu(unsigned int cpu) 1587 { 1588 if (!etmdrvdata[cpu]) 1589 return etm4_probe_cpu(cpu); 1590 1591 if (etmdrvdata[cpu]->boot_enable && !etmdrvdata[cpu]->sticky_enable) 1592 coresight_enable(etmdrvdata[cpu]->csdev); 1593 return 0; 1594 } 1595 1596 static int etm4_starting_cpu(unsigned int cpu) 1597 { 1598 if (!etmdrvdata[cpu]) 1599 return 0; 1600 1601 spin_lock(&etmdrvdata[cpu]->spinlock); 1602 if (!etmdrvdata[cpu]->os_unlock) 1603 etm4_os_unlock(etmdrvdata[cpu]); 1604 1605 if (local_read(&etmdrvdata[cpu]->mode)) 1606 etm4_enable_hw(etmdrvdata[cpu]); 1607 spin_unlock(&etmdrvdata[cpu]->spinlock); 1608 return 0; 1609 } 1610 1611 static int etm4_dying_cpu(unsigned int cpu) 1612 { 1613 if (!etmdrvdata[cpu]) 1614 return 0; 1615 1616 spin_lock(&etmdrvdata[cpu]->spinlock); 1617 if (local_read(&etmdrvdata[cpu]->mode)) 1618 etm4_disable_hw(etmdrvdata[cpu]); 1619 spin_unlock(&etmdrvdata[cpu]->spinlock); 1620 return 0; 1621 } 1622 1623 static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) 1624 { 1625 int i, ret = 0; 1626 struct etmv4_save_state *state; 1627 struct coresight_device *csdev = drvdata->csdev; 1628 struct csdev_access *csa; 1629 struct device *etm_dev; 1630 1631 if (WARN_ON(!csdev)) 1632 return -ENODEV; 1633 1634 etm_dev = &csdev->dev; 1635 csa = &csdev->access; 1636 1637 /* 1638 * As recommended by 3.4.1 ("The procedure when powering down the PE") 1639 * of ARM IHI 0064D 1640 */ 1641 dsb(sy); 1642 isb(); 1643 1644 etm4_cs_unlock(drvdata, csa); 1645 /* Lock the OS lock to disable trace and external debugger access */ 1646 etm4_os_lock(drvdata); 1647 1648 /* wait for TRCSTATR.PMSTABLE to go up */ 1649 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) { 1650 dev_err(etm_dev, 1651 "timeout while waiting for PM Stable Status\n"); 1652 etm4_os_unlock(drvdata); 1653 ret = -EBUSY; 1654 goto out; 1655 } 1656 1657 state = drvdata->save_state; 1658 1659 state->trcprgctlr = etm4x_read32(csa, TRCPRGCTLR); 1660 if (drvdata->nr_pe) 1661 state->trcprocselr = etm4x_read32(csa, TRCPROCSELR); 1662 state->trcconfigr = etm4x_read32(csa, TRCCONFIGR); 1663 state->trcauxctlr = etm4x_read32(csa, TRCAUXCTLR); 1664 state->trceventctl0r = etm4x_read32(csa, TRCEVENTCTL0R); 1665 state->trceventctl1r = etm4x_read32(csa, TRCEVENTCTL1R); 1666 if (drvdata->stallctl) 1667 state->trcstallctlr = etm4x_read32(csa, TRCSTALLCTLR); 1668 state->trctsctlr = etm4x_read32(csa, TRCTSCTLR); 1669 state->trcsyncpr = etm4x_read32(csa, TRCSYNCPR); 1670 state->trcccctlr = etm4x_read32(csa, TRCCCCTLR); 1671 state->trcbbctlr = etm4x_read32(csa, TRCBBCTLR); 1672 state->trctraceidr = etm4x_read32(csa, TRCTRACEIDR); 1673 state->trcqctlr = etm4x_read32(csa, TRCQCTLR); 1674 1675 state->trcvictlr = etm4x_read32(csa, TRCVICTLR); 1676 state->trcviiectlr = etm4x_read32(csa, TRCVIIECTLR); 1677 state->trcvissctlr = etm4x_read32(csa, TRCVISSCTLR); 1678 if (drvdata->nr_pe_cmp) 1679 state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR); 1680 state->trcvdctlr = etm4x_read32(csa, TRCVDCTLR); 1681 state->trcvdsacctlr = etm4x_read32(csa, TRCVDSACCTLR); 1682 state->trcvdarcctlr = etm4x_read32(csa, TRCVDARCCTLR); 1683 1684 for (i = 0; i < drvdata->nrseqstate - 1; i++) 1685 state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i)); 1686 1687 if (drvdata->nrseqstate) { 1688 state->trcseqrstevr = etm4x_read32(csa, TRCSEQRSTEVR); 1689 state->trcseqstr = etm4x_read32(csa, TRCSEQSTR); 1690 } 1691 state->trcextinselr = etm4x_read32(csa, TRCEXTINSELR); 1692 1693 for (i = 0; i < drvdata->nr_cntr; i++) { 1694 state->trccntrldvr[i] = etm4x_read32(csa, TRCCNTRLDVRn(i)); 1695 state->trccntctlr[i] = etm4x_read32(csa, TRCCNTCTLRn(i)); 1696 state->trccntvr[i] = etm4x_read32(csa, TRCCNTVRn(i)); 1697 } 1698 1699 for (i = 0; i < drvdata->nr_resource * 2; i++) 1700 state->trcrsctlr[i] = etm4x_read32(csa, TRCRSCTLRn(i)); 1701 1702 for (i = 0; i < drvdata->nr_ss_cmp; i++) { 1703 state->trcssccr[i] = etm4x_read32(csa, TRCSSCCRn(i)); 1704 state->trcsscsr[i] = etm4x_read32(csa, TRCSSCSRn(i)); 1705 if (etm4x_sspcicrn_present(drvdata, i)) 1706 state->trcsspcicr[i] = etm4x_read32(csa, TRCSSPCICRn(i)); 1707 } 1708 1709 for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { 1710 state->trcacvr[i] = etm4x_read64(csa, TRCACVRn(i)); 1711 state->trcacatr[i] = etm4x_read64(csa, TRCACATRn(i)); 1712 } 1713 1714 /* 1715 * Data trace stream is architecturally prohibited for A profile cores 1716 * so we don't save (or later restore) trcdvcvr and trcdvcmr - As per 1717 * section 1.3.4 ("Possible functional configurations of an ETMv4 trace 1718 * unit") of ARM IHI 0064D. 1719 */ 1720 1721 for (i = 0; i < drvdata->numcidc; i++) 1722 state->trccidcvr[i] = etm4x_read64(csa, TRCCIDCVRn(i)); 1723 1724 for (i = 0; i < drvdata->numvmidc; i++) 1725 state->trcvmidcvr[i] = etm4x_read64(csa, TRCVMIDCVRn(i)); 1726 1727 state->trccidcctlr0 = etm4x_read32(csa, TRCCIDCCTLR0); 1728 if (drvdata->numcidc > 4) 1729 state->trccidcctlr1 = etm4x_read32(csa, TRCCIDCCTLR1); 1730 1731 state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0); 1732 if (drvdata->numvmidc > 4) 1733 state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1); 1734 1735 state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR); 1736 1737 if (!drvdata->skip_power_up) 1738 state->trcpdcr = etm4x_read32(csa, TRCPDCR); 1739 1740 /* wait for TRCSTATR.IDLE to go up */ 1741 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) { 1742 dev_err(etm_dev, 1743 "timeout while waiting for Idle Trace Status\n"); 1744 etm4_os_unlock(drvdata); 1745 ret = -EBUSY; 1746 goto out; 1747 } 1748 1749 drvdata->state_needs_restore = true; 1750 1751 /* 1752 * Power can be removed from the trace unit now. We do this to 1753 * potentially save power on systems that respect the TRCPDCR_PU 1754 * despite requesting software to save/restore state. 1755 */ 1756 if (!drvdata->skip_power_up) 1757 etm4x_relaxed_write32(csa, (state->trcpdcr & ~TRCPDCR_PU), 1758 TRCPDCR); 1759 out: 1760 etm4_cs_lock(drvdata, csa); 1761 return ret; 1762 } 1763 1764 static int etm4_cpu_save(struct etmv4_drvdata *drvdata) 1765 { 1766 int ret = 0; 1767 1768 /* Save the TRFCR irrespective of whether the ETM is ON */ 1769 if (drvdata->trfcr) 1770 drvdata->save_trfcr = read_trfcr(); 1771 /* 1772 * Save and restore the ETM Trace registers only if 1773 * the ETM is active. 1774 */ 1775 if (local_read(&drvdata->mode) && drvdata->save_state) 1776 ret = __etm4_cpu_save(drvdata); 1777 return ret; 1778 } 1779 1780 static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) 1781 { 1782 int i; 1783 struct etmv4_save_state *state = drvdata->save_state; 1784 struct csdev_access tmp_csa = CSDEV_ACCESS_IOMEM(drvdata->base); 1785 struct csdev_access *csa = &tmp_csa; 1786 1787 etm4_cs_unlock(drvdata, csa); 1788 etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); 1789 1790 etm4x_relaxed_write32(csa, state->trcprgctlr, TRCPRGCTLR); 1791 if (drvdata->nr_pe) 1792 etm4x_relaxed_write32(csa, state->trcprocselr, TRCPROCSELR); 1793 etm4x_relaxed_write32(csa, state->trcconfigr, TRCCONFIGR); 1794 etm4x_relaxed_write32(csa, state->trcauxctlr, TRCAUXCTLR); 1795 etm4x_relaxed_write32(csa, state->trceventctl0r, TRCEVENTCTL0R); 1796 etm4x_relaxed_write32(csa, state->trceventctl1r, TRCEVENTCTL1R); 1797 if (drvdata->stallctl) 1798 etm4x_relaxed_write32(csa, state->trcstallctlr, TRCSTALLCTLR); 1799 etm4x_relaxed_write32(csa, state->trctsctlr, TRCTSCTLR); 1800 etm4x_relaxed_write32(csa, state->trcsyncpr, TRCSYNCPR); 1801 etm4x_relaxed_write32(csa, state->trcccctlr, TRCCCCTLR); 1802 etm4x_relaxed_write32(csa, state->trcbbctlr, TRCBBCTLR); 1803 etm4x_relaxed_write32(csa, state->trctraceidr, TRCTRACEIDR); 1804 etm4x_relaxed_write32(csa, state->trcqctlr, TRCQCTLR); 1805 1806 etm4x_relaxed_write32(csa, state->trcvictlr, TRCVICTLR); 1807 etm4x_relaxed_write32(csa, state->trcviiectlr, TRCVIIECTLR); 1808 etm4x_relaxed_write32(csa, state->trcvissctlr, TRCVISSCTLR); 1809 if (drvdata->nr_pe_cmp) 1810 etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR); 1811 etm4x_relaxed_write32(csa, state->trcvdctlr, TRCVDCTLR); 1812 etm4x_relaxed_write32(csa, state->trcvdsacctlr, TRCVDSACCTLR); 1813 etm4x_relaxed_write32(csa, state->trcvdarcctlr, TRCVDARCCTLR); 1814 1815 for (i = 0; i < drvdata->nrseqstate - 1; i++) 1816 etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i)); 1817 1818 if (drvdata->nrseqstate) { 1819 etm4x_relaxed_write32(csa, state->trcseqrstevr, TRCSEQRSTEVR); 1820 etm4x_relaxed_write32(csa, state->trcseqstr, TRCSEQSTR); 1821 } 1822 etm4x_relaxed_write32(csa, state->trcextinselr, TRCEXTINSELR); 1823 1824 for (i = 0; i < drvdata->nr_cntr; i++) { 1825 etm4x_relaxed_write32(csa, state->trccntrldvr[i], TRCCNTRLDVRn(i)); 1826 etm4x_relaxed_write32(csa, state->trccntctlr[i], TRCCNTCTLRn(i)); 1827 etm4x_relaxed_write32(csa, state->trccntvr[i], TRCCNTVRn(i)); 1828 } 1829 1830 for (i = 0; i < drvdata->nr_resource * 2; i++) 1831 etm4x_relaxed_write32(csa, state->trcrsctlr[i], TRCRSCTLRn(i)); 1832 1833 for (i = 0; i < drvdata->nr_ss_cmp; i++) { 1834 etm4x_relaxed_write32(csa, state->trcssccr[i], TRCSSCCRn(i)); 1835 etm4x_relaxed_write32(csa, state->trcsscsr[i], TRCSSCSRn(i)); 1836 if (etm4x_sspcicrn_present(drvdata, i)) 1837 etm4x_relaxed_write32(csa, state->trcsspcicr[i], TRCSSPCICRn(i)); 1838 } 1839 1840 for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { 1841 etm4x_relaxed_write64(csa, state->trcacvr[i], TRCACVRn(i)); 1842 etm4x_relaxed_write64(csa, state->trcacatr[i], TRCACATRn(i)); 1843 } 1844 1845 for (i = 0; i < drvdata->numcidc; i++) 1846 etm4x_relaxed_write64(csa, state->trccidcvr[i], TRCCIDCVRn(i)); 1847 1848 for (i = 0; i < drvdata->numvmidc; i++) 1849 etm4x_relaxed_write64(csa, state->trcvmidcvr[i], TRCVMIDCVRn(i)); 1850 1851 etm4x_relaxed_write32(csa, state->trccidcctlr0, TRCCIDCCTLR0); 1852 if (drvdata->numcidc > 4) 1853 etm4x_relaxed_write32(csa, state->trccidcctlr1, TRCCIDCCTLR1); 1854 1855 etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0); 1856 if (drvdata->numvmidc > 4) 1857 etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1); 1858 1859 etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); 1860 1861 if (!drvdata->skip_power_up) 1862 etm4x_relaxed_write32(csa, state->trcpdcr, TRCPDCR); 1863 1864 drvdata->state_needs_restore = false; 1865 1866 /* 1867 * As recommended by section 4.3.7 ("Synchronization when using the 1868 * memory-mapped interface") of ARM IHI 0064D 1869 */ 1870 dsb(sy); 1871 isb(); 1872 1873 /* Unlock the OS lock to re-enable trace and external debug access */ 1874 etm4_os_unlock(drvdata); 1875 etm4_cs_lock(drvdata, csa); 1876 } 1877 1878 static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) 1879 { 1880 if (drvdata->trfcr) 1881 write_trfcr(drvdata->save_trfcr); 1882 if (drvdata->state_needs_restore) 1883 __etm4_cpu_restore(drvdata); 1884 } 1885 1886 static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, 1887 void *v) 1888 { 1889 struct etmv4_drvdata *drvdata; 1890 unsigned int cpu = smp_processor_id(); 1891 1892 if (!etmdrvdata[cpu]) 1893 return NOTIFY_OK; 1894 1895 drvdata = etmdrvdata[cpu]; 1896 1897 if (WARN_ON_ONCE(drvdata->cpu != cpu)) 1898 return NOTIFY_BAD; 1899 1900 switch (cmd) { 1901 case CPU_PM_ENTER: 1902 if (etm4_cpu_save(drvdata)) 1903 return NOTIFY_BAD; 1904 break; 1905 case CPU_PM_EXIT: 1906 case CPU_PM_ENTER_FAILED: 1907 etm4_cpu_restore(drvdata); 1908 break; 1909 default: 1910 return NOTIFY_DONE; 1911 } 1912 1913 return NOTIFY_OK; 1914 } 1915 1916 static struct notifier_block etm4_cpu_pm_nb = { 1917 .notifier_call = etm4_cpu_pm_notify, 1918 }; 1919 1920 /* Setup PM. Deals with error conditions and counts */ 1921 static int __init etm4_pm_setup(void) 1922 { 1923 int ret; 1924 1925 ret = cpu_pm_register_notifier(&etm4_cpu_pm_nb); 1926 if (ret) 1927 return ret; 1928 1929 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING, 1930 "arm/coresight4:starting", 1931 etm4_starting_cpu, etm4_dying_cpu); 1932 1933 if (ret) 1934 goto unregister_notifier; 1935 1936 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, 1937 "arm/coresight4:online", 1938 etm4_online_cpu, NULL); 1939 1940 /* HP dyn state ID returned in ret on success */ 1941 if (ret > 0) { 1942 hp_online = ret; 1943 return 0; 1944 } 1945 1946 /* failed dyn state - remove others */ 1947 cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); 1948 1949 unregister_notifier: 1950 cpu_pm_unregister_notifier(&etm4_cpu_pm_nb); 1951 return ret; 1952 } 1953 1954 static void etm4_pm_clear(void) 1955 { 1956 cpu_pm_unregister_notifier(&etm4_cpu_pm_nb); 1957 cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); 1958 if (hp_online) { 1959 cpuhp_remove_state_nocalls(hp_online); 1960 hp_online = 0; 1961 } 1962 } 1963 1964 static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg) 1965 { 1966 int ret; 1967 struct coresight_platform_data *pdata = NULL; 1968 struct device *dev = init_arg->dev; 1969 struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); 1970 struct coresight_desc desc = { 0 }; 1971 u8 major, minor; 1972 char *type_name; 1973 1974 if (!drvdata) 1975 return -EINVAL; 1976 1977 desc.access = *init_arg->csa; 1978 1979 if (!drvdata->arch) 1980 return -EINVAL; 1981 1982 /* TRCPDCR is not accessible with system instructions. */ 1983 if (!desc.access.io_mem || 1984 fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up")) 1985 drvdata->skip_power_up = true; 1986 1987 major = ETM_ARCH_MAJOR_VERSION(drvdata->arch); 1988 minor = ETM_ARCH_MINOR_VERSION(drvdata->arch); 1989 1990 if (etm4x_is_ete(drvdata)) { 1991 type_name = "ete"; 1992 /* ETE v1 has major version == 0b101. Adjust this for logging.*/ 1993 major -= 4; 1994 } else { 1995 type_name = "etm"; 1996 } 1997 1998 desc.name = devm_kasprintf(dev, GFP_KERNEL, 1999 "%s%d", type_name, drvdata->cpu); 2000 if (!desc.name) 2001 return -ENOMEM; 2002 2003 etm4_set_default(&drvdata->config); 2004 2005 pdata = coresight_get_platform_data(dev); 2006 if (IS_ERR(pdata)) 2007 return PTR_ERR(pdata); 2008 2009 dev->platform_data = pdata; 2010 2011 desc.type = CORESIGHT_DEV_TYPE_SOURCE; 2012 desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC; 2013 desc.ops = &etm4_cs_ops; 2014 desc.pdata = pdata; 2015 desc.dev = dev; 2016 desc.groups = coresight_etmv4_groups; 2017 drvdata->csdev = coresight_register(&desc); 2018 if (IS_ERR(drvdata->csdev)) 2019 return PTR_ERR(drvdata->csdev); 2020 2021 ret = etm_perf_symlink(drvdata->csdev, true); 2022 if (ret) { 2023 coresight_unregister(drvdata->csdev); 2024 return ret; 2025 } 2026 2027 /* register with config infrastructure & load any current features */ 2028 ret = etm4_cscfg_register(drvdata->csdev); 2029 if (ret) { 2030 coresight_unregister(drvdata->csdev); 2031 return ret; 2032 } 2033 2034 etmdrvdata[drvdata->cpu] = drvdata; 2035 2036 dev_info(&drvdata->csdev->dev, "CPU%d: %s v%d.%d initialized\n", 2037 drvdata->cpu, type_name, major, minor); 2038 2039 if (boot_enable) { 2040 coresight_enable(drvdata->csdev); 2041 drvdata->boot_enable = true; 2042 } 2043 2044 return 0; 2045 } 2046 2047 static int etm4_probe(struct device *dev, void __iomem *base, u32 etm_pid) 2048 { 2049 struct etmv4_drvdata *drvdata; 2050 struct csdev_access access = { 0 }; 2051 struct etm4_init_arg init_arg = { 0 }; 2052 struct etm4_init_arg *delayed; 2053 2054 drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); 2055 if (!drvdata) 2056 return -ENOMEM; 2057 2058 dev_set_drvdata(dev, drvdata); 2059 2060 if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE) 2061 pm_save_enable = coresight_loses_context_with_cpu(dev) ? 2062 PARAM_PM_SAVE_SELF_HOSTED : PARAM_PM_SAVE_NEVER; 2063 2064 if (pm_save_enable != PARAM_PM_SAVE_NEVER) { 2065 drvdata->save_state = devm_kmalloc(dev, 2066 sizeof(struct etmv4_save_state), GFP_KERNEL); 2067 if (!drvdata->save_state) 2068 return -ENOMEM; 2069 } 2070 2071 drvdata->base = base; 2072 2073 spin_lock_init(&drvdata->spinlock); 2074 2075 drvdata->cpu = coresight_get_cpu(dev); 2076 if (drvdata->cpu < 0) 2077 return drvdata->cpu; 2078 2079 init_arg.dev = dev; 2080 init_arg.csa = &access; 2081 init_arg.pid = etm_pid; 2082 2083 /* 2084 * Serialize against CPUHP callbacks to avoid race condition 2085 * between the smp call and saving the delayed probe. 2086 */ 2087 cpus_read_lock(); 2088 if (smp_call_function_single(drvdata->cpu, 2089 etm4_init_arch_data, &init_arg, 1)) { 2090 /* The CPU was offline, try again once it comes online. */ 2091 delayed = devm_kmalloc(dev, sizeof(*delayed), GFP_KERNEL); 2092 if (!delayed) { 2093 cpus_read_unlock(); 2094 return -ENOMEM; 2095 } 2096 2097 *delayed = init_arg; 2098 2099 per_cpu(delayed_probe, drvdata->cpu) = delayed; 2100 2101 cpus_read_unlock(); 2102 return 0; 2103 } 2104 cpus_read_unlock(); 2105 2106 return etm4_add_coresight_dev(&init_arg); 2107 } 2108 2109 static int etm4_probe_amba(struct amba_device *adev, const struct amba_id *id) 2110 { 2111 void __iomem *base; 2112 struct device *dev = &adev->dev; 2113 struct resource *res = &adev->res; 2114 int ret; 2115 2116 /* Validity for the resource is already checked by the AMBA core */ 2117 base = devm_ioremap_resource(dev, res); 2118 if (IS_ERR(base)) 2119 return PTR_ERR(base); 2120 2121 ret = etm4_probe(dev, base, id->id); 2122 if (!ret) 2123 pm_runtime_put(&adev->dev); 2124 2125 return ret; 2126 } 2127 2128 static int etm4_probe_platform_dev(struct platform_device *pdev) 2129 { 2130 int ret; 2131 2132 pm_runtime_get_noresume(&pdev->dev); 2133 pm_runtime_set_active(&pdev->dev); 2134 pm_runtime_enable(&pdev->dev); 2135 2136 /* 2137 * System register based devices could match the 2138 * HW by reading appropriate registers on the HW 2139 * and thus we could skip the PID. 2140 */ 2141 ret = etm4_probe(&pdev->dev, NULL, 0); 2142 2143 pm_runtime_put(&pdev->dev); 2144 return ret; 2145 } 2146 2147 static int etm4_probe_cpu(unsigned int cpu) 2148 { 2149 int ret; 2150 struct etm4_init_arg init_arg; 2151 struct csdev_access access = { 0 }; 2152 struct etm4_init_arg *iap = *this_cpu_ptr(&delayed_probe); 2153 2154 if (!iap) 2155 return 0; 2156 2157 init_arg = *iap; 2158 devm_kfree(init_arg.dev, iap); 2159 *this_cpu_ptr(&delayed_probe) = NULL; 2160 2161 ret = pm_runtime_resume_and_get(init_arg.dev); 2162 if (ret < 0) { 2163 dev_err(init_arg.dev, "Failed to get PM runtime!\n"); 2164 return 0; 2165 } 2166 2167 init_arg.csa = &access; 2168 etm4_init_arch_data(&init_arg); 2169 2170 etm4_add_coresight_dev(&init_arg); 2171 2172 pm_runtime_put(init_arg.dev); 2173 return 0; 2174 } 2175 2176 static struct amba_cs_uci_id uci_id_etm4[] = { 2177 { 2178 /* ETMv4 UCI data */ 2179 .devarch = ETM_DEVARCH_ETMv4x_ARCH, 2180 .devarch_mask = ETM_DEVARCH_ID_MASK, 2181 .devtype = 0x00000013, 2182 } 2183 }; 2184 2185 static void clear_etmdrvdata(void *info) 2186 { 2187 int cpu = *(int *)info; 2188 2189 etmdrvdata[cpu] = NULL; 2190 per_cpu(delayed_probe, cpu) = NULL; 2191 } 2192 2193 static int __exit etm4_remove_dev(struct etmv4_drvdata *drvdata) 2194 { 2195 bool had_delayed_probe; 2196 /* 2197 * Taking hotplug lock here to avoid racing between etm4_remove_dev() 2198 * and CPU hotplug call backs. 2199 */ 2200 cpus_read_lock(); 2201 2202 had_delayed_probe = per_cpu(delayed_probe, drvdata->cpu); 2203 2204 /* 2205 * The readers for etmdrvdata[] are CPU hotplug call backs 2206 * and PM notification call backs. Change etmdrvdata[i] on 2207 * CPU i ensures these call backs has consistent view 2208 * inside one call back function. 2209 */ 2210 if (smp_call_function_single(drvdata->cpu, clear_etmdrvdata, &drvdata->cpu, 1)) 2211 clear_etmdrvdata(&drvdata->cpu); 2212 2213 cpus_read_unlock(); 2214 2215 if (!had_delayed_probe) { 2216 etm_perf_symlink(drvdata->csdev, false); 2217 cscfg_unregister_csdev(drvdata->csdev); 2218 coresight_unregister(drvdata->csdev); 2219 } 2220 2221 return 0; 2222 } 2223 2224 static void __exit etm4_remove_amba(struct amba_device *adev) 2225 { 2226 struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev); 2227 2228 if (drvdata) 2229 etm4_remove_dev(drvdata); 2230 } 2231 2232 static int __exit etm4_remove_platform_dev(struct platform_device *pdev) 2233 { 2234 int ret = 0; 2235 struct etmv4_drvdata *drvdata = dev_get_drvdata(&pdev->dev); 2236 2237 if (drvdata) 2238 ret = etm4_remove_dev(drvdata); 2239 pm_runtime_disable(&pdev->dev); 2240 return ret; 2241 } 2242 2243 static const struct amba_id etm4_ids[] = { 2244 CS_AMBA_ID(0x000bb95d), /* Cortex-A53 */ 2245 CS_AMBA_ID(0x000bb95e), /* Cortex-A57 */ 2246 CS_AMBA_ID(0x000bb95a), /* Cortex-A72 */ 2247 CS_AMBA_ID(0x000bb959), /* Cortex-A73 */ 2248 CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),/* Cortex-A35 */ 2249 CS_AMBA_UCI_ID(0x000bbd05, uci_id_etm4),/* Cortex-A55 */ 2250 CS_AMBA_UCI_ID(0x000bbd0a, uci_id_etm4),/* Cortex-A75 */ 2251 CS_AMBA_UCI_ID(0x000bbd0c, uci_id_etm4),/* Neoverse N1 */ 2252 CS_AMBA_UCI_ID(0x000bbd41, uci_id_etm4),/* Cortex-A78 */ 2253 CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),/* Qualcomm Kryo */ 2254 CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),/* Qualcomm Kryo */ 2255 CS_AMBA_UCI_ID(0x000bb802, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A55 */ 2256 CS_AMBA_UCI_ID(0x000bb803, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A75 */ 2257 CS_AMBA_UCI_ID(0x000bb805, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A55 */ 2258 CS_AMBA_UCI_ID(0x000bb804, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A76 */ 2259 CS_AMBA_UCI_ID(0x000bbd0d, uci_id_etm4),/* Qualcomm Kryo 5XX Cortex-A77 */ 2260 CS_AMBA_UCI_ID(0x000cc0af, uci_id_etm4),/* Marvell ThunderX2 */ 2261 CS_AMBA_UCI_ID(0x000b6d01, uci_id_etm4),/* HiSilicon-Hip08 */ 2262 CS_AMBA_UCI_ID(0x000b6d02, uci_id_etm4),/* HiSilicon-Hip09 */ 2263 {}, 2264 }; 2265 2266 MODULE_DEVICE_TABLE(amba, etm4_ids); 2267 2268 static struct amba_driver etm4x_amba_driver = { 2269 .drv = { 2270 .name = "coresight-etm4x", 2271 .owner = THIS_MODULE, 2272 .suppress_bind_attrs = true, 2273 }, 2274 .probe = etm4_probe_amba, 2275 .remove = etm4_remove_amba, 2276 .id_table = etm4_ids, 2277 }; 2278 2279 static const struct of_device_id etm4_sysreg_match[] = { 2280 { .compatible = "arm,coresight-etm4x-sysreg" }, 2281 { .compatible = "arm,embedded-trace-extension" }, 2282 {} 2283 }; 2284 2285 static struct platform_driver etm4_platform_driver = { 2286 .probe = etm4_probe_platform_dev, 2287 .remove = etm4_remove_platform_dev, 2288 .driver = { 2289 .name = "coresight-etm4x", 2290 .of_match_table = etm4_sysreg_match, 2291 .suppress_bind_attrs = true, 2292 }, 2293 }; 2294 2295 static int __init etm4x_init(void) 2296 { 2297 int ret; 2298 2299 ret = etm4_pm_setup(); 2300 2301 /* etm4_pm_setup() does its own cleanup - exit on error */ 2302 if (ret) 2303 return ret; 2304 2305 ret = amba_driver_register(&etm4x_amba_driver); 2306 if (ret) { 2307 pr_err("Error registering etm4x AMBA driver\n"); 2308 goto clear_pm; 2309 } 2310 2311 ret = platform_driver_register(&etm4_platform_driver); 2312 if (!ret) 2313 return 0; 2314 2315 pr_err("Error registering etm4x platform driver\n"); 2316 amba_driver_unregister(&etm4x_amba_driver); 2317 2318 clear_pm: 2319 etm4_pm_clear(); 2320 return ret; 2321 } 2322 2323 static void __exit etm4x_exit(void) 2324 { 2325 amba_driver_unregister(&etm4x_amba_driver); 2326 platform_driver_unregister(&etm4_platform_driver); 2327 etm4_pm_clear(); 2328 } 2329 2330 module_init(etm4x_init); 2331 module_exit(etm4x_exit); 2332 2333 MODULE_AUTHOR("Pratik Patel <pratikp@codeaurora.org>"); 2334 MODULE_AUTHOR("Mathieu Poirier <mathieu.poirier@linaro.org>"); 2335 MODULE_DESCRIPTION("Arm CoreSight Program Flow Trace v4.x driver"); 2336 MODULE_LICENSE("GPL v2"); 2337