1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2014, The Linux Foundation. All rights reserved. 4 */ 5 6 #include <linux/bitops.h> 7 #include <linux/kernel.h> 8 #include <linux/moduleparam.h> 9 #include <linux/init.h> 10 #include <linux/types.h> 11 #include <linux/device.h> 12 #include <linux/io.h> 13 #include <linux/err.h> 14 #include <linux/fs.h> 15 #include <linux/slab.h> 16 #include <linux/delay.h> 17 #include <linux/smp.h> 18 #include <linux/sysfs.h> 19 #include <linux/stat.h> 20 #include <linux/clk.h> 21 #include <linux/cpu.h> 22 #include <linux/cpu_pm.h> 23 #include <linux/coresight.h> 24 #include <linux/coresight-pmu.h> 25 #include <linux/pm_wakeup.h> 26 #include <linux/amba/bus.h> 27 #include <linux/seq_file.h> 28 #include <linux/uaccess.h> 29 #include <linux/perf_event.h> 30 #include <linux/platform_device.h> 31 #include <linux/pm_runtime.h> 32 #include <linux/property.h> 33 34 #include <asm/barrier.h> 35 #include <asm/sections.h> 36 #include <asm/sysreg.h> 37 #include <asm/local.h> 38 #include <asm/virt.h> 39 40 #include "coresight-etm4x.h" 41 #include "coresight-etm-perf.h" 42 #include "coresight-etm4x-cfg.h" 43 #include "coresight-self-hosted-trace.h" 44 #include "coresight-syscfg.h" 45 #include "coresight-trace-id.h" 46 47 static int boot_enable; 48 module_param(boot_enable, int, 0444); 49 MODULE_PARM_DESC(boot_enable, "Enable tracing on boot"); 50 51 #define PARAM_PM_SAVE_FIRMWARE 0 /* save self-hosted state as per firmware */ 52 #define PARAM_PM_SAVE_NEVER 1 /* never save any state */ 53 #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */ 54 55 static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE; 56 module_param(pm_save_enable, int, 0444); 57 MODULE_PARM_DESC(pm_save_enable, 58 "Save/restore state on power down: 1 = never, 2 = self-hosted"); 59 60 static struct etmv4_drvdata *etmdrvdata[NR_CPUS]; 61 static void etm4_set_default_config(struct etmv4_config *config); 62 static int etm4_set_event_filters(struct etmv4_drvdata *drvdata, 63 struct perf_event *event); 64 static u64 etm4_get_access_type(struct etmv4_config *config); 65 66 static enum cpuhp_state hp_online; 67 68 struct etm4_init_arg { 69 unsigned int pid; 70 struct device *dev; 71 struct csdev_access *csa; 72 }; 73 74 static DEFINE_PER_CPU(struct etm4_init_arg *, delayed_probe); 75 static int etm4_probe_cpu(unsigned int cpu); 76 77 /* 78 * Check if TRCSSPCICRn(i) is implemented for a given instance. 79 * 80 * TRCSSPCICRn is implemented only if : 81 * TRCSSPCICR<n> is present only if all of the following are true: 82 * TRCIDR4.NUMSSCC > n. 83 * TRCIDR4.NUMPC > 0b0000 . 84 * TRCSSCSR<n>.PC == 0b1 85 */ 86 static inline bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n) 87 { 88 return (n < drvdata->nr_ss_cmp) && 89 drvdata->nr_pe && 90 (drvdata->config.ss_status[n] & TRCSSCSRn_PC); 91 } 92 93 u64 etm4x_sysreg_read(u32 offset, bool _relaxed, bool _64bit) 94 { 95 u64 res = 0; 96 97 switch (offset) { 98 ETM4x_READ_SYSREG_CASES(res) 99 default : 100 pr_warn_ratelimited("etm4x: trying to read unsupported register @%x\n", 101 offset); 102 } 103 104 if (!_relaxed) 105 __io_ar(res); /* Imitate the !relaxed I/O helpers */ 106 107 return res; 108 } 109 110 void etm4x_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit) 111 { 112 if (!_relaxed) 113 __io_bw(); /* Imitate the !relaxed I/O helpers */ 114 if (!_64bit) 115 val &= GENMASK(31, 0); 116 117 switch (offset) { 118 ETM4x_WRITE_SYSREG_CASES(val) 119 default : 120 pr_warn_ratelimited("etm4x: trying to write to unsupported register @%x\n", 121 offset); 122 } 123 } 124 125 static u64 ete_sysreg_read(u32 offset, bool _relaxed, bool _64bit) 126 { 127 u64 res = 0; 128 129 switch (offset) { 130 ETE_READ_CASES(res) 131 default : 132 pr_warn_ratelimited("ete: trying to read unsupported register @%x\n", 133 offset); 134 } 135 136 if (!_relaxed) 137 __io_ar(res); /* Imitate the !relaxed I/O helpers */ 138 139 return res; 140 } 141 142 static void ete_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit) 143 { 144 if (!_relaxed) 145 __io_bw(); /* Imitate the !relaxed I/O helpers */ 146 if (!_64bit) 147 val &= GENMASK(31, 0); 148 149 switch (offset) { 150 ETE_WRITE_CASES(val) 151 default : 152 pr_warn_ratelimited("ete: trying to write to unsupported register @%x\n", 153 offset); 154 } 155 } 156 157 static void etm_detect_os_lock(struct etmv4_drvdata *drvdata, 158 struct csdev_access *csa) 159 { 160 u32 oslsr = etm4x_relaxed_read32(csa, TRCOSLSR); 161 162 drvdata->os_lock_model = ETM_OSLSR_OSLM(oslsr); 163 } 164 165 static void etm_write_os_lock(struct etmv4_drvdata *drvdata, 166 struct csdev_access *csa, u32 val) 167 { 168 val = !!val; 169 170 switch (drvdata->os_lock_model) { 171 case ETM_OSLOCK_PRESENT: 172 etm4x_relaxed_write32(csa, val, TRCOSLAR); 173 break; 174 case ETM_OSLOCK_PE: 175 write_sysreg_s(val, SYS_OSLAR_EL1); 176 break; 177 default: 178 pr_warn_once("CPU%d: Unsupported Trace OSLock model: %x\n", 179 smp_processor_id(), drvdata->os_lock_model); 180 fallthrough; 181 case ETM_OSLOCK_NI: 182 return; 183 } 184 isb(); 185 } 186 187 static inline void etm4_os_unlock_csa(struct etmv4_drvdata *drvdata, 188 struct csdev_access *csa) 189 { 190 WARN_ON(drvdata->cpu != smp_processor_id()); 191 192 /* Writing 0 to OS Lock unlocks the trace unit registers */ 193 etm_write_os_lock(drvdata, csa, 0x0); 194 drvdata->os_unlock = true; 195 } 196 197 static void etm4_os_unlock(struct etmv4_drvdata *drvdata) 198 { 199 if (!WARN_ON(!drvdata->csdev)) 200 etm4_os_unlock_csa(drvdata, &drvdata->csdev->access); 201 } 202 203 static void etm4_os_lock(struct etmv4_drvdata *drvdata) 204 { 205 if (WARN_ON(!drvdata->csdev)) 206 return; 207 /* Writing 0x1 to OS Lock locks the trace registers */ 208 etm_write_os_lock(drvdata, &drvdata->csdev->access, 0x1); 209 drvdata->os_unlock = false; 210 } 211 212 static void etm4_cs_lock(struct etmv4_drvdata *drvdata, 213 struct csdev_access *csa) 214 { 215 /* Software Lock is only accessible via memory mapped interface */ 216 if (csa->io_mem) 217 CS_LOCK(csa->base); 218 } 219 220 static void etm4_cs_unlock(struct etmv4_drvdata *drvdata, 221 struct csdev_access *csa) 222 { 223 if (csa->io_mem) 224 CS_UNLOCK(csa->base); 225 } 226 227 static int etm4_cpu_id(struct coresight_device *csdev) 228 { 229 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 230 231 return drvdata->cpu; 232 } 233 234 int etm4_read_alloc_trace_id(struct etmv4_drvdata *drvdata) 235 { 236 int trace_id; 237 238 /* 239 * This will allocate a trace ID to the cpu, 240 * or return the one currently allocated. 241 * The trace id function has its own lock 242 */ 243 trace_id = coresight_trace_id_get_cpu_id(drvdata->cpu); 244 if (IS_VALID_CS_TRACE_ID(trace_id)) 245 drvdata->trcid = (u8)trace_id; 246 else 247 dev_err(&drvdata->csdev->dev, 248 "Failed to allocate trace ID for %s on CPU%d\n", 249 dev_name(&drvdata->csdev->dev), drvdata->cpu); 250 return trace_id; 251 } 252 253 void etm4_release_trace_id(struct etmv4_drvdata *drvdata) 254 { 255 coresight_trace_id_put_cpu_id(drvdata->cpu); 256 } 257 258 struct etm4_enable_arg { 259 struct etmv4_drvdata *drvdata; 260 int rc; 261 }; 262 263 /* 264 * etm4x_prohibit_trace - Prohibit the CPU from tracing at all ELs. 265 * When the CPU supports FEAT_TRF, we could move the ETM to a trace 266 * prohibited state by filtering the Exception levels via TRFCR_EL1. 267 */ 268 static void etm4x_prohibit_trace(struct etmv4_drvdata *drvdata) 269 { 270 /* If the CPU doesn't support FEAT_TRF, nothing to do */ 271 if (!drvdata->trfcr) 272 return; 273 cpu_prohibit_trace(); 274 } 275 276 /* 277 * etm4x_allow_trace - Allow CPU tracing in the respective ELs, 278 * as configured by the drvdata->config.mode for the current 279 * session. Even though we have TRCVICTLR bits to filter the 280 * trace in the ELs, it doesn't prevent the ETM from generating 281 * a packet (e.g, TraceInfo) that might contain the addresses from 282 * the excluded levels. Thus we use the additional controls provided 283 * via the Trace Filtering controls (FEAT_TRF) to make sure no trace 284 * is generated for the excluded ELs. 285 */ 286 static void etm4x_allow_trace(struct etmv4_drvdata *drvdata) 287 { 288 u64 trfcr = drvdata->trfcr; 289 290 /* If the CPU doesn't support FEAT_TRF, nothing to do */ 291 if (!trfcr) 292 return; 293 294 if (drvdata->config.mode & ETM_MODE_EXCL_KERN) 295 trfcr &= ~TRFCR_ELx_ExTRE; 296 if (drvdata->config.mode & ETM_MODE_EXCL_USER) 297 trfcr &= ~TRFCR_ELx_E0TRE; 298 299 write_trfcr(trfcr); 300 } 301 302 #ifdef CONFIG_ETM4X_IMPDEF_FEATURE 303 304 #define HISI_HIP08_AMBA_ID 0x000b6d01 305 #define ETM4_AMBA_MASK 0xfffff 306 #define HISI_HIP08_CORE_COMMIT_MASK 0x3000 307 #define HISI_HIP08_CORE_COMMIT_SHIFT 12 308 #define HISI_HIP08_CORE_COMMIT_FULL 0b00 309 #define HISI_HIP08_CORE_COMMIT_LVL_1 0b01 310 #define HISI_HIP08_CORE_COMMIT_REG sys_reg(3, 1, 15, 2, 5) 311 312 struct etm4_arch_features { 313 void (*arch_callback)(bool enable); 314 }; 315 316 static bool etm4_hisi_match_pid(unsigned int id) 317 { 318 return (id & ETM4_AMBA_MASK) == HISI_HIP08_AMBA_ID; 319 } 320 321 static void etm4_hisi_config_core_commit(bool enable) 322 { 323 u8 commit = enable ? HISI_HIP08_CORE_COMMIT_LVL_1 : 324 HISI_HIP08_CORE_COMMIT_FULL; 325 u64 val; 326 327 /* 328 * bit 12 and 13 of HISI_HIP08_CORE_COMMIT_REG are used together 329 * to set core-commit, 2'b00 means cpu is at full speed, 2'b01, 330 * 2'b10, 2'b11 mean reduce pipeline speed, and 2'b01 means level-1 331 * speed(minimun value). So bit 12 and 13 should be cleared together. 332 */ 333 val = read_sysreg_s(HISI_HIP08_CORE_COMMIT_REG); 334 val &= ~HISI_HIP08_CORE_COMMIT_MASK; 335 val |= commit << HISI_HIP08_CORE_COMMIT_SHIFT; 336 write_sysreg_s(val, HISI_HIP08_CORE_COMMIT_REG); 337 } 338 339 static struct etm4_arch_features etm4_features[] = { 340 [ETM4_IMPDEF_HISI_CORE_COMMIT] = { 341 .arch_callback = etm4_hisi_config_core_commit, 342 }, 343 {}, 344 }; 345 346 static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata) 347 { 348 struct etm4_arch_features *ftr; 349 int bit; 350 351 for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) { 352 ftr = &etm4_features[bit]; 353 354 if (ftr->arch_callback) 355 ftr->arch_callback(true); 356 } 357 } 358 359 static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata) 360 { 361 struct etm4_arch_features *ftr; 362 int bit; 363 364 for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) { 365 ftr = &etm4_features[bit]; 366 367 if (ftr->arch_callback) 368 ftr->arch_callback(false); 369 } 370 } 371 372 static void etm4_check_arch_features(struct etmv4_drvdata *drvdata, 373 unsigned int id) 374 { 375 if (etm4_hisi_match_pid(id)) 376 set_bit(ETM4_IMPDEF_HISI_CORE_COMMIT, drvdata->arch_features); 377 } 378 #else 379 static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata) 380 { 381 } 382 383 static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata) 384 { 385 } 386 387 static void etm4_check_arch_features(struct etmv4_drvdata *drvdata, 388 unsigned int id) 389 { 390 } 391 #endif /* CONFIG_ETM4X_IMPDEF_FEATURE */ 392 393 static int etm4_enable_hw(struct etmv4_drvdata *drvdata) 394 { 395 int i, rc; 396 struct etmv4_config *config = &drvdata->config; 397 struct coresight_device *csdev = drvdata->csdev; 398 struct device *etm_dev = &csdev->dev; 399 struct csdev_access *csa = &csdev->access; 400 401 402 etm4_cs_unlock(drvdata, csa); 403 etm4_enable_arch_specific(drvdata); 404 405 etm4_os_unlock(drvdata); 406 407 rc = coresight_claim_device_unlocked(csdev); 408 if (rc) 409 goto done; 410 411 /* Disable the trace unit before programming trace registers */ 412 etm4x_relaxed_write32(csa, 0, TRCPRGCTLR); 413 414 /* 415 * If we use system instructions, we need to synchronize the 416 * write to the TRCPRGCTLR, before accessing the TRCSTATR. 417 * See ARM IHI0064F, section 418 * "4.3.7 Synchronization of register updates" 419 */ 420 if (!csa->io_mem) 421 isb(); 422 423 /* wait for TRCSTATR.IDLE to go up */ 424 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) 425 dev_err(etm_dev, 426 "timeout while waiting for Idle Trace Status\n"); 427 if (drvdata->nr_pe) 428 etm4x_relaxed_write32(csa, config->pe_sel, TRCPROCSELR); 429 etm4x_relaxed_write32(csa, config->cfg, TRCCONFIGR); 430 /* nothing specific implemented */ 431 etm4x_relaxed_write32(csa, 0x0, TRCAUXCTLR); 432 etm4x_relaxed_write32(csa, config->eventctrl0, TRCEVENTCTL0R); 433 etm4x_relaxed_write32(csa, config->eventctrl1, TRCEVENTCTL1R); 434 if (drvdata->stallctl) 435 etm4x_relaxed_write32(csa, config->stall_ctrl, TRCSTALLCTLR); 436 etm4x_relaxed_write32(csa, config->ts_ctrl, TRCTSCTLR); 437 etm4x_relaxed_write32(csa, config->syncfreq, TRCSYNCPR); 438 etm4x_relaxed_write32(csa, config->ccctlr, TRCCCCTLR); 439 etm4x_relaxed_write32(csa, config->bb_ctrl, TRCBBCTLR); 440 etm4x_relaxed_write32(csa, drvdata->trcid, TRCTRACEIDR); 441 etm4x_relaxed_write32(csa, config->vinst_ctrl, TRCVICTLR); 442 etm4x_relaxed_write32(csa, config->viiectlr, TRCVIIECTLR); 443 etm4x_relaxed_write32(csa, config->vissctlr, TRCVISSCTLR); 444 if (drvdata->nr_pe_cmp) 445 etm4x_relaxed_write32(csa, config->vipcssctlr, TRCVIPCSSCTLR); 446 for (i = 0; i < drvdata->nrseqstate - 1; i++) 447 etm4x_relaxed_write32(csa, config->seq_ctrl[i], TRCSEQEVRn(i)); 448 if (drvdata->nrseqstate) { 449 etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR); 450 etm4x_relaxed_write32(csa, config->seq_state, TRCSEQSTR); 451 } 452 etm4x_relaxed_write32(csa, config->ext_inp, TRCEXTINSELR); 453 for (i = 0; i < drvdata->nr_cntr; i++) { 454 etm4x_relaxed_write32(csa, config->cntrldvr[i], TRCCNTRLDVRn(i)); 455 etm4x_relaxed_write32(csa, config->cntr_ctrl[i], TRCCNTCTLRn(i)); 456 etm4x_relaxed_write32(csa, config->cntr_val[i], TRCCNTVRn(i)); 457 } 458 459 /* 460 * Resource selector pair 0 is always implemented and reserved. As 461 * such start at 2. 462 */ 463 for (i = 2; i < drvdata->nr_resource * 2; i++) 464 etm4x_relaxed_write32(csa, config->res_ctrl[i], TRCRSCTLRn(i)); 465 466 for (i = 0; i < drvdata->nr_ss_cmp; i++) { 467 /* always clear status bit on restart if using single-shot */ 468 if (config->ss_ctrl[i] || config->ss_pe_cmp[i]) 469 config->ss_status[i] &= ~TRCSSCSRn_STATUS; 470 etm4x_relaxed_write32(csa, config->ss_ctrl[i], TRCSSCCRn(i)); 471 etm4x_relaxed_write32(csa, config->ss_status[i], TRCSSCSRn(i)); 472 if (etm4x_sspcicrn_present(drvdata, i)) 473 etm4x_relaxed_write32(csa, config->ss_pe_cmp[i], TRCSSPCICRn(i)); 474 } 475 for (i = 0; i < drvdata->nr_addr_cmp; i++) { 476 etm4x_relaxed_write64(csa, config->addr_val[i], TRCACVRn(i)); 477 etm4x_relaxed_write64(csa, config->addr_acc[i], TRCACATRn(i)); 478 } 479 for (i = 0; i < drvdata->numcidc; i++) 480 etm4x_relaxed_write64(csa, config->ctxid_pid[i], TRCCIDCVRn(i)); 481 etm4x_relaxed_write32(csa, config->ctxid_mask0, TRCCIDCCTLR0); 482 if (drvdata->numcidc > 4) 483 etm4x_relaxed_write32(csa, config->ctxid_mask1, TRCCIDCCTLR1); 484 485 for (i = 0; i < drvdata->numvmidc; i++) 486 etm4x_relaxed_write64(csa, config->vmid_val[i], TRCVMIDCVRn(i)); 487 etm4x_relaxed_write32(csa, config->vmid_mask0, TRCVMIDCCTLR0); 488 if (drvdata->numvmidc > 4) 489 etm4x_relaxed_write32(csa, config->vmid_mask1, TRCVMIDCCTLR1); 490 491 if (!drvdata->skip_power_up) { 492 u32 trcpdcr = etm4x_relaxed_read32(csa, TRCPDCR); 493 494 /* 495 * Request to keep the trace unit powered and also 496 * emulation of powerdown 497 */ 498 etm4x_relaxed_write32(csa, trcpdcr | TRCPDCR_PU, TRCPDCR); 499 } 500 501 /* 502 * ETE mandates that the TRCRSR is written to before 503 * enabling it. 504 */ 505 if (etm4x_is_ete(drvdata)) 506 etm4x_relaxed_write32(csa, TRCRSR_TA, TRCRSR); 507 508 etm4x_allow_trace(drvdata); 509 /* Enable the trace unit */ 510 etm4x_relaxed_write32(csa, 1, TRCPRGCTLR); 511 512 /* Synchronize the register updates for sysreg access */ 513 if (!csa->io_mem) 514 isb(); 515 516 /* wait for TRCSTATR.IDLE to go back down to '0' */ 517 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 0)) 518 dev_err(etm_dev, 519 "timeout while waiting for Idle Trace Status\n"); 520 521 /* 522 * As recommended by section 4.3.7 ("Synchronization when using the 523 * memory-mapped interface") of ARM IHI 0064D 524 */ 525 dsb(sy); 526 isb(); 527 528 done: 529 etm4_cs_lock(drvdata, csa); 530 531 dev_dbg(etm_dev, "cpu: %d enable smp call done: %d\n", 532 drvdata->cpu, rc); 533 return rc; 534 } 535 536 static void etm4_enable_hw_smp_call(void *info) 537 { 538 struct etm4_enable_arg *arg = info; 539 540 if (WARN_ON(!arg)) 541 return; 542 arg->rc = etm4_enable_hw(arg->drvdata); 543 } 544 545 /* 546 * The goal of function etm4_config_timestamp_event() is to configure a 547 * counter that will tell the tracer to emit a timestamp packet when it 548 * reaches zero. This is done in order to get a more fine grained idea 549 * of when instructions are executed so that they can be correlated 550 * with execution on other CPUs. 551 * 552 * To do this the counter itself is configured to self reload and 553 * TRCRSCTLR1 (always true) used to get the counter to decrement. From 554 * there a resource selector is configured with the counter and the 555 * timestamp control register to use the resource selector to trigger the 556 * event that will insert a timestamp packet in the stream. 557 */ 558 static int etm4_config_timestamp_event(struct etmv4_drvdata *drvdata) 559 { 560 int ctridx, ret = -EINVAL; 561 int counter, rselector; 562 u32 val = 0; 563 struct etmv4_config *config = &drvdata->config; 564 565 /* No point in trying if we don't have at least one counter */ 566 if (!drvdata->nr_cntr) 567 goto out; 568 569 /* Find a counter that hasn't been initialised */ 570 for (ctridx = 0; ctridx < drvdata->nr_cntr; ctridx++) 571 if (config->cntr_val[ctridx] == 0) 572 break; 573 574 /* All the counters have been configured already, bail out */ 575 if (ctridx == drvdata->nr_cntr) { 576 pr_debug("%s: no available counter found\n", __func__); 577 ret = -ENOSPC; 578 goto out; 579 } 580 581 /* 582 * Searching for an available resource selector to use, starting at 583 * '2' since every implementation has at least 2 resource selector. 584 * ETMIDR4 gives the number of resource selector _pairs_, 585 * hence multiply by 2. 586 */ 587 for (rselector = 2; rselector < drvdata->nr_resource * 2; rselector++) 588 if (!config->res_ctrl[rselector]) 589 break; 590 591 if (rselector == drvdata->nr_resource * 2) { 592 pr_debug("%s: no available resource selector found\n", 593 __func__); 594 ret = -ENOSPC; 595 goto out; 596 } 597 598 /* Remember what counter we used */ 599 counter = 1 << ctridx; 600 601 /* 602 * Initialise original and reload counter value to the smallest 603 * possible value in order to get as much precision as we can. 604 */ 605 config->cntr_val[ctridx] = 1; 606 config->cntrldvr[ctridx] = 1; 607 608 /* Set the trace counter control register */ 609 val = 0x1 << 16 | /* Bit 16, reload counter automatically */ 610 0x0 << 7 | /* Select single resource selector */ 611 0x1; /* Resource selector 1, i.e always true */ 612 613 config->cntr_ctrl[ctridx] = val; 614 615 val = 0x2 << 16 | /* Group 0b0010 - Counter and sequencers */ 616 counter << 0; /* Counter to use */ 617 618 config->res_ctrl[rselector] = val; 619 620 val = 0x0 << 7 | /* Select single resource selector */ 621 rselector; /* Resource selector */ 622 623 config->ts_ctrl = val; 624 625 ret = 0; 626 out: 627 return ret; 628 } 629 630 static int etm4_parse_event_config(struct coresight_device *csdev, 631 struct perf_event *event) 632 { 633 int ret = 0; 634 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 635 struct etmv4_config *config = &drvdata->config; 636 struct perf_event_attr *attr = &event->attr; 637 unsigned long cfg_hash; 638 int preset; 639 640 /* Clear configuration from previous run */ 641 memset(config, 0, sizeof(struct etmv4_config)); 642 643 if (attr->exclude_kernel) 644 config->mode = ETM_MODE_EXCL_KERN; 645 646 if (attr->exclude_user) 647 config->mode = ETM_MODE_EXCL_USER; 648 649 /* Always start from the default config */ 650 etm4_set_default_config(config); 651 652 /* Configure filters specified on the perf cmd line, if any. */ 653 ret = etm4_set_event_filters(drvdata, event); 654 if (ret) 655 goto out; 656 657 /* Go from generic option to ETMv4 specifics */ 658 if (attr->config & BIT(ETM_OPT_CYCACC)) { 659 config->cfg |= TRCCONFIGR_CCI; 660 /* TRM: Must program this for cycacc to work */ 661 config->ccctlr = ETM_CYC_THRESHOLD_DEFAULT; 662 } 663 if (attr->config & BIT(ETM_OPT_TS)) { 664 /* 665 * Configure timestamps to be emitted at regular intervals in 666 * order to correlate instructions executed on different CPUs 667 * (CPU-wide trace scenarios). 668 */ 669 ret = etm4_config_timestamp_event(drvdata); 670 671 /* 672 * No need to go further if timestamp intervals can't 673 * be configured. 674 */ 675 if (ret) 676 goto out; 677 678 /* bit[11], Global timestamp tracing bit */ 679 config->cfg |= TRCCONFIGR_TS; 680 } 681 682 /* Only trace contextID when runs in root PID namespace */ 683 if ((attr->config & BIT(ETM_OPT_CTXTID)) && 684 task_is_in_init_pid_ns(current)) 685 /* bit[6], Context ID tracing bit */ 686 config->cfg |= TRCCONFIGR_CID; 687 688 /* 689 * If set bit ETM_OPT_CTXTID2 in perf config, this asks to trace VMID 690 * for recording CONTEXTIDR_EL2. Do not enable VMID tracing if the 691 * kernel is not running in EL2. 692 */ 693 if (attr->config & BIT(ETM_OPT_CTXTID2)) { 694 if (!is_kernel_in_hyp_mode()) { 695 ret = -EINVAL; 696 goto out; 697 } 698 /* Only trace virtual contextID when runs in root PID namespace */ 699 if (task_is_in_init_pid_ns(current)) 700 config->cfg |= TRCCONFIGR_VMID | TRCCONFIGR_VMIDOPT; 701 } 702 703 /* return stack - enable if selected and supported */ 704 if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack) 705 /* bit[12], Return stack enable bit */ 706 config->cfg |= TRCCONFIGR_RS; 707 708 /* 709 * Set any selected configuration and preset. 710 * 711 * This extracts the values of PMU_FORMAT_ATTR(configid) and PMU_FORMAT_ATTR(preset) 712 * in the perf attributes defined in coresight-etm-perf.c. 713 * configid uses bits 63:32 of attr->config2, preset uses bits 3:0 of attr->config. 714 * A zero configid means no configuration active, preset = 0 means no preset selected. 715 */ 716 if (attr->config2 & GENMASK_ULL(63, 32)) { 717 cfg_hash = (u32)(attr->config2 >> 32); 718 preset = attr->config & 0xF; 719 ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); 720 } 721 722 /* branch broadcast - enable if selected and supported */ 723 if (attr->config & BIT(ETM_OPT_BRANCH_BROADCAST)) { 724 if (!drvdata->trcbb) { 725 /* 726 * Missing BB support could cause silent decode errors 727 * so fail to open if it's not supported. 728 */ 729 ret = -EINVAL; 730 goto out; 731 } else { 732 config->cfg |= BIT(ETM4_CFG_BIT_BB); 733 } 734 } 735 736 out: 737 return ret; 738 } 739 740 static int etm4_enable_perf(struct coresight_device *csdev, 741 struct perf_event *event) 742 { 743 int ret = 0, trace_id; 744 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 745 746 if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) { 747 ret = -EINVAL; 748 goto out; 749 } 750 751 /* Configure the tracer based on the session's specifics */ 752 ret = etm4_parse_event_config(csdev, event); 753 if (ret) 754 goto out; 755 756 /* 757 * perf allocates cpu ids as part of _setup_aux() - device needs to use 758 * the allocated ID. This reads the current version without allocation. 759 * 760 * This does not use the trace id lock to prevent lock_dep issues 761 * with perf locks - we know the ID cannot change until perf shuts down 762 * the session 763 */ 764 trace_id = coresight_trace_id_read_cpu_id(drvdata->cpu); 765 if (!IS_VALID_CS_TRACE_ID(trace_id)) { 766 dev_err(&drvdata->csdev->dev, "Failed to set trace ID for %s on CPU%d\n", 767 dev_name(&drvdata->csdev->dev), drvdata->cpu); 768 ret = -EINVAL; 769 goto out; 770 } 771 drvdata->trcid = (u8)trace_id; 772 773 /* And enable it */ 774 ret = etm4_enable_hw(drvdata); 775 776 out: 777 return ret; 778 } 779 780 static int etm4_enable_sysfs(struct coresight_device *csdev) 781 { 782 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 783 struct etm4_enable_arg arg = { }; 784 unsigned long cfg_hash; 785 int ret, preset; 786 787 /* enable any config activated by configfs */ 788 cscfg_config_sysfs_get_active_cfg(&cfg_hash, &preset); 789 if (cfg_hash) { 790 ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); 791 if (ret) 792 return ret; 793 } 794 795 spin_lock(&drvdata->spinlock); 796 797 /* sysfs needs to read and allocate a trace ID */ 798 ret = etm4_read_alloc_trace_id(drvdata); 799 if (ret < 0) 800 goto unlock_sysfs_enable; 801 802 /* 803 * Executing etm4_enable_hw on the cpu whose ETM is being enabled 804 * ensures that register writes occur when cpu is powered. 805 */ 806 arg.drvdata = drvdata; 807 ret = smp_call_function_single(drvdata->cpu, 808 etm4_enable_hw_smp_call, &arg, 1); 809 if (!ret) 810 ret = arg.rc; 811 if (!ret) 812 drvdata->sticky_enable = true; 813 814 if (ret) 815 etm4_release_trace_id(drvdata); 816 817 unlock_sysfs_enable: 818 spin_unlock(&drvdata->spinlock); 819 820 if (!ret) 821 dev_dbg(&csdev->dev, "ETM tracing enabled\n"); 822 return ret; 823 } 824 825 static int etm4_enable(struct coresight_device *csdev, 826 struct perf_event *event, u32 mode) 827 { 828 int ret; 829 u32 val; 830 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 831 832 val = local_cmpxchg(&drvdata->mode, CS_MODE_DISABLED, mode); 833 834 /* Someone is already using the tracer */ 835 if (val) 836 return -EBUSY; 837 838 switch (mode) { 839 case CS_MODE_SYSFS: 840 ret = etm4_enable_sysfs(csdev); 841 break; 842 case CS_MODE_PERF: 843 ret = etm4_enable_perf(csdev, event); 844 break; 845 default: 846 ret = -EINVAL; 847 } 848 849 /* The tracer didn't start */ 850 if (ret) 851 local_set(&drvdata->mode, CS_MODE_DISABLED); 852 853 return ret; 854 } 855 856 static void etm4_disable_hw(void *info) 857 { 858 u32 control; 859 struct etmv4_drvdata *drvdata = info; 860 struct etmv4_config *config = &drvdata->config; 861 struct coresight_device *csdev = drvdata->csdev; 862 struct device *etm_dev = &csdev->dev; 863 struct csdev_access *csa = &csdev->access; 864 int i; 865 866 etm4_cs_unlock(drvdata, csa); 867 etm4_disable_arch_specific(drvdata); 868 869 if (!drvdata->skip_power_up) { 870 /* power can be removed from the trace unit now */ 871 control = etm4x_relaxed_read32(csa, TRCPDCR); 872 control &= ~TRCPDCR_PU; 873 etm4x_relaxed_write32(csa, control, TRCPDCR); 874 } 875 876 control = etm4x_relaxed_read32(csa, TRCPRGCTLR); 877 878 /* EN, bit[0] Trace unit enable bit */ 879 control &= ~0x1; 880 881 /* 882 * If the CPU supports v8.4 Trace filter Control, 883 * set the ETM to trace prohibited region. 884 */ 885 etm4x_prohibit_trace(drvdata); 886 /* 887 * Make sure everything completes before disabling, as recommended 888 * by section 7.3.77 ("TRCVICTLR, ViewInst Main Control Register, 889 * SSTATUS") of ARM IHI 0064D 890 */ 891 dsb(sy); 892 isb(); 893 /* Trace synchronization barrier, is a nop if not supported */ 894 tsb_csync(); 895 etm4x_relaxed_write32(csa, control, TRCPRGCTLR); 896 897 /* wait for TRCSTATR.PMSTABLE to go to '1' */ 898 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) 899 dev_err(etm_dev, 900 "timeout while waiting for PM stable Trace Status\n"); 901 /* read the status of the single shot comparators */ 902 for (i = 0; i < drvdata->nr_ss_cmp; i++) { 903 config->ss_status[i] = 904 etm4x_relaxed_read32(csa, TRCSSCSRn(i)); 905 } 906 907 /* read back the current counter values */ 908 for (i = 0; i < drvdata->nr_cntr; i++) { 909 config->cntr_val[i] = 910 etm4x_relaxed_read32(csa, TRCCNTVRn(i)); 911 } 912 913 coresight_disclaim_device_unlocked(csdev); 914 etm4_cs_lock(drvdata, csa); 915 916 dev_dbg(&drvdata->csdev->dev, 917 "cpu: %d disable smp call done\n", drvdata->cpu); 918 } 919 920 static int etm4_disable_perf(struct coresight_device *csdev, 921 struct perf_event *event) 922 { 923 u32 control; 924 struct etm_filters *filters = event->hw.addr_filters; 925 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 926 struct perf_event_attr *attr = &event->attr; 927 928 if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) 929 return -EINVAL; 930 931 etm4_disable_hw(drvdata); 932 /* 933 * The config_id occupies bits 63:32 of the config2 perf event attr 934 * field. If this is non-zero then we will have enabled a config. 935 */ 936 if (attr->config2 & GENMASK_ULL(63, 32)) 937 cscfg_csdev_disable_active_config(csdev); 938 939 /* 940 * Check if the start/stop logic was active when the unit was stopped. 941 * That way we can re-enable the start/stop logic when the process is 942 * scheduled again. Configuration of the start/stop logic happens in 943 * function etm4_set_event_filters(). 944 */ 945 control = etm4x_relaxed_read32(&csdev->access, TRCVICTLR); 946 /* TRCVICTLR::SSSTATUS, bit[9] */ 947 filters->ssstatus = (control & BIT(9)); 948 949 /* 950 * perf will release trace ids when _free_aux() is 951 * called at the end of the session. 952 */ 953 954 return 0; 955 } 956 957 static void etm4_disable_sysfs(struct coresight_device *csdev) 958 { 959 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 960 961 /* 962 * Taking hotplug lock here protects from clocks getting disabled 963 * with tracing being left on (crash scenario) if user disable occurs 964 * after cpu online mask indicates the cpu is offline but before the 965 * DYING hotplug callback is serviced by the ETM driver. 966 */ 967 cpus_read_lock(); 968 spin_lock(&drvdata->spinlock); 969 970 /* 971 * Executing etm4_disable_hw on the cpu whose ETM is being disabled 972 * ensures that register writes occur when cpu is powered. 973 */ 974 smp_call_function_single(drvdata->cpu, etm4_disable_hw, drvdata, 1); 975 976 spin_unlock(&drvdata->spinlock); 977 cpus_read_unlock(); 978 979 /* 980 * we only release trace IDs when resetting sysfs. 981 * This permits sysfs users to read the trace ID after the trace 982 * session has completed. This maintains operational behaviour with 983 * prior trace id allocation method 984 */ 985 986 dev_dbg(&csdev->dev, "ETM tracing disabled\n"); 987 } 988 989 static void etm4_disable(struct coresight_device *csdev, 990 struct perf_event *event) 991 { 992 u32 mode; 993 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 994 995 /* 996 * For as long as the tracer isn't disabled another entity can't 997 * change its status. As such we can read the status here without 998 * fearing it will change under us. 999 */ 1000 mode = local_read(&drvdata->mode); 1001 1002 switch (mode) { 1003 case CS_MODE_DISABLED: 1004 break; 1005 case CS_MODE_SYSFS: 1006 etm4_disable_sysfs(csdev); 1007 break; 1008 case CS_MODE_PERF: 1009 etm4_disable_perf(csdev, event); 1010 break; 1011 } 1012 1013 if (mode) 1014 local_set(&drvdata->mode, CS_MODE_DISABLED); 1015 } 1016 1017 static const struct coresight_ops_source etm4_source_ops = { 1018 .cpu_id = etm4_cpu_id, 1019 .enable = etm4_enable, 1020 .disable = etm4_disable, 1021 }; 1022 1023 static const struct coresight_ops etm4_cs_ops = { 1024 .source_ops = &etm4_source_ops, 1025 }; 1026 1027 static inline bool cpu_supports_sysreg_trace(void) 1028 { 1029 u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); 1030 1031 return ((dfr0 >> ID_AA64DFR0_EL1_TraceVer_SHIFT) & 0xfUL) > 0; 1032 } 1033 1034 static bool etm4_init_sysreg_access(struct etmv4_drvdata *drvdata, 1035 struct csdev_access *csa) 1036 { 1037 u32 devarch; 1038 1039 if (!cpu_supports_sysreg_trace()) 1040 return false; 1041 1042 /* 1043 * ETMs implementing sysreg access must implement TRCDEVARCH. 1044 */ 1045 devarch = read_etm4x_sysreg_const_offset(TRCDEVARCH); 1046 switch (devarch & ETM_DEVARCH_ID_MASK) { 1047 case ETM_DEVARCH_ETMv4x_ARCH: 1048 *csa = (struct csdev_access) { 1049 .io_mem = false, 1050 .read = etm4x_sysreg_read, 1051 .write = etm4x_sysreg_write, 1052 }; 1053 break; 1054 case ETM_DEVARCH_ETE_ARCH: 1055 *csa = (struct csdev_access) { 1056 .io_mem = false, 1057 .read = ete_sysreg_read, 1058 .write = ete_sysreg_write, 1059 }; 1060 break; 1061 default: 1062 return false; 1063 } 1064 1065 drvdata->arch = etm_devarch_to_arch(devarch); 1066 return true; 1067 } 1068 1069 static bool etm4_init_iomem_access(struct etmv4_drvdata *drvdata, 1070 struct csdev_access *csa) 1071 { 1072 u32 devarch = readl_relaxed(drvdata->base + TRCDEVARCH); 1073 u32 idr1 = readl_relaxed(drvdata->base + TRCIDR1); 1074 1075 /* 1076 * All ETMs must implement TRCDEVARCH to indicate that 1077 * the component is an ETMv4. To support any broken 1078 * implementations we fall back to TRCIDR1 check, which 1079 * is not really reliable. 1080 */ 1081 if ((devarch & ETM_DEVARCH_ID_MASK) == ETM_DEVARCH_ETMv4x_ARCH) { 1082 drvdata->arch = etm_devarch_to_arch(devarch); 1083 } else { 1084 pr_warn("CPU%d: ETM4x incompatible TRCDEVARCH: %x, falling back to TRCIDR1\n", 1085 smp_processor_id(), devarch); 1086 1087 if (ETM_TRCIDR1_ARCH_MAJOR(idr1) != ETM_TRCIDR1_ARCH_ETMv4) 1088 return false; 1089 drvdata->arch = etm_trcidr_to_arch(idr1); 1090 } 1091 1092 *csa = CSDEV_ACCESS_IOMEM(drvdata->base); 1093 return true; 1094 } 1095 1096 static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata, 1097 struct csdev_access *csa) 1098 { 1099 /* 1100 * Always choose the memory mapped io, if there is 1101 * a memory map to prevent sysreg access on broken 1102 * systems. 1103 */ 1104 if (drvdata->base) 1105 return etm4_init_iomem_access(drvdata, csa); 1106 1107 if (etm4_init_sysreg_access(drvdata, csa)) 1108 return true; 1109 1110 return false; 1111 } 1112 1113 static void cpu_detect_trace_filtering(struct etmv4_drvdata *drvdata) 1114 { 1115 u64 dfr0 = read_sysreg(id_aa64dfr0_el1); 1116 u64 trfcr; 1117 1118 drvdata->trfcr = 0; 1119 if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_TraceFilt_SHIFT)) 1120 return; 1121 1122 /* 1123 * If the CPU supports v8.4 SelfHosted Tracing, enable 1124 * tracing at the kernel EL and EL0, forcing to use the 1125 * virtual time as the timestamp. 1126 */ 1127 trfcr = (TRFCR_ELx_TS_VIRTUAL | 1128 TRFCR_ELx_ExTRE | 1129 TRFCR_ELx_E0TRE); 1130 1131 /* If we are running at EL2, allow tracing the CONTEXTIDR_EL2. */ 1132 if (is_kernel_in_hyp_mode()) 1133 trfcr |= TRFCR_EL2_CX; 1134 1135 drvdata->trfcr = trfcr; 1136 } 1137 1138 static void etm4_init_arch_data(void *info) 1139 { 1140 u32 etmidr0; 1141 u32 etmidr2; 1142 u32 etmidr3; 1143 u32 etmidr4; 1144 u32 etmidr5; 1145 struct etm4_init_arg *init_arg = info; 1146 struct etmv4_drvdata *drvdata; 1147 struct csdev_access *csa; 1148 int i; 1149 1150 drvdata = dev_get_drvdata(init_arg->dev); 1151 csa = init_arg->csa; 1152 1153 /* 1154 * If we are unable to detect the access mechanism, 1155 * or unable to detect the trace unit type, fail 1156 * early. 1157 */ 1158 if (!etm4_init_csdev_access(drvdata, csa)) 1159 return; 1160 1161 /* Detect the support for OS Lock before we actually use it */ 1162 etm_detect_os_lock(drvdata, csa); 1163 1164 /* Make sure all registers are accessible */ 1165 etm4_os_unlock_csa(drvdata, csa); 1166 etm4_cs_unlock(drvdata, csa); 1167 1168 etm4_check_arch_features(drvdata, init_arg->pid); 1169 1170 /* find all capabilities of the tracing unit */ 1171 etmidr0 = etm4x_relaxed_read32(csa, TRCIDR0); 1172 1173 /* INSTP0, bits[2:1] P0 tracing support field */ 1174 drvdata->instrp0 = !!(FIELD_GET(TRCIDR0_INSTP0_MASK, etmidr0) == 0b11); 1175 /* TRCBB, bit[5] Branch broadcast tracing support bit */ 1176 drvdata->trcbb = !!(etmidr0 & TRCIDR0_TRCBB); 1177 /* TRCCOND, bit[6] Conditional instruction tracing support bit */ 1178 drvdata->trccond = !!(etmidr0 & TRCIDR0_TRCCOND); 1179 /* TRCCCI, bit[7] Cycle counting instruction bit */ 1180 drvdata->trccci = !!(etmidr0 & TRCIDR0_TRCCCI); 1181 /* RETSTACK, bit[9] Return stack bit */ 1182 drvdata->retstack = !!(etmidr0 & TRCIDR0_RETSTACK); 1183 /* NUMEVENT, bits[11:10] Number of events field */ 1184 drvdata->nr_event = FIELD_GET(TRCIDR0_NUMEVENT_MASK, etmidr0); 1185 /* QSUPP, bits[16:15] Q element support field */ 1186 drvdata->q_support = FIELD_GET(TRCIDR0_QSUPP_MASK, etmidr0); 1187 /* TSSIZE, bits[28:24] Global timestamp size field */ 1188 drvdata->ts_size = FIELD_GET(TRCIDR0_TSSIZE_MASK, etmidr0); 1189 1190 /* maximum size of resources */ 1191 etmidr2 = etm4x_relaxed_read32(csa, TRCIDR2); 1192 /* CIDSIZE, bits[9:5] Indicates the Context ID size */ 1193 drvdata->ctxid_size = FIELD_GET(TRCIDR2_CIDSIZE_MASK, etmidr2); 1194 /* VMIDSIZE, bits[14:10] Indicates the VMID size */ 1195 drvdata->vmid_size = FIELD_GET(TRCIDR2_VMIDSIZE_MASK, etmidr2); 1196 /* CCSIZE, bits[28:25] size of the cycle counter in bits minus 12 */ 1197 drvdata->ccsize = FIELD_GET(TRCIDR2_CCSIZE_MASK, etmidr2); 1198 1199 etmidr3 = etm4x_relaxed_read32(csa, TRCIDR3); 1200 /* CCITMIN, bits[11:0] minimum threshold value that can be programmed */ 1201 drvdata->ccitmin = FIELD_GET(TRCIDR3_CCITMIN_MASK, etmidr3); 1202 /* EXLEVEL_S, bits[19:16] Secure state instruction tracing */ 1203 drvdata->s_ex_level = FIELD_GET(TRCIDR3_EXLEVEL_S_MASK, etmidr3); 1204 drvdata->config.s_ex_level = drvdata->s_ex_level; 1205 /* EXLEVEL_NS, bits[23:20] Non-secure state instruction tracing */ 1206 drvdata->ns_ex_level = FIELD_GET(TRCIDR3_EXLEVEL_NS_MASK, etmidr3); 1207 /* 1208 * TRCERR, bit[24] whether a trace unit can trace a 1209 * system error exception. 1210 */ 1211 drvdata->trc_error = !!(etmidr3 & TRCIDR3_TRCERR); 1212 /* SYNCPR, bit[25] implementation has a fixed synchronization period? */ 1213 drvdata->syncpr = !!(etmidr3 & TRCIDR3_SYNCPR); 1214 /* STALLCTL, bit[26] is stall control implemented? */ 1215 drvdata->stallctl = !!(etmidr3 & TRCIDR3_STALLCTL); 1216 /* SYSSTALL, bit[27] implementation can support stall control? */ 1217 drvdata->sysstall = !!(etmidr3 & TRCIDR3_SYSSTALL); 1218 /* 1219 * NUMPROC - the number of PEs available for tracing, 5bits 1220 * = TRCIDR3.bits[13:12]bits[30:28] 1221 * bits[4:3] = TRCIDR3.bits[13:12] (since etm-v4.2, otherwise RES0) 1222 * bits[3:0] = TRCIDR3.bits[30:28] 1223 */ 1224 drvdata->nr_pe = (FIELD_GET(TRCIDR3_NUMPROC_HI_MASK, etmidr3) << 3) | 1225 FIELD_GET(TRCIDR3_NUMPROC_LO_MASK, etmidr3); 1226 /* NOOVERFLOW, bit[31] is trace overflow prevention supported */ 1227 drvdata->nooverflow = !!(etmidr3 & TRCIDR3_NOOVERFLOW); 1228 1229 /* number of resources trace unit supports */ 1230 etmidr4 = etm4x_relaxed_read32(csa, TRCIDR4); 1231 /* NUMACPAIRS, bits[0:3] number of addr comparator pairs for tracing */ 1232 drvdata->nr_addr_cmp = FIELD_GET(TRCIDR4_NUMACPAIRS_MASK, etmidr4); 1233 /* NUMPC, bits[15:12] number of PE comparator inputs for tracing */ 1234 drvdata->nr_pe_cmp = FIELD_GET(TRCIDR4_NUMPC_MASK, etmidr4); 1235 /* 1236 * NUMRSPAIR, bits[19:16] 1237 * The number of resource pairs conveyed by the HW starts at 0, i.e a 1238 * value of 0x0 indicate 1 resource pair, 0x1 indicate two and so on. 1239 * As such add 1 to the value of NUMRSPAIR for a better representation. 1240 * 1241 * For ETM v4.3 and later, 0x0 means 0, and no pairs are available - 1242 * the default TRUE and FALSE resource selectors are omitted. 1243 * Otherwise for values 0x1 and above the number is N + 1 as per v4.2. 1244 */ 1245 drvdata->nr_resource = FIELD_GET(TRCIDR4_NUMRSPAIR_MASK, etmidr4); 1246 if ((drvdata->arch < ETM_ARCH_V4_3) || (drvdata->nr_resource > 0)) 1247 drvdata->nr_resource += 1; 1248 /* 1249 * NUMSSCC, bits[23:20] the number of single-shot 1250 * comparator control for tracing. Read any status regs as these 1251 * also contain RO capability data. 1252 */ 1253 drvdata->nr_ss_cmp = FIELD_GET(TRCIDR4_NUMSSCC_MASK, etmidr4); 1254 for (i = 0; i < drvdata->nr_ss_cmp; i++) { 1255 drvdata->config.ss_status[i] = 1256 etm4x_relaxed_read32(csa, TRCSSCSRn(i)); 1257 } 1258 /* NUMCIDC, bits[27:24] number of Context ID comparators for tracing */ 1259 drvdata->numcidc = FIELD_GET(TRCIDR4_NUMCIDC_MASK, etmidr4); 1260 /* NUMVMIDC, bits[31:28] number of VMID comparators for tracing */ 1261 drvdata->numvmidc = FIELD_GET(TRCIDR4_NUMVMIDC_MASK, etmidr4); 1262 1263 etmidr5 = etm4x_relaxed_read32(csa, TRCIDR5); 1264 /* NUMEXTIN, bits[8:0] number of external inputs implemented */ 1265 drvdata->nr_ext_inp = FIELD_GET(TRCIDR5_NUMEXTIN_MASK, etmidr5); 1266 /* TRACEIDSIZE, bits[21:16] indicates the trace ID width */ 1267 drvdata->trcid_size = FIELD_GET(TRCIDR5_TRACEIDSIZE_MASK, etmidr5); 1268 /* ATBTRIG, bit[22] implementation can support ATB triggers? */ 1269 drvdata->atbtrig = !!(etmidr5 & TRCIDR5_ATBTRIG); 1270 /* 1271 * LPOVERRIDE, bit[23] implementation supports 1272 * low-power state override 1273 */ 1274 drvdata->lpoverride = (etmidr5 & TRCIDR5_LPOVERRIDE) && (!drvdata->skip_power_up); 1275 /* NUMSEQSTATE, bits[27:25] number of sequencer states implemented */ 1276 drvdata->nrseqstate = FIELD_GET(TRCIDR5_NUMSEQSTATE_MASK, etmidr5); 1277 /* NUMCNTR, bits[30:28] number of counters available for tracing */ 1278 drvdata->nr_cntr = FIELD_GET(TRCIDR5_NUMCNTR_MASK, etmidr5); 1279 etm4_cs_lock(drvdata, csa); 1280 cpu_detect_trace_filtering(drvdata); 1281 } 1282 1283 static inline u32 etm4_get_victlr_access_type(struct etmv4_config *config) 1284 { 1285 return etm4_get_access_type(config) << __bf_shf(TRCVICTLR_EXLEVEL_MASK); 1286 } 1287 1288 /* Set ELx trace filter access in the TRCVICTLR register */ 1289 static void etm4_set_victlr_access(struct etmv4_config *config) 1290 { 1291 config->vinst_ctrl &= ~TRCVICTLR_EXLEVEL_MASK; 1292 config->vinst_ctrl |= etm4_get_victlr_access_type(config); 1293 } 1294 1295 static void etm4_set_default_config(struct etmv4_config *config) 1296 { 1297 /* disable all events tracing */ 1298 config->eventctrl0 = 0x0; 1299 config->eventctrl1 = 0x0; 1300 1301 /* disable stalling */ 1302 config->stall_ctrl = 0x0; 1303 1304 /* enable trace synchronization every 4096 bytes, if available */ 1305 config->syncfreq = 0xC; 1306 1307 /* disable timestamp event */ 1308 config->ts_ctrl = 0x0; 1309 1310 /* TRCVICTLR::EVENT = 0x01, select the always on logic */ 1311 config->vinst_ctrl = FIELD_PREP(TRCVICTLR_EVENT_MASK, 0x01); 1312 1313 /* TRCVICTLR::EXLEVEL_NS:EXLEVELS: Set kernel / user filtering */ 1314 etm4_set_victlr_access(config); 1315 } 1316 1317 static u64 etm4_get_ns_access_type(struct etmv4_config *config) 1318 { 1319 u64 access_type = 0; 1320 1321 /* 1322 * EXLEVEL_NS, for NonSecure Exception levels. 1323 * The mask here is a generic value and must be 1324 * shifted to the corresponding field for the registers 1325 */ 1326 if (!is_kernel_in_hyp_mode()) { 1327 /* Stay away from hypervisor mode for non-VHE */ 1328 access_type = ETM_EXLEVEL_NS_HYP; 1329 if (config->mode & ETM_MODE_EXCL_KERN) 1330 access_type |= ETM_EXLEVEL_NS_OS; 1331 } else if (config->mode & ETM_MODE_EXCL_KERN) { 1332 access_type = ETM_EXLEVEL_NS_HYP; 1333 } 1334 1335 if (config->mode & ETM_MODE_EXCL_USER) 1336 access_type |= ETM_EXLEVEL_NS_APP; 1337 1338 return access_type; 1339 } 1340 1341 /* 1342 * Construct the exception level masks for a given config. 1343 * This must be shifted to the corresponding register field 1344 * for usage. 1345 */ 1346 static u64 etm4_get_access_type(struct etmv4_config *config) 1347 { 1348 /* All Secure exception levels are excluded from the trace */ 1349 return etm4_get_ns_access_type(config) | (u64)config->s_ex_level; 1350 } 1351 1352 static u64 etm4_get_comparator_access_type(struct etmv4_config *config) 1353 { 1354 return etm4_get_access_type(config) << TRCACATR_EXLEVEL_SHIFT; 1355 } 1356 1357 static void etm4_set_comparator_filter(struct etmv4_config *config, 1358 u64 start, u64 stop, int comparator) 1359 { 1360 u64 access_type = etm4_get_comparator_access_type(config); 1361 1362 /* First half of default address comparator */ 1363 config->addr_val[comparator] = start; 1364 config->addr_acc[comparator] = access_type; 1365 config->addr_type[comparator] = ETM_ADDR_TYPE_RANGE; 1366 1367 /* Second half of default address comparator */ 1368 config->addr_val[comparator + 1] = stop; 1369 config->addr_acc[comparator + 1] = access_type; 1370 config->addr_type[comparator + 1] = ETM_ADDR_TYPE_RANGE; 1371 1372 /* 1373 * Configure the ViewInst function to include this address range 1374 * comparator. 1375 * 1376 * @comparator is divided by two since it is the index in the 1377 * etmv4_config::addr_val array but register TRCVIIECTLR deals with 1378 * address range comparator _pairs_. 1379 * 1380 * Therefore: 1381 * index 0 -> compatator pair 0 1382 * index 2 -> comparator pair 1 1383 * index 4 -> comparator pair 2 1384 * ... 1385 * index 14 -> comparator pair 7 1386 */ 1387 config->viiectlr |= BIT(comparator / 2); 1388 } 1389 1390 static void etm4_set_start_stop_filter(struct etmv4_config *config, 1391 u64 address, int comparator, 1392 enum etm_addr_type type) 1393 { 1394 int shift; 1395 u64 access_type = etm4_get_comparator_access_type(config); 1396 1397 /* Configure the comparator */ 1398 config->addr_val[comparator] = address; 1399 config->addr_acc[comparator] = access_type; 1400 config->addr_type[comparator] = type; 1401 1402 /* 1403 * Configure ViewInst Start-Stop control register. 1404 * Addresses configured to start tracing go from bit 0 to n-1, 1405 * while those configured to stop tracing from 16 to 16 + n-1. 1406 */ 1407 shift = (type == ETM_ADDR_TYPE_START ? 0 : 16); 1408 config->vissctlr |= BIT(shift + comparator); 1409 } 1410 1411 static void etm4_set_default_filter(struct etmv4_config *config) 1412 { 1413 /* Trace everything 'default' filter achieved by no filtering */ 1414 config->viiectlr = 0x0; 1415 1416 /* 1417 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is 1418 * in the started state 1419 */ 1420 config->vinst_ctrl |= TRCVICTLR_SSSTATUS; 1421 config->mode |= ETM_MODE_VIEWINST_STARTSTOP; 1422 1423 /* No start-stop filtering for ViewInst */ 1424 config->vissctlr = 0x0; 1425 } 1426 1427 static void etm4_set_default(struct etmv4_config *config) 1428 { 1429 if (WARN_ON_ONCE(!config)) 1430 return; 1431 1432 /* 1433 * Make default initialisation trace everything 1434 * 1435 * This is done by a minimum default config sufficient to enable 1436 * full instruction trace - with a default filter for trace all 1437 * achieved by having no filtering. 1438 */ 1439 etm4_set_default_config(config); 1440 etm4_set_default_filter(config); 1441 } 1442 1443 static int etm4_get_next_comparator(struct etmv4_drvdata *drvdata, u32 type) 1444 { 1445 int nr_comparator, index = 0; 1446 struct etmv4_config *config = &drvdata->config; 1447 1448 /* 1449 * nr_addr_cmp holds the number of comparator _pair_, so time 2 1450 * for the total number of comparators. 1451 */ 1452 nr_comparator = drvdata->nr_addr_cmp * 2; 1453 1454 /* Go through the tally of comparators looking for a free one. */ 1455 while (index < nr_comparator) { 1456 switch (type) { 1457 case ETM_ADDR_TYPE_RANGE: 1458 if (config->addr_type[index] == ETM_ADDR_TYPE_NONE && 1459 config->addr_type[index + 1] == ETM_ADDR_TYPE_NONE) 1460 return index; 1461 1462 /* Address range comparators go in pairs */ 1463 index += 2; 1464 break; 1465 case ETM_ADDR_TYPE_START: 1466 case ETM_ADDR_TYPE_STOP: 1467 if (config->addr_type[index] == ETM_ADDR_TYPE_NONE) 1468 return index; 1469 1470 /* Start/stop address can have odd indexes */ 1471 index += 1; 1472 break; 1473 default: 1474 return -EINVAL; 1475 } 1476 } 1477 1478 /* If we are here all the comparators have been used. */ 1479 return -ENOSPC; 1480 } 1481 1482 static int etm4_set_event_filters(struct etmv4_drvdata *drvdata, 1483 struct perf_event *event) 1484 { 1485 int i, comparator, ret = 0; 1486 u64 address; 1487 struct etmv4_config *config = &drvdata->config; 1488 struct etm_filters *filters = event->hw.addr_filters; 1489 1490 if (!filters) 1491 goto default_filter; 1492 1493 /* Sync events with what Perf got */ 1494 perf_event_addr_filters_sync(event); 1495 1496 /* 1497 * If there are no filters to deal with simply go ahead with 1498 * the default filter, i.e the entire address range. 1499 */ 1500 if (!filters->nr_filters) 1501 goto default_filter; 1502 1503 for (i = 0; i < filters->nr_filters; i++) { 1504 struct etm_filter *filter = &filters->etm_filter[i]; 1505 enum etm_addr_type type = filter->type; 1506 1507 /* See if a comparator is free. */ 1508 comparator = etm4_get_next_comparator(drvdata, type); 1509 if (comparator < 0) { 1510 ret = comparator; 1511 goto out; 1512 } 1513 1514 switch (type) { 1515 case ETM_ADDR_TYPE_RANGE: 1516 etm4_set_comparator_filter(config, 1517 filter->start_addr, 1518 filter->stop_addr, 1519 comparator); 1520 /* 1521 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is 1522 * in the started state 1523 */ 1524 config->vinst_ctrl |= TRCVICTLR_SSSTATUS; 1525 1526 /* No start-stop filtering for ViewInst */ 1527 config->vissctlr = 0x0; 1528 break; 1529 case ETM_ADDR_TYPE_START: 1530 case ETM_ADDR_TYPE_STOP: 1531 /* Get the right start or stop address */ 1532 address = (type == ETM_ADDR_TYPE_START ? 1533 filter->start_addr : 1534 filter->stop_addr); 1535 1536 /* Configure comparator */ 1537 etm4_set_start_stop_filter(config, address, 1538 comparator, type); 1539 1540 /* 1541 * If filters::ssstatus == 1, trace acquisition was 1542 * started but the process was yanked away before the 1543 * stop address was hit. As such the start/stop 1544 * logic needs to be re-started so that tracing can 1545 * resume where it left. 1546 * 1547 * The start/stop logic status when a process is 1548 * scheduled out is checked in function 1549 * etm4_disable_perf(). 1550 */ 1551 if (filters->ssstatus) 1552 config->vinst_ctrl |= TRCVICTLR_SSSTATUS; 1553 1554 /* No include/exclude filtering for ViewInst */ 1555 config->viiectlr = 0x0; 1556 break; 1557 default: 1558 ret = -EINVAL; 1559 goto out; 1560 } 1561 } 1562 1563 goto out; 1564 1565 1566 default_filter: 1567 etm4_set_default_filter(config); 1568 1569 out: 1570 return ret; 1571 } 1572 1573 void etm4_config_trace_mode(struct etmv4_config *config) 1574 { 1575 u32 mode; 1576 1577 mode = config->mode; 1578 mode &= (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER); 1579 1580 /* excluding kernel AND user space doesn't make sense */ 1581 WARN_ON_ONCE(mode == (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER)); 1582 1583 /* nothing to do if neither flags are set */ 1584 if (!(mode & ETM_MODE_EXCL_KERN) && !(mode & ETM_MODE_EXCL_USER)) 1585 return; 1586 1587 etm4_set_victlr_access(config); 1588 } 1589 1590 static int etm4_online_cpu(unsigned int cpu) 1591 { 1592 if (!etmdrvdata[cpu]) 1593 return etm4_probe_cpu(cpu); 1594 1595 if (etmdrvdata[cpu]->boot_enable && !etmdrvdata[cpu]->sticky_enable) 1596 coresight_enable(etmdrvdata[cpu]->csdev); 1597 return 0; 1598 } 1599 1600 static int etm4_starting_cpu(unsigned int cpu) 1601 { 1602 if (!etmdrvdata[cpu]) 1603 return 0; 1604 1605 spin_lock(&etmdrvdata[cpu]->spinlock); 1606 if (!etmdrvdata[cpu]->os_unlock) 1607 etm4_os_unlock(etmdrvdata[cpu]); 1608 1609 if (local_read(&etmdrvdata[cpu]->mode)) 1610 etm4_enable_hw(etmdrvdata[cpu]); 1611 spin_unlock(&etmdrvdata[cpu]->spinlock); 1612 return 0; 1613 } 1614 1615 static int etm4_dying_cpu(unsigned int cpu) 1616 { 1617 if (!etmdrvdata[cpu]) 1618 return 0; 1619 1620 spin_lock(&etmdrvdata[cpu]->spinlock); 1621 if (local_read(&etmdrvdata[cpu]->mode)) 1622 etm4_disable_hw(etmdrvdata[cpu]); 1623 spin_unlock(&etmdrvdata[cpu]->spinlock); 1624 return 0; 1625 } 1626 1627 static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) 1628 { 1629 int i, ret = 0; 1630 struct etmv4_save_state *state; 1631 struct coresight_device *csdev = drvdata->csdev; 1632 struct csdev_access *csa; 1633 struct device *etm_dev; 1634 1635 if (WARN_ON(!csdev)) 1636 return -ENODEV; 1637 1638 etm_dev = &csdev->dev; 1639 csa = &csdev->access; 1640 1641 /* 1642 * As recommended by 3.4.1 ("The procedure when powering down the PE") 1643 * of ARM IHI 0064D 1644 */ 1645 dsb(sy); 1646 isb(); 1647 1648 etm4_cs_unlock(drvdata, csa); 1649 /* Lock the OS lock to disable trace and external debugger access */ 1650 etm4_os_lock(drvdata); 1651 1652 /* wait for TRCSTATR.PMSTABLE to go up */ 1653 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) { 1654 dev_err(etm_dev, 1655 "timeout while waiting for PM Stable Status\n"); 1656 etm4_os_unlock(drvdata); 1657 ret = -EBUSY; 1658 goto out; 1659 } 1660 1661 state = drvdata->save_state; 1662 1663 state->trcprgctlr = etm4x_read32(csa, TRCPRGCTLR); 1664 if (drvdata->nr_pe) 1665 state->trcprocselr = etm4x_read32(csa, TRCPROCSELR); 1666 state->trcconfigr = etm4x_read32(csa, TRCCONFIGR); 1667 state->trcauxctlr = etm4x_read32(csa, TRCAUXCTLR); 1668 state->trceventctl0r = etm4x_read32(csa, TRCEVENTCTL0R); 1669 state->trceventctl1r = etm4x_read32(csa, TRCEVENTCTL1R); 1670 if (drvdata->stallctl) 1671 state->trcstallctlr = etm4x_read32(csa, TRCSTALLCTLR); 1672 state->trctsctlr = etm4x_read32(csa, TRCTSCTLR); 1673 state->trcsyncpr = etm4x_read32(csa, TRCSYNCPR); 1674 state->trcccctlr = etm4x_read32(csa, TRCCCCTLR); 1675 state->trcbbctlr = etm4x_read32(csa, TRCBBCTLR); 1676 state->trctraceidr = etm4x_read32(csa, TRCTRACEIDR); 1677 state->trcqctlr = etm4x_read32(csa, TRCQCTLR); 1678 1679 state->trcvictlr = etm4x_read32(csa, TRCVICTLR); 1680 state->trcviiectlr = etm4x_read32(csa, TRCVIIECTLR); 1681 state->trcvissctlr = etm4x_read32(csa, TRCVISSCTLR); 1682 if (drvdata->nr_pe_cmp) 1683 state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR); 1684 state->trcvdctlr = etm4x_read32(csa, TRCVDCTLR); 1685 state->trcvdsacctlr = etm4x_read32(csa, TRCVDSACCTLR); 1686 state->trcvdarcctlr = etm4x_read32(csa, TRCVDARCCTLR); 1687 1688 for (i = 0; i < drvdata->nrseqstate - 1; i++) 1689 state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i)); 1690 1691 if (drvdata->nrseqstate) { 1692 state->trcseqrstevr = etm4x_read32(csa, TRCSEQRSTEVR); 1693 state->trcseqstr = etm4x_read32(csa, TRCSEQSTR); 1694 } 1695 state->trcextinselr = etm4x_read32(csa, TRCEXTINSELR); 1696 1697 for (i = 0; i < drvdata->nr_cntr; i++) { 1698 state->trccntrldvr[i] = etm4x_read32(csa, TRCCNTRLDVRn(i)); 1699 state->trccntctlr[i] = etm4x_read32(csa, TRCCNTCTLRn(i)); 1700 state->trccntvr[i] = etm4x_read32(csa, TRCCNTVRn(i)); 1701 } 1702 1703 for (i = 0; i < drvdata->nr_resource * 2; i++) 1704 state->trcrsctlr[i] = etm4x_read32(csa, TRCRSCTLRn(i)); 1705 1706 for (i = 0; i < drvdata->nr_ss_cmp; i++) { 1707 state->trcssccr[i] = etm4x_read32(csa, TRCSSCCRn(i)); 1708 state->trcsscsr[i] = etm4x_read32(csa, TRCSSCSRn(i)); 1709 if (etm4x_sspcicrn_present(drvdata, i)) 1710 state->trcsspcicr[i] = etm4x_read32(csa, TRCSSPCICRn(i)); 1711 } 1712 1713 for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { 1714 state->trcacvr[i] = etm4x_read64(csa, TRCACVRn(i)); 1715 state->trcacatr[i] = etm4x_read64(csa, TRCACATRn(i)); 1716 } 1717 1718 /* 1719 * Data trace stream is architecturally prohibited for A profile cores 1720 * so we don't save (or later restore) trcdvcvr and trcdvcmr - As per 1721 * section 1.3.4 ("Possible functional configurations of an ETMv4 trace 1722 * unit") of ARM IHI 0064D. 1723 */ 1724 1725 for (i = 0; i < drvdata->numcidc; i++) 1726 state->trccidcvr[i] = etm4x_read64(csa, TRCCIDCVRn(i)); 1727 1728 for (i = 0; i < drvdata->numvmidc; i++) 1729 state->trcvmidcvr[i] = etm4x_read64(csa, TRCVMIDCVRn(i)); 1730 1731 state->trccidcctlr0 = etm4x_read32(csa, TRCCIDCCTLR0); 1732 if (drvdata->numcidc > 4) 1733 state->trccidcctlr1 = etm4x_read32(csa, TRCCIDCCTLR1); 1734 1735 state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0); 1736 if (drvdata->numvmidc > 4) 1737 state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1); 1738 1739 state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR); 1740 1741 if (!drvdata->skip_power_up) 1742 state->trcpdcr = etm4x_read32(csa, TRCPDCR); 1743 1744 /* wait for TRCSTATR.IDLE to go up */ 1745 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) { 1746 dev_err(etm_dev, 1747 "timeout while waiting for Idle Trace Status\n"); 1748 etm4_os_unlock(drvdata); 1749 ret = -EBUSY; 1750 goto out; 1751 } 1752 1753 drvdata->state_needs_restore = true; 1754 1755 /* 1756 * Power can be removed from the trace unit now. We do this to 1757 * potentially save power on systems that respect the TRCPDCR_PU 1758 * despite requesting software to save/restore state. 1759 */ 1760 if (!drvdata->skip_power_up) 1761 etm4x_relaxed_write32(csa, (state->trcpdcr & ~TRCPDCR_PU), 1762 TRCPDCR); 1763 out: 1764 etm4_cs_lock(drvdata, csa); 1765 return ret; 1766 } 1767 1768 static int etm4_cpu_save(struct etmv4_drvdata *drvdata) 1769 { 1770 int ret = 0; 1771 1772 /* Save the TRFCR irrespective of whether the ETM is ON */ 1773 if (drvdata->trfcr) 1774 drvdata->save_trfcr = read_trfcr(); 1775 /* 1776 * Save and restore the ETM Trace registers only if 1777 * the ETM is active. 1778 */ 1779 if (local_read(&drvdata->mode) && drvdata->save_state) 1780 ret = __etm4_cpu_save(drvdata); 1781 return ret; 1782 } 1783 1784 static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) 1785 { 1786 int i; 1787 struct etmv4_save_state *state = drvdata->save_state; 1788 struct csdev_access tmp_csa = CSDEV_ACCESS_IOMEM(drvdata->base); 1789 struct csdev_access *csa = &tmp_csa; 1790 1791 etm4_cs_unlock(drvdata, csa); 1792 etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); 1793 1794 etm4x_relaxed_write32(csa, state->trcprgctlr, TRCPRGCTLR); 1795 if (drvdata->nr_pe) 1796 etm4x_relaxed_write32(csa, state->trcprocselr, TRCPROCSELR); 1797 etm4x_relaxed_write32(csa, state->trcconfigr, TRCCONFIGR); 1798 etm4x_relaxed_write32(csa, state->trcauxctlr, TRCAUXCTLR); 1799 etm4x_relaxed_write32(csa, state->trceventctl0r, TRCEVENTCTL0R); 1800 etm4x_relaxed_write32(csa, state->trceventctl1r, TRCEVENTCTL1R); 1801 if (drvdata->stallctl) 1802 etm4x_relaxed_write32(csa, state->trcstallctlr, TRCSTALLCTLR); 1803 etm4x_relaxed_write32(csa, state->trctsctlr, TRCTSCTLR); 1804 etm4x_relaxed_write32(csa, state->trcsyncpr, TRCSYNCPR); 1805 etm4x_relaxed_write32(csa, state->trcccctlr, TRCCCCTLR); 1806 etm4x_relaxed_write32(csa, state->trcbbctlr, TRCBBCTLR); 1807 etm4x_relaxed_write32(csa, state->trctraceidr, TRCTRACEIDR); 1808 etm4x_relaxed_write32(csa, state->trcqctlr, TRCQCTLR); 1809 1810 etm4x_relaxed_write32(csa, state->trcvictlr, TRCVICTLR); 1811 etm4x_relaxed_write32(csa, state->trcviiectlr, TRCVIIECTLR); 1812 etm4x_relaxed_write32(csa, state->trcvissctlr, TRCVISSCTLR); 1813 if (drvdata->nr_pe_cmp) 1814 etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR); 1815 etm4x_relaxed_write32(csa, state->trcvdctlr, TRCVDCTLR); 1816 etm4x_relaxed_write32(csa, state->trcvdsacctlr, TRCVDSACCTLR); 1817 etm4x_relaxed_write32(csa, state->trcvdarcctlr, TRCVDARCCTLR); 1818 1819 for (i = 0; i < drvdata->nrseqstate - 1; i++) 1820 etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i)); 1821 1822 if (drvdata->nrseqstate) { 1823 etm4x_relaxed_write32(csa, state->trcseqrstevr, TRCSEQRSTEVR); 1824 etm4x_relaxed_write32(csa, state->trcseqstr, TRCSEQSTR); 1825 } 1826 etm4x_relaxed_write32(csa, state->trcextinselr, TRCEXTINSELR); 1827 1828 for (i = 0; i < drvdata->nr_cntr; i++) { 1829 etm4x_relaxed_write32(csa, state->trccntrldvr[i], TRCCNTRLDVRn(i)); 1830 etm4x_relaxed_write32(csa, state->trccntctlr[i], TRCCNTCTLRn(i)); 1831 etm4x_relaxed_write32(csa, state->trccntvr[i], TRCCNTVRn(i)); 1832 } 1833 1834 for (i = 0; i < drvdata->nr_resource * 2; i++) 1835 etm4x_relaxed_write32(csa, state->trcrsctlr[i], TRCRSCTLRn(i)); 1836 1837 for (i = 0; i < drvdata->nr_ss_cmp; i++) { 1838 etm4x_relaxed_write32(csa, state->trcssccr[i], TRCSSCCRn(i)); 1839 etm4x_relaxed_write32(csa, state->trcsscsr[i], TRCSSCSRn(i)); 1840 if (etm4x_sspcicrn_present(drvdata, i)) 1841 etm4x_relaxed_write32(csa, state->trcsspcicr[i], TRCSSPCICRn(i)); 1842 } 1843 1844 for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { 1845 etm4x_relaxed_write64(csa, state->trcacvr[i], TRCACVRn(i)); 1846 etm4x_relaxed_write64(csa, state->trcacatr[i], TRCACATRn(i)); 1847 } 1848 1849 for (i = 0; i < drvdata->numcidc; i++) 1850 etm4x_relaxed_write64(csa, state->trccidcvr[i], TRCCIDCVRn(i)); 1851 1852 for (i = 0; i < drvdata->numvmidc; i++) 1853 etm4x_relaxed_write64(csa, state->trcvmidcvr[i], TRCVMIDCVRn(i)); 1854 1855 etm4x_relaxed_write32(csa, state->trccidcctlr0, TRCCIDCCTLR0); 1856 if (drvdata->numcidc > 4) 1857 etm4x_relaxed_write32(csa, state->trccidcctlr1, TRCCIDCCTLR1); 1858 1859 etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0); 1860 if (drvdata->numvmidc > 4) 1861 etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1); 1862 1863 etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); 1864 1865 if (!drvdata->skip_power_up) 1866 etm4x_relaxed_write32(csa, state->trcpdcr, TRCPDCR); 1867 1868 drvdata->state_needs_restore = false; 1869 1870 /* 1871 * As recommended by section 4.3.7 ("Synchronization when using the 1872 * memory-mapped interface") of ARM IHI 0064D 1873 */ 1874 dsb(sy); 1875 isb(); 1876 1877 /* Unlock the OS lock to re-enable trace and external debug access */ 1878 etm4_os_unlock(drvdata); 1879 etm4_cs_lock(drvdata, csa); 1880 } 1881 1882 static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) 1883 { 1884 if (drvdata->trfcr) 1885 write_trfcr(drvdata->save_trfcr); 1886 if (drvdata->state_needs_restore) 1887 __etm4_cpu_restore(drvdata); 1888 } 1889 1890 static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, 1891 void *v) 1892 { 1893 struct etmv4_drvdata *drvdata; 1894 unsigned int cpu = smp_processor_id(); 1895 1896 if (!etmdrvdata[cpu]) 1897 return NOTIFY_OK; 1898 1899 drvdata = etmdrvdata[cpu]; 1900 1901 if (WARN_ON_ONCE(drvdata->cpu != cpu)) 1902 return NOTIFY_BAD; 1903 1904 switch (cmd) { 1905 case CPU_PM_ENTER: 1906 if (etm4_cpu_save(drvdata)) 1907 return NOTIFY_BAD; 1908 break; 1909 case CPU_PM_EXIT: 1910 case CPU_PM_ENTER_FAILED: 1911 etm4_cpu_restore(drvdata); 1912 break; 1913 default: 1914 return NOTIFY_DONE; 1915 } 1916 1917 return NOTIFY_OK; 1918 } 1919 1920 static struct notifier_block etm4_cpu_pm_nb = { 1921 .notifier_call = etm4_cpu_pm_notify, 1922 }; 1923 1924 /* Setup PM. Deals with error conditions and counts */ 1925 static int __init etm4_pm_setup(void) 1926 { 1927 int ret; 1928 1929 ret = cpu_pm_register_notifier(&etm4_cpu_pm_nb); 1930 if (ret) 1931 return ret; 1932 1933 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING, 1934 "arm/coresight4:starting", 1935 etm4_starting_cpu, etm4_dying_cpu); 1936 1937 if (ret) 1938 goto unregister_notifier; 1939 1940 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, 1941 "arm/coresight4:online", 1942 etm4_online_cpu, NULL); 1943 1944 /* HP dyn state ID returned in ret on success */ 1945 if (ret > 0) { 1946 hp_online = ret; 1947 return 0; 1948 } 1949 1950 /* failed dyn state - remove others */ 1951 cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); 1952 1953 unregister_notifier: 1954 cpu_pm_unregister_notifier(&etm4_cpu_pm_nb); 1955 return ret; 1956 } 1957 1958 static void etm4_pm_clear(void) 1959 { 1960 cpu_pm_unregister_notifier(&etm4_cpu_pm_nb); 1961 cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); 1962 if (hp_online) { 1963 cpuhp_remove_state_nocalls(hp_online); 1964 hp_online = 0; 1965 } 1966 } 1967 1968 static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg) 1969 { 1970 int ret; 1971 struct coresight_platform_data *pdata = NULL; 1972 struct device *dev = init_arg->dev; 1973 struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); 1974 struct coresight_desc desc = { 0 }; 1975 u8 major, minor; 1976 char *type_name; 1977 1978 if (!drvdata) 1979 return -EINVAL; 1980 1981 desc.access = *init_arg->csa; 1982 1983 if (!drvdata->arch) 1984 return -EINVAL; 1985 1986 /* TRCPDCR is not accessible with system instructions. */ 1987 if (!desc.access.io_mem || 1988 fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up")) 1989 drvdata->skip_power_up = true; 1990 1991 major = ETM_ARCH_MAJOR_VERSION(drvdata->arch); 1992 minor = ETM_ARCH_MINOR_VERSION(drvdata->arch); 1993 1994 if (etm4x_is_ete(drvdata)) { 1995 type_name = "ete"; 1996 /* ETE v1 has major version == 0b101. Adjust this for logging.*/ 1997 major -= 4; 1998 } else { 1999 type_name = "etm"; 2000 } 2001 2002 desc.name = devm_kasprintf(dev, GFP_KERNEL, 2003 "%s%d", type_name, drvdata->cpu); 2004 if (!desc.name) 2005 return -ENOMEM; 2006 2007 etm4_set_default(&drvdata->config); 2008 2009 pdata = coresight_get_platform_data(dev); 2010 if (IS_ERR(pdata)) 2011 return PTR_ERR(pdata); 2012 2013 dev->platform_data = pdata; 2014 2015 desc.type = CORESIGHT_DEV_TYPE_SOURCE; 2016 desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC; 2017 desc.ops = &etm4_cs_ops; 2018 desc.pdata = pdata; 2019 desc.dev = dev; 2020 desc.groups = coresight_etmv4_groups; 2021 drvdata->csdev = coresight_register(&desc); 2022 if (IS_ERR(drvdata->csdev)) 2023 return PTR_ERR(drvdata->csdev); 2024 2025 ret = etm_perf_symlink(drvdata->csdev, true); 2026 if (ret) { 2027 coresight_unregister(drvdata->csdev); 2028 return ret; 2029 } 2030 2031 /* register with config infrastructure & load any current features */ 2032 ret = etm4_cscfg_register(drvdata->csdev); 2033 if (ret) { 2034 coresight_unregister(drvdata->csdev); 2035 return ret; 2036 } 2037 2038 etmdrvdata[drvdata->cpu] = drvdata; 2039 2040 dev_info(&drvdata->csdev->dev, "CPU%d: %s v%d.%d initialized\n", 2041 drvdata->cpu, type_name, major, minor); 2042 2043 if (boot_enable) { 2044 coresight_enable(drvdata->csdev); 2045 drvdata->boot_enable = true; 2046 } 2047 2048 return 0; 2049 } 2050 2051 static int etm4_probe(struct device *dev, void __iomem *base, u32 etm_pid) 2052 { 2053 struct etmv4_drvdata *drvdata; 2054 struct csdev_access access = { 0 }; 2055 struct etm4_init_arg init_arg = { 0 }; 2056 struct etm4_init_arg *delayed; 2057 2058 drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); 2059 if (!drvdata) 2060 return -ENOMEM; 2061 2062 dev_set_drvdata(dev, drvdata); 2063 2064 if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE) 2065 pm_save_enable = coresight_loses_context_with_cpu(dev) ? 2066 PARAM_PM_SAVE_SELF_HOSTED : PARAM_PM_SAVE_NEVER; 2067 2068 if (pm_save_enable != PARAM_PM_SAVE_NEVER) { 2069 drvdata->save_state = devm_kmalloc(dev, 2070 sizeof(struct etmv4_save_state), GFP_KERNEL); 2071 if (!drvdata->save_state) 2072 return -ENOMEM; 2073 } 2074 2075 drvdata->base = base; 2076 2077 spin_lock_init(&drvdata->spinlock); 2078 2079 drvdata->cpu = coresight_get_cpu(dev); 2080 if (drvdata->cpu < 0) 2081 return drvdata->cpu; 2082 2083 init_arg.dev = dev; 2084 init_arg.csa = &access; 2085 init_arg.pid = etm_pid; 2086 2087 /* 2088 * Serialize against CPUHP callbacks to avoid race condition 2089 * between the smp call and saving the delayed probe. 2090 */ 2091 cpus_read_lock(); 2092 if (smp_call_function_single(drvdata->cpu, 2093 etm4_init_arch_data, &init_arg, 1)) { 2094 /* The CPU was offline, try again once it comes online. */ 2095 delayed = devm_kmalloc(dev, sizeof(*delayed), GFP_KERNEL); 2096 if (!delayed) { 2097 cpus_read_unlock(); 2098 return -ENOMEM; 2099 } 2100 2101 *delayed = init_arg; 2102 2103 per_cpu(delayed_probe, drvdata->cpu) = delayed; 2104 2105 cpus_read_unlock(); 2106 return 0; 2107 } 2108 cpus_read_unlock(); 2109 2110 return etm4_add_coresight_dev(&init_arg); 2111 } 2112 2113 static int etm4_probe_amba(struct amba_device *adev, const struct amba_id *id) 2114 { 2115 void __iomem *base; 2116 struct device *dev = &adev->dev; 2117 struct resource *res = &adev->res; 2118 int ret; 2119 2120 /* Validity for the resource is already checked by the AMBA core */ 2121 base = devm_ioremap_resource(dev, res); 2122 if (IS_ERR(base)) 2123 return PTR_ERR(base); 2124 2125 ret = etm4_probe(dev, base, id->id); 2126 if (!ret) 2127 pm_runtime_put(&adev->dev); 2128 2129 return ret; 2130 } 2131 2132 static int etm4_probe_platform_dev(struct platform_device *pdev) 2133 { 2134 int ret; 2135 2136 pm_runtime_get_noresume(&pdev->dev); 2137 pm_runtime_set_active(&pdev->dev); 2138 pm_runtime_enable(&pdev->dev); 2139 2140 /* 2141 * System register based devices could match the 2142 * HW by reading appropriate registers on the HW 2143 * and thus we could skip the PID. 2144 */ 2145 ret = etm4_probe(&pdev->dev, NULL, 0); 2146 2147 pm_runtime_put(&pdev->dev); 2148 return ret; 2149 } 2150 2151 static int etm4_probe_cpu(unsigned int cpu) 2152 { 2153 int ret; 2154 struct etm4_init_arg init_arg; 2155 struct csdev_access access = { 0 }; 2156 struct etm4_init_arg *iap = *this_cpu_ptr(&delayed_probe); 2157 2158 if (!iap) 2159 return 0; 2160 2161 init_arg = *iap; 2162 devm_kfree(init_arg.dev, iap); 2163 *this_cpu_ptr(&delayed_probe) = NULL; 2164 2165 ret = pm_runtime_resume_and_get(init_arg.dev); 2166 if (ret < 0) { 2167 dev_err(init_arg.dev, "Failed to get PM runtime!\n"); 2168 return 0; 2169 } 2170 2171 init_arg.csa = &access; 2172 etm4_init_arch_data(&init_arg); 2173 2174 etm4_add_coresight_dev(&init_arg); 2175 2176 pm_runtime_put(init_arg.dev); 2177 return 0; 2178 } 2179 2180 static struct amba_cs_uci_id uci_id_etm4[] = { 2181 { 2182 /* ETMv4 UCI data */ 2183 .devarch = ETM_DEVARCH_ETMv4x_ARCH, 2184 .devarch_mask = ETM_DEVARCH_ID_MASK, 2185 .devtype = 0x00000013, 2186 } 2187 }; 2188 2189 static void clear_etmdrvdata(void *info) 2190 { 2191 int cpu = *(int *)info; 2192 2193 etmdrvdata[cpu] = NULL; 2194 per_cpu(delayed_probe, cpu) = NULL; 2195 } 2196 2197 static int __exit etm4_remove_dev(struct etmv4_drvdata *drvdata) 2198 { 2199 bool had_delayed_probe; 2200 /* 2201 * Taking hotplug lock here to avoid racing between etm4_remove_dev() 2202 * and CPU hotplug call backs. 2203 */ 2204 cpus_read_lock(); 2205 2206 had_delayed_probe = per_cpu(delayed_probe, drvdata->cpu); 2207 2208 /* 2209 * The readers for etmdrvdata[] are CPU hotplug call backs 2210 * and PM notification call backs. Change etmdrvdata[i] on 2211 * CPU i ensures these call backs has consistent view 2212 * inside one call back function. 2213 */ 2214 if (smp_call_function_single(drvdata->cpu, clear_etmdrvdata, &drvdata->cpu, 1)) 2215 clear_etmdrvdata(&drvdata->cpu); 2216 2217 cpus_read_unlock(); 2218 2219 if (!had_delayed_probe) { 2220 etm_perf_symlink(drvdata->csdev, false); 2221 cscfg_unregister_csdev(drvdata->csdev); 2222 coresight_unregister(drvdata->csdev); 2223 } 2224 2225 return 0; 2226 } 2227 2228 static void __exit etm4_remove_amba(struct amba_device *adev) 2229 { 2230 struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev); 2231 2232 if (drvdata) 2233 etm4_remove_dev(drvdata); 2234 } 2235 2236 static int __exit etm4_remove_platform_dev(struct platform_device *pdev) 2237 { 2238 int ret = 0; 2239 struct etmv4_drvdata *drvdata = dev_get_drvdata(&pdev->dev); 2240 2241 if (drvdata) 2242 ret = etm4_remove_dev(drvdata); 2243 pm_runtime_disable(&pdev->dev); 2244 return ret; 2245 } 2246 2247 static const struct amba_id etm4_ids[] = { 2248 CS_AMBA_ID(0x000bb95d), /* Cortex-A53 */ 2249 CS_AMBA_ID(0x000bb95e), /* Cortex-A57 */ 2250 CS_AMBA_ID(0x000bb95a), /* Cortex-A72 */ 2251 CS_AMBA_ID(0x000bb959), /* Cortex-A73 */ 2252 CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),/* Cortex-A35 */ 2253 CS_AMBA_UCI_ID(0x000bbd05, uci_id_etm4),/* Cortex-A55 */ 2254 CS_AMBA_UCI_ID(0x000bbd0a, uci_id_etm4),/* Cortex-A75 */ 2255 CS_AMBA_UCI_ID(0x000bbd0c, uci_id_etm4),/* Neoverse N1 */ 2256 CS_AMBA_UCI_ID(0x000bbd41, uci_id_etm4),/* Cortex-A78 */ 2257 CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),/* Qualcomm Kryo */ 2258 CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),/* Qualcomm Kryo */ 2259 CS_AMBA_UCI_ID(0x000bb802, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A55 */ 2260 CS_AMBA_UCI_ID(0x000bb803, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A75 */ 2261 CS_AMBA_UCI_ID(0x000bb805, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A55 */ 2262 CS_AMBA_UCI_ID(0x000bb804, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A76 */ 2263 CS_AMBA_UCI_ID(0x000bbd0d, uci_id_etm4),/* Qualcomm Kryo 5XX Cortex-A77 */ 2264 CS_AMBA_UCI_ID(0x000cc0af, uci_id_etm4),/* Marvell ThunderX2 */ 2265 CS_AMBA_UCI_ID(0x000b6d01, uci_id_etm4),/* HiSilicon-Hip08 */ 2266 CS_AMBA_UCI_ID(0x000b6d02, uci_id_etm4),/* HiSilicon-Hip09 */ 2267 {}, 2268 }; 2269 2270 MODULE_DEVICE_TABLE(amba, etm4_ids); 2271 2272 static struct amba_driver etm4x_amba_driver = { 2273 .drv = { 2274 .name = "coresight-etm4x", 2275 .owner = THIS_MODULE, 2276 .suppress_bind_attrs = true, 2277 }, 2278 .probe = etm4_probe_amba, 2279 .remove = etm4_remove_amba, 2280 .id_table = etm4_ids, 2281 }; 2282 2283 static const struct of_device_id etm4_sysreg_match[] = { 2284 { .compatible = "arm,coresight-etm4x-sysreg" }, 2285 { .compatible = "arm,embedded-trace-extension" }, 2286 {} 2287 }; 2288 2289 static struct platform_driver etm4_platform_driver = { 2290 .probe = etm4_probe_platform_dev, 2291 .remove = etm4_remove_platform_dev, 2292 .driver = { 2293 .name = "coresight-etm4x", 2294 .of_match_table = etm4_sysreg_match, 2295 .suppress_bind_attrs = true, 2296 }, 2297 }; 2298 2299 static int __init etm4x_init(void) 2300 { 2301 int ret; 2302 2303 ret = etm4_pm_setup(); 2304 2305 /* etm4_pm_setup() does its own cleanup - exit on error */ 2306 if (ret) 2307 return ret; 2308 2309 ret = amba_driver_register(&etm4x_amba_driver); 2310 if (ret) { 2311 pr_err("Error registering etm4x AMBA driver\n"); 2312 goto clear_pm; 2313 } 2314 2315 ret = platform_driver_register(&etm4_platform_driver); 2316 if (!ret) 2317 return 0; 2318 2319 pr_err("Error registering etm4x platform driver\n"); 2320 amba_driver_unregister(&etm4x_amba_driver); 2321 2322 clear_pm: 2323 etm4_pm_clear(); 2324 return ret; 2325 } 2326 2327 static void __exit etm4x_exit(void) 2328 { 2329 amba_driver_unregister(&etm4x_amba_driver); 2330 platform_driver_unregister(&etm4_platform_driver); 2331 etm4_pm_clear(); 2332 } 2333 2334 module_init(etm4x_init); 2335 module_exit(etm4x_exit); 2336 2337 MODULE_AUTHOR("Pratik Patel <pratikp@codeaurora.org>"); 2338 MODULE_AUTHOR("Mathieu Poirier <mathieu.poirier@linaro.org>"); 2339 MODULE_DESCRIPTION("Arm CoreSight Program Flow Trace v4.x driver"); 2340 MODULE_LICENSE("GPL v2"); 2341