1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "df_v3_6.h" 25 26 #include "df/df_3_6_default.h" 27 #include "df/df_3_6_offset.h" 28 #include "df/df_3_6_sh_mask.h" 29 30 static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0, 31 16, 32, 0, 0, 0, 2, 4, 8}; 32 33 /* init df format attrs */ 34 AMDGPU_PMU_ATTR(event, "config:0-7"); 35 AMDGPU_PMU_ATTR(instance, "config:8-15"); 36 AMDGPU_PMU_ATTR(umask, "config:16-23"); 37 38 /* df format attributes */ 39 static struct attribute *df_v3_6_format_attrs[] = { 40 &pmu_attr_event.attr, 41 &pmu_attr_instance.attr, 42 &pmu_attr_umask.attr, 43 NULL 44 }; 45 46 /* df format attribute group */ 47 static struct attribute_group df_v3_6_format_attr_group = { 48 .name = "format", 49 .attrs = df_v3_6_format_attrs, 50 }; 51 52 /* df event attrs */ 53 AMDGPU_PMU_ATTR(cake0_pcsout_txdata, 54 "event=0x7,instance=0x46,umask=0x2"); 55 AMDGPU_PMU_ATTR(cake1_pcsout_txdata, 56 "event=0x7,instance=0x47,umask=0x2"); 57 AMDGPU_PMU_ATTR(cake0_pcsout_txmeta, 58 "event=0x7,instance=0x46,umask=0x4"); 59 AMDGPU_PMU_ATTR(cake1_pcsout_txmeta, 60 "event=0x7,instance=0x47,umask=0x4"); 61 AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc, 62 "event=0xb,instance=0x46,umask=0x4"); 63 AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc, 64 "event=0xb,instance=0x47,umask=0x4"); 65 AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc, 66 "event=0xb,instance=0x46,umask=0x8"); 67 AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc, 68 "event=0xb,instance=0x47,umask=0x8"); 69 70 /* df event attributes */ 71 static struct attribute *df_v3_6_event_attrs[] = { 72 &pmu_attr_cake0_pcsout_txdata.attr, 73 &pmu_attr_cake1_pcsout_txdata.attr, 74 &pmu_attr_cake0_pcsout_txmeta.attr, 75 &pmu_attr_cake1_pcsout_txmeta.attr, 76 &pmu_attr_cake0_ftiinstat_reqalloc.attr, 77 &pmu_attr_cake1_ftiinstat_reqalloc.attr, 78 &pmu_attr_cake0_ftiinstat_rspalloc.attr, 79 &pmu_attr_cake1_ftiinstat_rspalloc.attr, 80 NULL 81 }; 82 83 /* df event attribute group */ 84 static struct attribute_group df_v3_6_event_attr_group = { 85 .name = "events", 86 .attrs = df_v3_6_event_attrs 87 }; 88 89 /* df event attr groups */ 90 const struct attribute_group *df_v3_6_attr_groups[] = { 91 &df_v3_6_format_attr_group, 92 &df_v3_6_event_attr_group, 93 NULL 94 }; 95 96 static uint64_t df_v3_6_get_fica(struct amdgpu_device *adev, 97 uint32_t ficaa_val) 98 { 99 unsigned long flags, address, data; 100 uint32_t ficadl_val, ficadh_val; 101 102 address = adev->nbio.funcs->get_pcie_index_offset(adev); 103 data = adev->nbio.funcs->get_pcie_data_offset(adev); 104 105 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 106 WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessAddress3); 107 WREG32(data, ficaa_val); 108 109 WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataLo3); 110 ficadl_val = RREG32(data); 111 112 WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataHi3); 113 ficadh_val = RREG32(data); 114 115 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 116 117 return (((ficadh_val & 0xFFFFFFFFFFFFFFFF) << 32) | ficadl_val); 118 } 119 120 static void df_v3_6_set_fica(struct amdgpu_device *adev, uint32_t ficaa_val, 121 uint32_t ficadl_val, uint32_t ficadh_val) 122 { 123 unsigned long flags, address, data; 124 125 address = adev->nbio.funcs->get_pcie_index_offset(adev); 126 data = adev->nbio.funcs->get_pcie_data_offset(adev); 127 128 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 129 WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessAddress3); 130 WREG32(data, ficaa_val); 131 132 WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataLo3); 133 WREG32(data, ficadl_val); 134 135 WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataHi3); 136 WREG32(data, ficadh_val); 137 138 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 139 } 140 141 /* 142 * df_v3_6_perfmon_rreg - read perfmon lo and hi 143 * 144 * required to be atomic. no mmio method provided so subsequent reads for lo 145 * and hi require to preserve df finite state machine 146 */ 147 static void df_v3_6_perfmon_rreg(struct amdgpu_device *adev, 148 uint32_t lo_addr, uint32_t *lo_val, 149 uint32_t hi_addr, uint32_t *hi_val) 150 { 151 unsigned long flags, address, data; 152 153 address = adev->nbio.funcs->get_pcie_index_offset(adev); 154 data = adev->nbio.funcs->get_pcie_data_offset(adev); 155 156 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 157 WREG32(address, lo_addr); 158 *lo_val = RREG32(data); 159 WREG32(address, hi_addr); 160 *hi_val = RREG32(data); 161 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 162 } 163 164 /* 165 * df_v3_6_perfmon_wreg - write to perfmon lo and hi 166 * 167 * required to be atomic. no mmio method provided so subsequent reads after 168 * data writes cannot occur to preserve data fabrics finite state machine. 169 */ 170 static void df_v3_6_perfmon_wreg(struct amdgpu_device *adev, uint32_t lo_addr, 171 uint32_t lo_val, uint32_t hi_addr, uint32_t hi_val) 172 { 173 unsigned long flags, address, data; 174 175 address = adev->nbio.funcs->get_pcie_index_offset(adev); 176 data = adev->nbio.funcs->get_pcie_data_offset(adev); 177 178 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 179 WREG32(address, lo_addr); 180 WREG32(data, lo_val); 181 WREG32(address, hi_addr); 182 WREG32(data, hi_val); 183 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 184 } 185 186 /* get the number of df counters available */ 187 static ssize_t df_v3_6_get_df_cntr_avail(struct device *dev, 188 struct device_attribute *attr, 189 char *buf) 190 { 191 struct amdgpu_device *adev; 192 struct drm_device *ddev; 193 int i, count; 194 195 ddev = dev_get_drvdata(dev); 196 adev = ddev->dev_private; 197 count = 0; 198 199 for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) { 200 if (adev->df_perfmon_config_assign_mask[i] == 0) 201 count++; 202 } 203 204 return snprintf(buf, PAGE_SIZE, "%i\n", count); 205 } 206 207 /* device attr for available perfmon counters */ 208 static DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL); 209 210 /* init perfmons */ 211 static void df_v3_6_sw_init(struct amdgpu_device *adev) 212 { 213 int i, ret; 214 215 ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail); 216 if (ret) 217 DRM_ERROR("failed to create file for available df counters\n"); 218 219 for (i = 0; i < AMDGPU_MAX_DF_PERFMONS; i++) 220 adev->df_perfmon_config_assign_mask[i] = 0; 221 } 222 223 static void df_v3_6_sw_fini(struct amdgpu_device *adev) 224 { 225 226 device_remove_file(adev->dev, &dev_attr_df_cntr_avail); 227 228 } 229 230 static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev, 231 bool enable) 232 { 233 u32 tmp; 234 235 if (enable) { 236 tmp = RREG32_SOC15(DF, 0, mmFabricConfigAccessControl); 237 tmp &= ~FabricConfigAccessControl__CfgRegInstAccEn_MASK; 238 WREG32_SOC15(DF, 0, mmFabricConfigAccessControl, tmp); 239 } else 240 WREG32_SOC15(DF, 0, mmFabricConfigAccessControl, 241 mmFabricConfigAccessControl_DEFAULT); 242 } 243 244 static u32 df_v3_6_get_fb_channel_number(struct amdgpu_device *adev) 245 { 246 u32 tmp; 247 248 tmp = RREG32_SOC15(DF, 0, mmDF_CS_UMC_AON0_DramBaseAddress0); 249 tmp &= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK; 250 tmp >>= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan__SHIFT; 251 252 return tmp; 253 } 254 255 static u32 df_v3_6_get_hbm_channel_number(struct amdgpu_device *adev) 256 { 257 int fb_channel_number; 258 259 fb_channel_number = adev->df_funcs->get_fb_channel_number(adev); 260 if (fb_channel_number >= ARRAY_SIZE(df_v3_6_channel_number)) 261 fb_channel_number = 0; 262 263 return df_v3_6_channel_number[fb_channel_number]; 264 } 265 266 static void df_v3_6_update_medium_grain_clock_gating(struct amdgpu_device *adev, 267 bool enable) 268 { 269 u32 tmp; 270 271 if (adev->cg_flags & AMD_CG_SUPPORT_DF_MGCG) { 272 /* Put DF on broadcast mode */ 273 adev->df_funcs->enable_broadcast_mode(adev, true); 274 275 if (enable) { 276 tmp = RREG32_SOC15(DF, 0, 277 mmDF_PIE_AON0_DfGlobalClkGater); 278 tmp &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK; 279 tmp |= DF_V3_6_MGCG_ENABLE_15_CYCLE_DELAY; 280 WREG32_SOC15(DF, 0, 281 mmDF_PIE_AON0_DfGlobalClkGater, tmp); 282 } else { 283 tmp = RREG32_SOC15(DF, 0, 284 mmDF_PIE_AON0_DfGlobalClkGater); 285 tmp &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK; 286 tmp |= DF_V3_6_MGCG_DISABLE; 287 WREG32_SOC15(DF, 0, 288 mmDF_PIE_AON0_DfGlobalClkGater, tmp); 289 } 290 291 /* Exit broadcast mode */ 292 adev->df_funcs->enable_broadcast_mode(adev, false); 293 } 294 } 295 296 static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev, 297 u32 *flags) 298 { 299 u32 tmp; 300 301 /* AMD_CG_SUPPORT_DF_MGCG */ 302 tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater); 303 if (tmp & DF_V3_6_MGCG_ENABLE_15_CYCLE_DELAY) 304 *flags |= AMD_CG_SUPPORT_DF_MGCG; 305 } 306 307 /* get assigned df perfmon ctr as int */ 308 static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev, 309 uint64_t config) 310 { 311 int i; 312 313 for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) { 314 if ((config & 0x0FFFFFFUL) == 315 adev->df_perfmon_config_assign_mask[i]) 316 return i; 317 } 318 319 return -EINVAL; 320 } 321 322 /* get address based on counter assignment */ 323 static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev, 324 uint64_t config, 325 int is_ctrl, 326 uint32_t *lo_base_addr, 327 uint32_t *hi_base_addr) 328 { 329 int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config); 330 331 if (target_cntr < 0) 332 return; 333 334 switch (target_cntr) { 335 336 case 0: 337 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo0 : smnPerfMonCtrLo0; 338 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi0 : smnPerfMonCtrHi0; 339 break; 340 case 1: 341 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo1 : smnPerfMonCtrLo1; 342 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi1 : smnPerfMonCtrHi1; 343 break; 344 case 2: 345 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo2 : smnPerfMonCtrLo2; 346 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi2 : smnPerfMonCtrHi2; 347 break; 348 case 3: 349 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo3 : smnPerfMonCtrLo3; 350 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi3 : smnPerfMonCtrHi3; 351 break; 352 353 } 354 355 } 356 357 /* get read counter address */ 358 static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev, 359 uint64_t config, 360 uint32_t *lo_base_addr, 361 uint32_t *hi_base_addr) 362 { 363 df_v3_6_pmc_get_addr(adev, config, 0, lo_base_addr, hi_base_addr); 364 } 365 366 /* get control counter settings i.e. address and values to set */ 367 static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev, 368 uint64_t config, 369 uint32_t *lo_base_addr, 370 uint32_t *hi_base_addr, 371 uint32_t *lo_val, 372 uint32_t *hi_val) 373 { 374 375 uint32_t eventsel, instance, unitmask; 376 uint32_t instance_10, instance_5432, instance_76; 377 378 df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr); 379 380 if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) { 381 DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x", 382 *lo_base_addr, *hi_base_addr); 383 return -ENXIO; 384 } 385 386 eventsel = DF_V3_6_GET_EVENT(config) & 0x3f; 387 unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf; 388 instance = DF_V3_6_GET_INSTANCE(config); 389 390 instance_10 = instance & 0x3; 391 instance_5432 = (instance >> 2) & 0xf; 392 instance_76 = (instance >> 6) & 0x3; 393 394 *lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel | (1 << 22); 395 *hi_val = (instance_76 << 29) | instance_5432; 396 397 DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x", 398 config, *lo_base_addr, *hi_base_addr, *lo_val, *hi_val); 399 400 return 0; 401 } 402 403 /* add df performance counters for read */ 404 static int df_v3_6_pmc_add_cntr(struct amdgpu_device *adev, 405 uint64_t config) 406 { 407 int i, target_cntr; 408 409 target_cntr = df_v3_6_pmc_config_2_cntr(adev, config); 410 411 if (target_cntr >= 0) 412 return 0; 413 414 for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) { 415 if (adev->df_perfmon_config_assign_mask[i] == 0U) { 416 adev->df_perfmon_config_assign_mask[i] = 417 config & 0x0FFFFFFUL; 418 return 0; 419 } 420 } 421 422 return -ENOSPC; 423 } 424 425 /* release performance counter */ 426 static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev, 427 uint64_t config) 428 { 429 int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config); 430 431 if (target_cntr >= 0) 432 adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL; 433 } 434 435 436 static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev, 437 uint64_t config) 438 { 439 uint32_t lo_base_addr, hi_base_addr; 440 441 df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr, 442 &hi_base_addr); 443 444 if ((lo_base_addr == 0) || (hi_base_addr == 0)) 445 return; 446 447 df_v3_6_perfmon_wreg(adev, lo_base_addr, 0, hi_base_addr, 0); 448 } 449 450 static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config, 451 int is_enable) 452 { 453 uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val; 454 int ret = 0; 455 456 switch (adev->asic_type) { 457 case CHIP_VEGA20: 458 459 df_v3_6_reset_perfmon_cntr(adev, config); 460 461 if (is_enable) { 462 ret = df_v3_6_pmc_add_cntr(adev, config); 463 } else { 464 ret = df_v3_6_pmc_get_ctrl_settings(adev, 465 config, 466 &lo_base_addr, 467 &hi_base_addr, 468 &lo_val, 469 &hi_val); 470 471 if (ret) 472 return ret; 473 474 df_v3_6_perfmon_wreg(adev, lo_base_addr, lo_val, 475 hi_base_addr, hi_val); 476 } 477 478 break; 479 default: 480 break; 481 } 482 483 return ret; 484 } 485 486 static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config, 487 int is_disable) 488 { 489 uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val; 490 int ret = 0; 491 492 switch (adev->asic_type) { 493 case CHIP_VEGA20: 494 ret = df_v3_6_pmc_get_ctrl_settings(adev, 495 config, 496 &lo_base_addr, 497 &hi_base_addr, 498 &lo_val, 499 &hi_val); 500 501 if (ret) 502 return ret; 503 504 df_v3_6_perfmon_wreg(adev, lo_base_addr, 0, hi_base_addr, 0); 505 506 if (is_disable) 507 df_v3_6_pmc_release_cntr(adev, config); 508 509 break; 510 default: 511 break; 512 } 513 514 return ret; 515 } 516 517 static void df_v3_6_pmc_get_count(struct amdgpu_device *adev, 518 uint64_t config, 519 uint64_t *count) 520 { 521 uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val; 522 *count = 0; 523 524 switch (adev->asic_type) { 525 case CHIP_VEGA20: 526 527 df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr, 528 &hi_base_addr); 529 530 if ((lo_base_addr == 0) || (hi_base_addr == 0)) 531 return; 532 533 df_v3_6_perfmon_rreg(adev, lo_base_addr, &lo_val, 534 hi_base_addr, &hi_val); 535 536 *count = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL); 537 538 if (*count >= DF_V3_6_PERFMON_OVERFLOW) 539 *count = 0; 540 541 DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x", 542 config, lo_base_addr, hi_base_addr, lo_val, hi_val); 543 544 break; 545 546 default: 547 break; 548 } 549 } 550 551 const struct amdgpu_df_funcs df_v3_6_funcs = { 552 .sw_init = df_v3_6_sw_init, 553 .sw_fini = df_v3_6_sw_fini, 554 .enable_broadcast_mode = df_v3_6_enable_broadcast_mode, 555 .get_fb_channel_number = df_v3_6_get_fb_channel_number, 556 .get_hbm_channel_number = df_v3_6_get_hbm_channel_number, 557 .update_medium_grain_clock_gating = 558 df_v3_6_update_medium_grain_clock_gating, 559 .get_clockgating_state = df_v3_6_get_clockgating_state, 560 .pmc_start = df_v3_6_pmc_start, 561 .pmc_stop = df_v3_6_pmc_stop, 562 .pmc_get_count = df_v3_6_pmc_get_count, 563 .get_fica = df_v3_6_get_fica, 564 .set_fica = df_v3_6_set_fica 565 }; 566