1 /* 2 * Copyright 2012 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <linux/module.h> 26 #include <drm/drmP.h> 27 #include "radeon.h" 28 #include "radeon_asic.h" 29 #include "radeon_audio.h" 30 #include "cikd.h" 31 #include "atom.h" 32 #include "cik_blit_shaders.h" 33 #include "radeon_ucode.h" 34 #include "clearstate_ci.h" 35 36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin"); 37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin"); 38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin"); 39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin"); 40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin"); 41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin"); 42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin"); 43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin"); 44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin"); 45 46 MODULE_FIRMWARE("radeon/bonaire_pfp.bin"); 47 MODULE_FIRMWARE("radeon/bonaire_me.bin"); 48 MODULE_FIRMWARE("radeon/bonaire_ce.bin"); 49 MODULE_FIRMWARE("radeon/bonaire_mec.bin"); 50 MODULE_FIRMWARE("radeon/bonaire_mc.bin"); 51 MODULE_FIRMWARE("radeon/bonaire_rlc.bin"); 52 MODULE_FIRMWARE("radeon/bonaire_sdma.bin"); 53 MODULE_FIRMWARE("radeon/bonaire_smc.bin"); 54 55 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin"); 56 MODULE_FIRMWARE("radeon/HAWAII_me.bin"); 57 MODULE_FIRMWARE("radeon/HAWAII_ce.bin"); 58 MODULE_FIRMWARE("radeon/HAWAII_mec.bin"); 59 MODULE_FIRMWARE("radeon/HAWAII_mc.bin"); 60 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin"); 61 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin"); 62 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin"); 63 MODULE_FIRMWARE("radeon/HAWAII_smc.bin"); 64 65 MODULE_FIRMWARE("radeon/hawaii_pfp.bin"); 66 MODULE_FIRMWARE("radeon/hawaii_me.bin"); 67 MODULE_FIRMWARE("radeon/hawaii_ce.bin"); 68 MODULE_FIRMWARE("radeon/hawaii_mec.bin"); 69 MODULE_FIRMWARE("radeon/hawaii_mc.bin"); 70 MODULE_FIRMWARE("radeon/hawaii_rlc.bin"); 71 MODULE_FIRMWARE("radeon/hawaii_sdma.bin"); 72 MODULE_FIRMWARE("radeon/hawaii_smc.bin"); 73 74 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin"); 75 MODULE_FIRMWARE("radeon/KAVERI_me.bin"); 76 MODULE_FIRMWARE("radeon/KAVERI_ce.bin"); 77 MODULE_FIRMWARE("radeon/KAVERI_mec.bin"); 78 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin"); 79 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin"); 80 81 MODULE_FIRMWARE("radeon/kaveri_pfp.bin"); 82 MODULE_FIRMWARE("radeon/kaveri_me.bin"); 83 MODULE_FIRMWARE("radeon/kaveri_ce.bin"); 84 MODULE_FIRMWARE("radeon/kaveri_mec.bin"); 85 MODULE_FIRMWARE("radeon/kaveri_mec2.bin"); 86 MODULE_FIRMWARE("radeon/kaveri_rlc.bin"); 87 MODULE_FIRMWARE("radeon/kaveri_sdma.bin"); 88 89 MODULE_FIRMWARE("radeon/KABINI_pfp.bin"); 90 MODULE_FIRMWARE("radeon/KABINI_me.bin"); 91 MODULE_FIRMWARE("radeon/KABINI_ce.bin"); 92 MODULE_FIRMWARE("radeon/KABINI_mec.bin"); 93 MODULE_FIRMWARE("radeon/KABINI_rlc.bin"); 94 MODULE_FIRMWARE("radeon/KABINI_sdma.bin"); 95 96 MODULE_FIRMWARE("radeon/kabini_pfp.bin"); 97 MODULE_FIRMWARE("radeon/kabini_me.bin"); 98 MODULE_FIRMWARE("radeon/kabini_ce.bin"); 99 MODULE_FIRMWARE("radeon/kabini_mec.bin"); 100 MODULE_FIRMWARE("radeon/kabini_rlc.bin"); 101 MODULE_FIRMWARE("radeon/kabini_sdma.bin"); 102 103 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin"); 104 MODULE_FIRMWARE("radeon/MULLINS_me.bin"); 105 MODULE_FIRMWARE("radeon/MULLINS_ce.bin"); 106 MODULE_FIRMWARE("radeon/MULLINS_mec.bin"); 107 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin"); 108 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin"); 109 110 MODULE_FIRMWARE("radeon/mullins_pfp.bin"); 111 MODULE_FIRMWARE("radeon/mullins_me.bin"); 112 MODULE_FIRMWARE("radeon/mullins_ce.bin"); 113 MODULE_FIRMWARE("radeon/mullins_mec.bin"); 114 MODULE_FIRMWARE("radeon/mullins_rlc.bin"); 115 MODULE_FIRMWARE("radeon/mullins_sdma.bin"); 116 117 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh); 118 static void cik_rlc_stop(struct radeon_device *rdev); 119 static void cik_pcie_gen3_enable(struct radeon_device *rdev); 120 static void cik_program_aspm(struct radeon_device *rdev); 121 static void cik_init_pg(struct radeon_device *rdev); 122 static void cik_init_cg(struct radeon_device *rdev); 123 static void cik_fini_pg(struct radeon_device *rdev); 124 static void cik_fini_cg(struct radeon_device *rdev); 125 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, 126 bool enable); 127 /** 128 * cik_get_allowed_info_register - fetch the register for the info ioctl 129 * 130 * @rdev: radeon_device pointer 131 * @reg: register offset in bytes 132 * @val: register value 133 * 134 * Returns 0 for success or -EINVAL for an invalid register 135 * 136 */ 137 int cik_get_allowed_info_register(struct radeon_device *rdev, 138 u32 reg, u32 *val) 139 { 140 switch (reg) { 141 case GRBM_STATUS: 142 case GRBM_STATUS2: 143 case GRBM_STATUS_SE0: 144 case GRBM_STATUS_SE1: 145 case GRBM_STATUS_SE2: 146 case GRBM_STATUS_SE3: 147 case SRBM_STATUS: 148 case SRBM_STATUS2: 149 case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET): 150 case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET): 151 case UVD_STATUS: 152 /* TODO VCE */ 153 *val = RREG32(reg); 154 return 0; 155 default: 156 return -EINVAL; 157 } 158 } 159 160 /* 161 * Indirect registers accessor 162 */ 163 u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg) 164 { 165 unsigned long flags; 166 u32 r; 167 168 spin_lock_irqsave(&rdev->didt_idx_lock, flags); 169 WREG32(CIK_DIDT_IND_INDEX, (reg)); 170 r = RREG32(CIK_DIDT_IND_DATA); 171 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags); 172 return r; 173 } 174 175 void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v) 176 { 177 unsigned long flags; 178 179 spin_lock_irqsave(&rdev->didt_idx_lock, flags); 180 WREG32(CIK_DIDT_IND_INDEX, (reg)); 181 WREG32(CIK_DIDT_IND_DATA, (v)); 182 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags); 183 } 184 185 /* get temperature in millidegrees */ 186 int ci_get_temp(struct radeon_device *rdev) 187 { 188 u32 temp; 189 int actual_temp = 0; 190 191 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >> 192 CTF_TEMP_SHIFT; 193 194 if (temp & 0x200) 195 actual_temp = 255; 196 else 197 actual_temp = temp & 0x1ff; 198 199 actual_temp = actual_temp * 1000; 200 201 return actual_temp; 202 } 203 204 /* get temperature in millidegrees */ 205 int kv_get_temp(struct radeon_device *rdev) 206 { 207 u32 temp; 208 int actual_temp = 0; 209 210 temp = RREG32_SMC(0xC0300E0C); 211 212 if (temp) 213 actual_temp = (temp / 8) - 49; 214 else 215 actual_temp = 0; 216 217 actual_temp = actual_temp * 1000; 218 219 return actual_temp; 220 } 221 222 /* 223 * Indirect registers accessor 224 */ 225 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg) 226 { 227 unsigned long flags; 228 u32 r; 229 230 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); 231 WREG32(PCIE_INDEX, reg); 232 (void)RREG32(PCIE_INDEX); 233 r = RREG32(PCIE_DATA); 234 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); 235 return r; 236 } 237 238 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v) 239 { 240 unsigned long flags; 241 242 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); 243 WREG32(PCIE_INDEX, reg); 244 (void)RREG32(PCIE_INDEX); 245 WREG32(PCIE_DATA, v); 246 (void)RREG32(PCIE_DATA); 247 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); 248 } 249 250 static const u32 spectre_rlc_save_restore_register_list[] = 251 { 252 (0x0e00 << 16) | (0xc12c >> 2), 253 0x00000000, 254 (0x0e00 << 16) | (0xc140 >> 2), 255 0x00000000, 256 (0x0e00 << 16) | (0xc150 >> 2), 257 0x00000000, 258 (0x0e00 << 16) | (0xc15c >> 2), 259 0x00000000, 260 (0x0e00 << 16) | (0xc168 >> 2), 261 0x00000000, 262 (0x0e00 << 16) | (0xc170 >> 2), 263 0x00000000, 264 (0x0e00 << 16) | (0xc178 >> 2), 265 0x00000000, 266 (0x0e00 << 16) | (0xc204 >> 2), 267 0x00000000, 268 (0x0e00 << 16) | (0xc2b4 >> 2), 269 0x00000000, 270 (0x0e00 << 16) | (0xc2b8 >> 2), 271 0x00000000, 272 (0x0e00 << 16) | (0xc2bc >> 2), 273 0x00000000, 274 (0x0e00 << 16) | (0xc2c0 >> 2), 275 0x00000000, 276 (0x0e00 << 16) | (0x8228 >> 2), 277 0x00000000, 278 (0x0e00 << 16) | (0x829c >> 2), 279 0x00000000, 280 (0x0e00 << 16) | (0x869c >> 2), 281 0x00000000, 282 (0x0600 << 16) | (0x98f4 >> 2), 283 0x00000000, 284 (0x0e00 << 16) | (0x98f8 >> 2), 285 0x00000000, 286 (0x0e00 << 16) | (0x9900 >> 2), 287 0x00000000, 288 (0x0e00 << 16) | (0xc260 >> 2), 289 0x00000000, 290 (0x0e00 << 16) | (0x90e8 >> 2), 291 0x00000000, 292 (0x0e00 << 16) | (0x3c000 >> 2), 293 0x00000000, 294 (0x0e00 << 16) | (0x3c00c >> 2), 295 0x00000000, 296 (0x0e00 << 16) | (0x8c1c >> 2), 297 0x00000000, 298 (0x0e00 << 16) | (0x9700 >> 2), 299 0x00000000, 300 (0x0e00 << 16) | (0xcd20 >> 2), 301 0x00000000, 302 (0x4e00 << 16) | (0xcd20 >> 2), 303 0x00000000, 304 (0x5e00 << 16) | (0xcd20 >> 2), 305 0x00000000, 306 (0x6e00 << 16) | (0xcd20 >> 2), 307 0x00000000, 308 (0x7e00 << 16) | (0xcd20 >> 2), 309 0x00000000, 310 (0x8e00 << 16) | (0xcd20 >> 2), 311 0x00000000, 312 (0x9e00 << 16) | (0xcd20 >> 2), 313 0x00000000, 314 (0xae00 << 16) | (0xcd20 >> 2), 315 0x00000000, 316 (0xbe00 << 16) | (0xcd20 >> 2), 317 0x00000000, 318 (0x0e00 << 16) | (0x89bc >> 2), 319 0x00000000, 320 (0x0e00 << 16) | (0x8900 >> 2), 321 0x00000000, 322 0x3, 323 (0x0e00 << 16) | (0xc130 >> 2), 324 0x00000000, 325 (0x0e00 << 16) | (0xc134 >> 2), 326 0x00000000, 327 (0x0e00 << 16) | (0xc1fc >> 2), 328 0x00000000, 329 (0x0e00 << 16) | (0xc208 >> 2), 330 0x00000000, 331 (0x0e00 << 16) | (0xc264 >> 2), 332 0x00000000, 333 (0x0e00 << 16) | (0xc268 >> 2), 334 0x00000000, 335 (0x0e00 << 16) | (0xc26c >> 2), 336 0x00000000, 337 (0x0e00 << 16) | (0xc270 >> 2), 338 0x00000000, 339 (0x0e00 << 16) | (0xc274 >> 2), 340 0x00000000, 341 (0x0e00 << 16) | (0xc278 >> 2), 342 0x00000000, 343 (0x0e00 << 16) | (0xc27c >> 2), 344 0x00000000, 345 (0x0e00 << 16) | (0xc280 >> 2), 346 0x00000000, 347 (0x0e00 << 16) | (0xc284 >> 2), 348 0x00000000, 349 (0x0e00 << 16) | (0xc288 >> 2), 350 0x00000000, 351 (0x0e00 << 16) | (0xc28c >> 2), 352 0x00000000, 353 (0x0e00 << 16) | (0xc290 >> 2), 354 0x00000000, 355 (0x0e00 << 16) | (0xc294 >> 2), 356 0x00000000, 357 (0x0e00 << 16) | (0xc298 >> 2), 358 0x00000000, 359 (0x0e00 << 16) | (0xc29c >> 2), 360 0x00000000, 361 (0x0e00 << 16) | (0xc2a0 >> 2), 362 0x00000000, 363 (0x0e00 << 16) | (0xc2a4 >> 2), 364 0x00000000, 365 (0x0e00 << 16) | (0xc2a8 >> 2), 366 0x00000000, 367 (0x0e00 << 16) | (0xc2ac >> 2), 368 0x00000000, 369 (0x0e00 << 16) | (0xc2b0 >> 2), 370 0x00000000, 371 (0x0e00 << 16) | (0x301d0 >> 2), 372 0x00000000, 373 (0x0e00 << 16) | (0x30238 >> 2), 374 0x00000000, 375 (0x0e00 << 16) | (0x30250 >> 2), 376 0x00000000, 377 (0x0e00 << 16) | (0x30254 >> 2), 378 0x00000000, 379 (0x0e00 << 16) | (0x30258 >> 2), 380 0x00000000, 381 (0x0e00 << 16) | (0x3025c >> 2), 382 0x00000000, 383 (0x4e00 << 16) | (0xc900 >> 2), 384 0x00000000, 385 (0x5e00 << 16) | (0xc900 >> 2), 386 0x00000000, 387 (0x6e00 << 16) | (0xc900 >> 2), 388 0x00000000, 389 (0x7e00 << 16) | (0xc900 >> 2), 390 0x00000000, 391 (0x8e00 << 16) | (0xc900 >> 2), 392 0x00000000, 393 (0x9e00 << 16) | (0xc900 >> 2), 394 0x00000000, 395 (0xae00 << 16) | (0xc900 >> 2), 396 0x00000000, 397 (0xbe00 << 16) | (0xc900 >> 2), 398 0x00000000, 399 (0x4e00 << 16) | (0xc904 >> 2), 400 0x00000000, 401 (0x5e00 << 16) | (0xc904 >> 2), 402 0x00000000, 403 (0x6e00 << 16) | (0xc904 >> 2), 404 0x00000000, 405 (0x7e00 << 16) | (0xc904 >> 2), 406 0x00000000, 407 (0x8e00 << 16) | (0xc904 >> 2), 408 0x00000000, 409 (0x9e00 << 16) | (0xc904 >> 2), 410 0x00000000, 411 (0xae00 << 16) | (0xc904 >> 2), 412 0x00000000, 413 (0xbe00 << 16) | (0xc904 >> 2), 414 0x00000000, 415 (0x4e00 << 16) | (0xc908 >> 2), 416 0x00000000, 417 (0x5e00 << 16) | (0xc908 >> 2), 418 0x00000000, 419 (0x6e00 << 16) | (0xc908 >> 2), 420 0x00000000, 421 (0x7e00 << 16) | (0xc908 >> 2), 422 0x00000000, 423 (0x8e00 << 16) | (0xc908 >> 2), 424 0x00000000, 425 (0x9e00 << 16) | (0xc908 >> 2), 426 0x00000000, 427 (0xae00 << 16) | (0xc908 >> 2), 428 0x00000000, 429 (0xbe00 << 16) | (0xc908 >> 2), 430 0x00000000, 431 (0x4e00 << 16) | (0xc90c >> 2), 432 0x00000000, 433 (0x5e00 << 16) | (0xc90c >> 2), 434 0x00000000, 435 (0x6e00 << 16) | (0xc90c >> 2), 436 0x00000000, 437 (0x7e00 << 16) | (0xc90c >> 2), 438 0x00000000, 439 (0x8e00 << 16) | (0xc90c >> 2), 440 0x00000000, 441 (0x9e00 << 16) | (0xc90c >> 2), 442 0x00000000, 443 (0xae00 << 16) | (0xc90c >> 2), 444 0x00000000, 445 (0xbe00 << 16) | (0xc90c >> 2), 446 0x00000000, 447 (0x4e00 << 16) | (0xc910 >> 2), 448 0x00000000, 449 (0x5e00 << 16) | (0xc910 >> 2), 450 0x00000000, 451 (0x6e00 << 16) | (0xc910 >> 2), 452 0x00000000, 453 (0x7e00 << 16) | (0xc910 >> 2), 454 0x00000000, 455 (0x8e00 << 16) | (0xc910 >> 2), 456 0x00000000, 457 (0x9e00 << 16) | (0xc910 >> 2), 458 0x00000000, 459 (0xae00 << 16) | (0xc910 >> 2), 460 0x00000000, 461 (0xbe00 << 16) | (0xc910 >> 2), 462 0x00000000, 463 (0x0e00 << 16) | (0xc99c >> 2), 464 0x00000000, 465 (0x0e00 << 16) | (0x9834 >> 2), 466 0x00000000, 467 (0x0000 << 16) | (0x30f00 >> 2), 468 0x00000000, 469 (0x0001 << 16) | (0x30f00 >> 2), 470 0x00000000, 471 (0x0000 << 16) | (0x30f04 >> 2), 472 0x00000000, 473 (0x0001 << 16) | (0x30f04 >> 2), 474 0x00000000, 475 (0x0000 << 16) | (0x30f08 >> 2), 476 0x00000000, 477 (0x0001 << 16) | (0x30f08 >> 2), 478 0x00000000, 479 (0x0000 << 16) | (0x30f0c >> 2), 480 0x00000000, 481 (0x0001 << 16) | (0x30f0c >> 2), 482 0x00000000, 483 (0x0600 << 16) | (0x9b7c >> 2), 484 0x00000000, 485 (0x0e00 << 16) | (0x8a14 >> 2), 486 0x00000000, 487 (0x0e00 << 16) | (0x8a18 >> 2), 488 0x00000000, 489 (0x0600 << 16) | (0x30a00 >> 2), 490 0x00000000, 491 (0x0e00 << 16) | (0x8bf0 >> 2), 492 0x00000000, 493 (0x0e00 << 16) | (0x8bcc >> 2), 494 0x00000000, 495 (0x0e00 << 16) | (0x8b24 >> 2), 496 0x00000000, 497 (0x0e00 << 16) | (0x30a04 >> 2), 498 0x00000000, 499 (0x0600 << 16) | (0x30a10 >> 2), 500 0x00000000, 501 (0x0600 << 16) | (0x30a14 >> 2), 502 0x00000000, 503 (0x0600 << 16) | (0x30a18 >> 2), 504 0x00000000, 505 (0x0600 << 16) | (0x30a2c >> 2), 506 0x00000000, 507 (0x0e00 << 16) | (0xc700 >> 2), 508 0x00000000, 509 (0x0e00 << 16) | (0xc704 >> 2), 510 0x00000000, 511 (0x0e00 << 16) | (0xc708 >> 2), 512 0x00000000, 513 (0x0e00 << 16) | (0xc768 >> 2), 514 0x00000000, 515 (0x0400 << 16) | (0xc770 >> 2), 516 0x00000000, 517 (0x0400 << 16) | (0xc774 >> 2), 518 0x00000000, 519 (0x0400 << 16) | (0xc778 >> 2), 520 0x00000000, 521 (0x0400 << 16) | (0xc77c >> 2), 522 0x00000000, 523 (0x0400 << 16) | (0xc780 >> 2), 524 0x00000000, 525 (0x0400 << 16) | (0xc784 >> 2), 526 0x00000000, 527 (0x0400 << 16) | (0xc788 >> 2), 528 0x00000000, 529 (0x0400 << 16) | (0xc78c >> 2), 530 0x00000000, 531 (0x0400 << 16) | (0xc798 >> 2), 532 0x00000000, 533 (0x0400 << 16) | (0xc79c >> 2), 534 0x00000000, 535 (0x0400 << 16) | (0xc7a0 >> 2), 536 0x00000000, 537 (0x0400 << 16) | (0xc7a4 >> 2), 538 0x00000000, 539 (0x0400 << 16) | (0xc7a8 >> 2), 540 0x00000000, 541 (0x0400 << 16) | (0xc7ac >> 2), 542 0x00000000, 543 (0x0400 << 16) | (0xc7b0 >> 2), 544 0x00000000, 545 (0x0400 << 16) | (0xc7b4 >> 2), 546 0x00000000, 547 (0x0e00 << 16) | (0x9100 >> 2), 548 0x00000000, 549 (0x0e00 << 16) | (0x3c010 >> 2), 550 0x00000000, 551 (0x0e00 << 16) | (0x92a8 >> 2), 552 0x00000000, 553 (0x0e00 << 16) | (0x92ac >> 2), 554 0x00000000, 555 (0x0e00 << 16) | (0x92b4 >> 2), 556 0x00000000, 557 (0x0e00 << 16) | (0x92b8 >> 2), 558 0x00000000, 559 (0x0e00 << 16) | (0x92bc >> 2), 560 0x00000000, 561 (0x0e00 << 16) | (0x92c0 >> 2), 562 0x00000000, 563 (0x0e00 << 16) | (0x92c4 >> 2), 564 0x00000000, 565 (0x0e00 << 16) | (0x92c8 >> 2), 566 0x00000000, 567 (0x0e00 << 16) | (0x92cc >> 2), 568 0x00000000, 569 (0x0e00 << 16) | (0x92d0 >> 2), 570 0x00000000, 571 (0x0e00 << 16) | (0x8c00 >> 2), 572 0x00000000, 573 (0x0e00 << 16) | (0x8c04 >> 2), 574 0x00000000, 575 (0x0e00 << 16) | (0x8c20 >> 2), 576 0x00000000, 577 (0x0e00 << 16) | (0x8c38 >> 2), 578 0x00000000, 579 (0x0e00 << 16) | (0x8c3c >> 2), 580 0x00000000, 581 (0x0e00 << 16) | (0xae00 >> 2), 582 0x00000000, 583 (0x0e00 << 16) | (0x9604 >> 2), 584 0x00000000, 585 (0x0e00 << 16) | (0xac08 >> 2), 586 0x00000000, 587 (0x0e00 << 16) | (0xac0c >> 2), 588 0x00000000, 589 (0x0e00 << 16) | (0xac10 >> 2), 590 0x00000000, 591 (0x0e00 << 16) | (0xac14 >> 2), 592 0x00000000, 593 (0x0e00 << 16) | (0xac58 >> 2), 594 0x00000000, 595 (0x0e00 << 16) | (0xac68 >> 2), 596 0x00000000, 597 (0x0e00 << 16) | (0xac6c >> 2), 598 0x00000000, 599 (0x0e00 << 16) | (0xac70 >> 2), 600 0x00000000, 601 (0x0e00 << 16) | (0xac74 >> 2), 602 0x00000000, 603 (0x0e00 << 16) | (0xac78 >> 2), 604 0x00000000, 605 (0x0e00 << 16) | (0xac7c >> 2), 606 0x00000000, 607 (0x0e00 << 16) | (0xac80 >> 2), 608 0x00000000, 609 (0x0e00 << 16) | (0xac84 >> 2), 610 0x00000000, 611 (0x0e00 << 16) | (0xac88 >> 2), 612 0x00000000, 613 (0x0e00 << 16) | (0xac8c >> 2), 614 0x00000000, 615 (0x0e00 << 16) | (0x970c >> 2), 616 0x00000000, 617 (0x0e00 << 16) | (0x9714 >> 2), 618 0x00000000, 619 (0x0e00 << 16) | (0x9718 >> 2), 620 0x00000000, 621 (0x0e00 << 16) | (0x971c >> 2), 622 0x00000000, 623 (0x0e00 << 16) | (0x31068 >> 2), 624 0x00000000, 625 (0x4e00 << 16) | (0x31068 >> 2), 626 0x00000000, 627 (0x5e00 << 16) | (0x31068 >> 2), 628 0x00000000, 629 (0x6e00 << 16) | (0x31068 >> 2), 630 0x00000000, 631 (0x7e00 << 16) | (0x31068 >> 2), 632 0x00000000, 633 (0x8e00 << 16) | (0x31068 >> 2), 634 0x00000000, 635 (0x9e00 << 16) | (0x31068 >> 2), 636 0x00000000, 637 (0xae00 << 16) | (0x31068 >> 2), 638 0x00000000, 639 (0xbe00 << 16) | (0x31068 >> 2), 640 0x00000000, 641 (0x0e00 << 16) | (0xcd10 >> 2), 642 0x00000000, 643 (0x0e00 << 16) | (0xcd14 >> 2), 644 0x00000000, 645 (0x0e00 << 16) | (0x88b0 >> 2), 646 0x00000000, 647 (0x0e00 << 16) | (0x88b4 >> 2), 648 0x00000000, 649 (0x0e00 << 16) | (0x88b8 >> 2), 650 0x00000000, 651 (0x0e00 << 16) | (0x88bc >> 2), 652 0x00000000, 653 (0x0400 << 16) | (0x89c0 >> 2), 654 0x00000000, 655 (0x0e00 << 16) | (0x88c4 >> 2), 656 0x00000000, 657 (0x0e00 << 16) | (0x88c8 >> 2), 658 0x00000000, 659 (0x0e00 << 16) | (0x88d0 >> 2), 660 0x00000000, 661 (0x0e00 << 16) | (0x88d4 >> 2), 662 0x00000000, 663 (0x0e00 << 16) | (0x88d8 >> 2), 664 0x00000000, 665 (0x0e00 << 16) | (0x8980 >> 2), 666 0x00000000, 667 (0x0e00 << 16) | (0x30938 >> 2), 668 0x00000000, 669 (0x0e00 << 16) | (0x3093c >> 2), 670 0x00000000, 671 (0x0e00 << 16) | (0x30940 >> 2), 672 0x00000000, 673 (0x0e00 << 16) | (0x89a0 >> 2), 674 0x00000000, 675 (0x0e00 << 16) | (0x30900 >> 2), 676 0x00000000, 677 (0x0e00 << 16) | (0x30904 >> 2), 678 0x00000000, 679 (0x0e00 << 16) | (0x89b4 >> 2), 680 0x00000000, 681 (0x0e00 << 16) | (0x3c210 >> 2), 682 0x00000000, 683 (0x0e00 << 16) | (0x3c214 >> 2), 684 0x00000000, 685 (0x0e00 << 16) | (0x3c218 >> 2), 686 0x00000000, 687 (0x0e00 << 16) | (0x8904 >> 2), 688 0x00000000, 689 0x5, 690 (0x0e00 << 16) | (0x8c28 >> 2), 691 (0x0e00 << 16) | (0x8c2c >> 2), 692 (0x0e00 << 16) | (0x8c30 >> 2), 693 (0x0e00 << 16) | (0x8c34 >> 2), 694 (0x0e00 << 16) | (0x9600 >> 2), 695 }; 696 697 static const u32 kalindi_rlc_save_restore_register_list[] = 698 { 699 (0x0e00 << 16) | (0xc12c >> 2), 700 0x00000000, 701 (0x0e00 << 16) | (0xc140 >> 2), 702 0x00000000, 703 (0x0e00 << 16) | (0xc150 >> 2), 704 0x00000000, 705 (0x0e00 << 16) | (0xc15c >> 2), 706 0x00000000, 707 (0x0e00 << 16) | (0xc168 >> 2), 708 0x00000000, 709 (0x0e00 << 16) | (0xc170 >> 2), 710 0x00000000, 711 (0x0e00 << 16) | (0xc204 >> 2), 712 0x00000000, 713 (0x0e00 << 16) | (0xc2b4 >> 2), 714 0x00000000, 715 (0x0e00 << 16) | (0xc2b8 >> 2), 716 0x00000000, 717 (0x0e00 << 16) | (0xc2bc >> 2), 718 0x00000000, 719 (0x0e00 << 16) | (0xc2c0 >> 2), 720 0x00000000, 721 (0x0e00 << 16) | (0x8228 >> 2), 722 0x00000000, 723 (0x0e00 << 16) | (0x829c >> 2), 724 0x00000000, 725 (0x0e00 << 16) | (0x869c >> 2), 726 0x00000000, 727 (0x0600 << 16) | (0x98f4 >> 2), 728 0x00000000, 729 (0x0e00 << 16) | (0x98f8 >> 2), 730 0x00000000, 731 (0x0e00 << 16) | (0x9900 >> 2), 732 0x00000000, 733 (0x0e00 << 16) | (0xc260 >> 2), 734 0x00000000, 735 (0x0e00 << 16) | (0x90e8 >> 2), 736 0x00000000, 737 (0x0e00 << 16) | (0x3c000 >> 2), 738 0x00000000, 739 (0x0e00 << 16) | (0x3c00c >> 2), 740 0x00000000, 741 (0x0e00 << 16) | (0x8c1c >> 2), 742 0x00000000, 743 (0x0e00 << 16) | (0x9700 >> 2), 744 0x00000000, 745 (0x0e00 << 16) | (0xcd20 >> 2), 746 0x00000000, 747 (0x4e00 << 16) | (0xcd20 >> 2), 748 0x00000000, 749 (0x5e00 << 16) | (0xcd20 >> 2), 750 0x00000000, 751 (0x6e00 << 16) | (0xcd20 >> 2), 752 0x00000000, 753 (0x7e00 << 16) | (0xcd20 >> 2), 754 0x00000000, 755 (0x0e00 << 16) | (0x89bc >> 2), 756 0x00000000, 757 (0x0e00 << 16) | (0x8900 >> 2), 758 0x00000000, 759 0x3, 760 (0x0e00 << 16) | (0xc130 >> 2), 761 0x00000000, 762 (0x0e00 << 16) | (0xc134 >> 2), 763 0x00000000, 764 (0x0e00 << 16) | (0xc1fc >> 2), 765 0x00000000, 766 (0x0e00 << 16) | (0xc208 >> 2), 767 0x00000000, 768 (0x0e00 << 16) | (0xc264 >> 2), 769 0x00000000, 770 (0x0e00 << 16) | (0xc268 >> 2), 771 0x00000000, 772 (0x0e00 << 16) | (0xc26c >> 2), 773 0x00000000, 774 (0x0e00 << 16) | (0xc270 >> 2), 775 0x00000000, 776 (0x0e00 << 16) | (0xc274 >> 2), 777 0x00000000, 778 (0x0e00 << 16) | (0xc28c >> 2), 779 0x00000000, 780 (0x0e00 << 16) | (0xc290 >> 2), 781 0x00000000, 782 (0x0e00 << 16) | (0xc294 >> 2), 783 0x00000000, 784 (0x0e00 << 16) | (0xc298 >> 2), 785 0x00000000, 786 (0x0e00 << 16) | (0xc2a0 >> 2), 787 0x00000000, 788 (0x0e00 << 16) | (0xc2a4 >> 2), 789 0x00000000, 790 (0x0e00 << 16) | (0xc2a8 >> 2), 791 0x00000000, 792 (0x0e00 << 16) | (0xc2ac >> 2), 793 0x00000000, 794 (0x0e00 << 16) | (0x301d0 >> 2), 795 0x00000000, 796 (0x0e00 << 16) | (0x30238 >> 2), 797 0x00000000, 798 (0x0e00 << 16) | (0x30250 >> 2), 799 0x00000000, 800 (0x0e00 << 16) | (0x30254 >> 2), 801 0x00000000, 802 (0x0e00 << 16) | (0x30258 >> 2), 803 0x00000000, 804 (0x0e00 << 16) | (0x3025c >> 2), 805 0x00000000, 806 (0x4e00 << 16) | (0xc900 >> 2), 807 0x00000000, 808 (0x5e00 << 16) | (0xc900 >> 2), 809 0x00000000, 810 (0x6e00 << 16) | (0xc900 >> 2), 811 0x00000000, 812 (0x7e00 << 16) | (0xc900 >> 2), 813 0x00000000, 814 (0x4e00 << 16) | (0xc904 >> 2), 815 0x00000000, 816 (0x5e00 << 16) | (0xc904 >> 2), 817 0x00000000, 818 (0x6e00 << 16) | (0xc904 >> 2), 819 0x00000000, 820 (0x7e00 << 16) | (0xc904 >> 2), 821 0x00000000, 822 (0x4e00 << 16) | (0xc908 >> 2), 823 0x00000000, 824 (0x5e00 << 16) | (0xc908 >> 2), 825 0x00000000, 826 (0x6e00 << 16) | (0xc908 >> 2), 827 0x00000000, 828 (0x7e00 << 16) | (0xc908 >> 2), 829 0x00000000, 830 (0x4e00 << 16) | (0xc90c >> 2), 831 0x00000000, 832 (0x5e00 << 16) | (0xc90c >> 2), 833 0x00000000, 834 (0x6e00 << 16) | (0xc90c >> 2), 835 0x00000000, 836 (0x7e00 << 16) | (0xc90c >> 2), 837 0x00000000, 838 (0x4e00 << 16) | (0xc910 >> 2), 839 0x00000000, 840 (0x5e00 << 16) | (0xc910 >> 2), 841 0x00000000, 842 (0x6e00 << 16) | (0xc910 >> 2), 843 0x00000000, 844 (0x7e00 << 16) | (0xc910 >> 2), 845 0x00000000, 846 (0x0e00 << 16) | (0xc99c >> 2), 847 0x00000000, 848 (0x0e00 << 16) | (0x9834 >> 2), 849 0x00000000, 850 (0x0000 << 16) | (0x30f00 >> 2), 851 0x00000000, 852 (0x0000 << 16) | (0x30f04 >> 2), 853 0x00000000, 854 (0x0000 << 16) | (0x30f08 >> 2), 855 0x00000000, 856 (0x0000 << 16) | (0x30f0c >> 2), 857 0x00000000, 858 (0x0600 << 16) | (0x9b7c >> 2), 859 0x00000000, 860 (0x0e00 << 16) | (0x8a14 >> 2), 861 0x00000000, 862 (0x0e00 << 16) | (0x8a18 >> 2), 863 0x00000000, 864 (0x0600 << 16) | (0x30a00 >> 2), 865 0x00000000, 866 (0x0e00 << 16) | (0x8bf0 >> 2), 867 0x00000000, 868 (0x0e00 << 16) | (0x8bcc >> 2), 869 0x00000000, 870 (0x0e00 << 16) | (0x8b24 >> 2), 871 0x00000000, 872 (0x0e00 << 16) | (0x30a04 >> 2), 873 0x00000000, 874 (0x0600 << 16) | (0x30a10 >> 2), 875 0x00000000, 876 (0x0600 << 16) | (0x30a14 >> 2), 877 0x00000000, 878 (0x0600 << 16) | (0x30a18 >> 2), 879 0x00000000, 880 (0x0600 << 16) | (0x30a2c >> 2), 881 0x00000000, 882 (0x0e00 << 16) | (0xc700 >> 2), 883 0x00000000, 884 (0x0e00 << 16) | (0xc704 >> 2), 885 0x00000000, 886 (0x0e00 << 16) | (0xc708 >> 2), 887 0x00000000, 888 (0x0e00 << 16) | (0xc768 >> 2), 889 0x00000000, 890 (0x0400 << 16) | (0xc770 >> 2), 891 0x00000000, 892 (0x0400 << 16) | (0xc774 >> 2), 893 0x00000000, 894 (0x0400 << 16) | (0xc798 >> 2), 895 0x00000000, 896 (0x0400 << 16) | (0xc79c >> 2), 897 0x00000000, 898 (0x0e00 << 16) | (0x9100 >> 2), 899 0x00000000, 900 (0x0e00 << 16) | (0x3c010 >> 2), 901 0x00000000, 902 (0x0e00 << 16) | (0x8c00 >> 2), 903 0x00000000, 904 (0x0e00 << 16) | (0x8c04 >> 2), 905 0x00000000, 906 (0x0e00 << 16) | (0x8c20 >> 2), 907 0x00000000, 908 (0x0e00 << 16) | (0x8c38 >> 2), 909 0x00000000, 910 (0x0e00 << 16) | (0x8c3c >> 2), 911 0x00000000, 912 (0x0e00 << 16) | (0xae00 >> 2), 913 0x00000000, 914 (0x0e00 << 16) | (0x9604 >> 2), 915 0x00000000, 916 (0x0e00 << 16) | (0xac08 >> 2), 917 0x00000000, 918 (0x0e00 << 16) | (0xac0c >> 2), 919 0x00000000, 920 (0x0e00 << 16) | (0xac10 >> 2), 921 0x00000000, 922 (0x0e00 << 16) | (0xac14 >> 2), 923 0x00000000, 924 (0x0e00 << 16) | (0xac58 >> 2), 925 0x00000000, 926 (0x0e00 << 16) | (0xac68 >> 2), 927 0x00000000, 928 (0x0e00 << 16) | (0xac6c >> 2), 929 0x00000000, 930 (0x0e00 << 16) | (0xac70 >> 2), 931 0x00000000, 932 (0x0e00 << 16) | (0xac74 >> 2), 933 0x00000000, 934 (0x0e00 << 16) | (0xac78 >> 2), 935 0x00000000, 936 (0x0e00 << 16) | (0xac7c >> 2), 937 0x00000000, 938 (0x0e00 << 16) | (0xac80 >> 2), 939 0x00000000, 940 (0x0e00 << 16) | (0xac84 >> 2), 941 0x00000000, 942 (0x0e00 << 16) | (0xac88 >> 2), 943 0x00000000, 944 (0x0e00 << 16) | (0xac8c >> 2), 945 0x00000000, 946 (0x0e00 << 16) | (0x970c >> 2), 947 0x00000000, 948 (0x0e00 << 16) | (0x9714 >> 2), 949 0x00000000, 950 (0x0e00 << 16) | (0x9718 >> 2), 951 0x00000000, 952 (0x0e00 << 16) | (0x971c >> 2), 953 0x00000000, 954 (0x0e00 << 16) | (0x31068 >> 2), 955 0x00000000, 956 (0x4e00 << 16) | (0x31068 >> 2), 957 0x00000000, 958 (0x5e00 << 16) | (0x31068 >> 2), 959 0x00000000, 960 (0x6e00 << 16) | (0x31068 >> 2), 961 0x00000000, 962 (0x7e00 << 16) | (0x31068 >> 2), 963 0x00000000, 964 (0x0e00 << 16) | (0xcd10 >> 2), 965 0x00000000, 966 (0x0e00 << 16) | (0xcd14 >> 2), 967 0x00000000, 968 (0x0e00 << 16) | (0x88b0 >> 2), 969 0x00000000, 970 (0x0e00 << 16) | (0x88b4 >> 2), 971 0x00000000, 972 (0x0e00 << 16) | (0x88b8 >> 2), 973 0x00000000, 974 (0x0e00 << 16) | (0x88bc >> 2), 975 0x00000000, 976 (0x0400 << 16) | (0x89c0 >> 2), 977 0x00000000, 978 (0x0e00 << 16) | (0x88c4 >> 2), 979 0x00000000, 980 (0x0e00 << 16) | (0x88c8 >> 2), 981 0x00000000, 982 (0x0e00 << 16) | (0x88d0 >> 2), 983 0x00000000, 984 (0x0e00 << 16) | (0x88d4 >> 2), 985 0x00000000, 986 (0x0e00 << 16) | (0x88d8 >> 2), 987 0x00000000, 988 (0x0e00 << 16) | (0x8980 >> 2), 989 0x00000000, 990 (0x0e00 << 16) | (0x30938 >> 2), 991 0x00000000, 992 (0x0e00 << 16) | (0x3093c >> 2), 993 0x00000000, 994 (0x0e00 << 16) | (0x30940 >> 2), 995 0x00000000, 996 (0x0e00 << 16) | (0x89a0 >> 2), 997 0x00000000, 998 (0x0e00 << 16) | (0x30900 >> 2), 999 0x00000000, 1000 (0x0e00 << 16) | (0x30904 >> 2), 1001 0x00000000, 1002 (0x0e00 << 16) | (0x89b4 >> 2), 1003 0x00000000, 1004 (0x0e00 << 16) | (0x3e1fc >> 2), 1005 0x00000000, 1006 (0x0e00 << 16) | (0x3c210 >> 2), 1007 0x00000000, 1008 (0x0e00 << 16) | (0x3c214 >> 2), 1009 0x00000000, 1010 (0x0e00 << 16) | (0x3c218 >> 2), 1011 0x00000000, 1012 (0x0e00 << 16) | (0x8904 >> 2), 1013 0x00000000, 1014 0x5, 1015 (0x0e00 << 16) | (0x8c28 >> 2), 1016 (0x0e00 << 16) | (0x8c2c >> 2), 1017 (0x0e00 << 16) | (0x8c30 >> 2), 1018 (0x0e00 << 16) | (0x8c34 >> 2), 1019 (0x0e00 << 16) | (0x9600 >> 2), 1020 }; 1021 1022 static const u32 bonaire_golden_spm_registers[] = 1023 { 1024 0x30800, 0xe0ffffff, 0xe0000000 1025 }; 1026 1027 static const u32 bonaire_golden_common_registers[] = 1028 { 1029 0xc770, 0xffffffff, 0x00000800, 1030 0xc774, 0xffffffff, 0x00000800, 1031 0xc798, 0xffffffff, 0x00007fbf, 1032 0xc79c, 0xffffffff, 0x00007faf 1033 }; 1034 1035 static const u32 bonaire_golden_registers[] = 1036 { 1037 0x3354, 0x00000333, 0x00000333, 1038 0x3350, 0x000c0fc0, 0x00040200, 1039 0x9a10, 0x00010000, 0x00058208, 1040 0x3c000, 0xffff1fff, 0x00140000, 1041 0x3c200, 0xfdfc0fff, 0x00000100, 1042 0x3c234, 0x40000000, 0x40000200, 1043 0x9830, 0xffffffff, 0x00000000, 1044 0x9834, 0xf00fffff, 0x00000400, 1045 0x9838, 0x0002021c, 0x00020200, 1046 0xc78, 0x00000080, 0x00000000, 1047 0x5bb0, 0x000000f0, 0x00000070, 1048 0x5bc0, 0xf0311fff, 0x80300000, 1049 0x98f8, 0x73773777, 0x12010001, 1050 0x350c, 0x00810000, 0x408af000, 1051 0x7030, 0x31000111, 0x00000011, 1052 0x2f48, 0x73773777, 0x12010001, 1053 0x220c, 0x00007fb6, 0x0021a1b1, 1054 0x2210, 0x00007fb6, 0x002021b1, 1055 0x2180, 0x00007fb6, 0x00002191, 1056 0x2218, 0x00007fb6, 0x002121b1, 1057 0x221c, 0x00007fb6, 0x002021b1, 1058 0x21dc, 0x00007fb6, 0x00002191, 1059 0x21e0, 0x00007fb6, 0x00002191, 1060 0x3628, 0x0000003f, 0x0000000a, 1061 0x362c, 0x0000003f, 0x0000000a, 1062 0x2ae4, 0x00073ffe, 0x000022a2, 1063 0x240c, 0x000007ff, 0x00000000, 1064 0x8a14, 0xf000003f, 0x00000007, 1065 0x8bf0, 0x00002001, 0x00000001, 1066 0x8b24, 0xffffffff, 0x00ffffff, 1067 0x30a04, 0x0000ff0f, 0x00000000, 1068 0x28a4c, 0x07ffffff, 0x06000000, 1069 0x4d8, 0x00000fff, 0x00000100, 1070 0x3e78, 0x00000001, 0x00000002, 1071 0x9100, 0x03000000, 0x0362c688, 1072 0x8c00, 0x000000ff, 0x00000001, 1073 0xe40, 0x00001fff, 0x00001fff, 1074 0x9060, 0x0000007f, 0x00000020, 1075 0x9508, 0x00010000, 0x00010000, 1076 0xac14, 0x000003ff, 0x000000f3, 1077 0xac0c, 0xffffffff, 0x00001032 1078 }; 1079 1080 static const u32 bonaire_mgcg_cgcg_init[] = 1081 { 1082 0xc420, 0xffffffff, 0xfffffffc, 1083 0x30800, 0xffffffff, 0xe0000000, 1084 0x3c2a0, 0xffffffff, 0x00000100, 1085 0x3c208, 0xffffffff, 0x00000100, 1086 0x3c2c0, 0xffffffff, 0xc0000100, 1087 0x3c2c8, 0xffffffff, 0xc0000100, 1088 0x3c2c4, 0xffffffff, 0xc0000100, 1089 0x55e4, 0xffffffff, 0x00600100, 1090 0x3c280, 0xffffffff, 0x00000100, 1091 0x3c214, 0xffffffff, 0x06000100, 1092 0x3c220, 0xffffffff, 0x00000100, 1093 0x3c218, 0xffffffff, 0x06000100, 1094 0x3c204, 0xffffffff, 0x00000100, 1095 0x3c2e0, 0xffffffff, 0x00000100, 1096 0x3c224, 0xffffffff, 0x00000100, 1097 0x3c200, 0xffffffff, 0x00000100, 1098 0x3c230, 0xffffffff, 0x00000100, 1099 0x3c234, 0xffffffff, 0x00000100, 1100 0x3c250, 0xffffffff, 0x00000100, 1101 0x3c254, 0xffffffff, 0x00000100, 1102 0x3c258, 0xffffffff, 0x00000100, 1103 0x3c25c, 0xffffffff, 0x00000100, 1104 0x3c260, 0xffffffff, 0x00000100, 1105 0x3c27c, 0xffffffff, 0x00000100, 1106 0x3c278, 0xffffffff, 0x00000100, 1107 0x3c210, 0xffffffff, 0x06000100, 1108 0x3c290, 0xffffffff, 0x00000100, 1109 0x3c274, 0xffffffff, 0x00000100, 1110 0x3c2b4, 0xffffffff, 0x00000100, 1111 0x3c2b0, 0xffffffff, 0x00000100, 1112 0x3c270, 0xffffffff, 0x00000100, 1113 0x30800, 0xffffffff, 0xe0000000, 1114 0x3c020, 0xffffffff, 0x00010000, 1115 0x3c024, 0xffffffff, 0x00030002, 1116 0x3c028, 0xffffffff, 0x00040007, 1117 0x3c02c, 0xffffffff, 0x00060005, 1118 0x3c030, 0xffffffff, 0x00090008, 1119 0x3c034, 0xffffffff, 0x00010000, 1120 0x3c038, 0xffffffff, 0x00030002, 1121 0x3c03c, 0xffffffff, 0x00040007, 1122 0x3c040, 0xffffffff, 0x00060005, 1123 0x3c044, 0xffffffff, 0x00090008, 1124 0x3c048, 0xffffffff, 0x00010000, 1125 0x3c04c, 0xffffffff, 0x00030002, 1126 0x3c050, 0xffffffff, 0x00040007, 1127 0x3c054, 0xffffffff, 0x00060005, 1128 0x3c058, 0xffffffff, 0x00090008, 1129 0x3c05c, 0xffffffff, 0x00010000, 1130 0x3c060, 0xffffffff, 0x00030002, 1131 0x3c064, 0xffffffff, 0x00040007, 1132 0x3c068, 0xffffffff, 0x00060005, 1133 0x3c06c, 0xffffffff, 0x00090008, 1134 0x3c070, 0xffffffff, 0x00010000, 1135 0x3c074, 0xffffffff, 0x00030002, 1136 0x3c078, 0xffffffff, 0x00040007, 1137 0x3c07c, 0xffffffff, 0x00060005, 1138 0x3c080, 0xffffffff, 0x00090008, 1139 0x3c084, 0xffffffff, 0x00010000, 1140 0x3c088, 0xffffffff, 0x00030002, 1141 0x3c08c, 0xffffffff, 0x00040007, 1142 0x3c090, 0xffffffff, 0x00060005, 1143 0x3c094, 0xffffffff, 0x00090008, 1144 0x3c098, 0xffffffff, 0x00010000, 1145 0x3c09c, 0xffffffff, 0x00030002, 1146 0x3c0a0, 0xffffffff, 0x00040007, 1147 0x3c0a4, 0xffffffff, 0x00060005, 1148 0x3c0a8, 0xffffffff, 0x00090008, 1149 0x3c000, 0xffffffff, 0x96e00200, 1150 0x8708, 0xffffffff, 0x00900100, 1151 0xc424, 0xffffffff, 0x0020003f, 1152 0x38, 0xffffffff, 0x0140001c, 1153 0x3c, 0x000f0000, 0x000f0000, 1154 0x220, 0xffffffff, 0xC060000C, 1155 0x224, 0xc0000fff, 0x00000100, 1156 0xf90, 0xffffffff, 0x00000100, 1157 0xf98, 0x00000101, 0x00000000, 1158 0x20a8, 0xffffffff, 0x00000104, 1159 0x55e4, 0xff000fff, 0x00000100, 1160 0x30cc, 0xc0000fff, 0x00000104, 1161 0xc1e4, 0x00000001, 0x00000001, 1162 0xd00c, 0xff000ff0, 0x00000100, 1163 0xd80c, 0xff000ff0, 0x00000100 1164 }; 1165 1166 static const u32 spectre_golden_spm_registers[] = 1167 { 1168 0x30800, 0xe0ffffff, 0xe0000000 1169 }; 1170 1171 static const u32 spectre_golden_common_registers[] = 1172 { 1173 0xc770, 0xffffffff, 0x00000800, 1174 0xc774, 0xffffffff, 0x00000800, 1175 0xc798, 0xffffffff, 0x00007fbf, 1176 0xc79c, 0xffffffff, 0x00007faf 1177 }; 1178 1179 static const u32 spectre_golden_registers[] = 1180 { 1181 0x3c000, 0xffff1fff, 0x96940200, 1182 0x3c00c, 0xffff0001, 0xff000000, 1183 0x3c200, 0xfffc0fff, 0x00000100, 1184 0x6ed8, 0x00010101, 0x00010000, 1185 0x9834, 0xf00fffff, 0x00000400, 1186 0x9838, 0xfffffffc, 0x00020200, 1187 0x5bb0, 0x000000f0, 0x00000070, 1188 0x5bc0, 0xf0311fff, 0x80300000, 1189 0x98f8, 0x73773777, 0x12010001, 1190 0x9b7c, 0x00ff0000, 0x00fc0000, 1191 0x2f48, 0x73773777, 0x12010001, 1192 0x8a14, 0xf000003f, 0x00000007, 1193 0x8b24, 0xffffffff, 0x00ffffff, 1194 0x28350, 0x3f3f3fff, 0x00000082, 1195 0x28354, 0x0000003f, 0x00000000, 1196 0x3e78, 0x00000001, 0x00000002, 1197 0x913c, 0xffff03df, 0x00000004, 1198 0xc768, 0x00000008, 0x00000008, 1199 0x8c00, 0x000008ff, 0x00000800, 1200 0x9508, 0x00010000, 0x00010000, 1201 0xac0c, 0xffffffff, 0x54763210, 1202 0x214f8, 0x01ff01ff, 0x00000002, 1203 0x21498, 0x007ff800, 0x00200000, 1204 0x2015c, 0xffffffff, 0x00000f40, 1205 0x30934, 0xffffffff, 0x00000001 1206 }; 1207 1208 static const u32 spectre_mgcg_cgcg_init[] = 1209 { 1210 0xc420, 0xffffffff, 0xfffffffc, 1211 0x30800, 0xffffffff, 0xe0000000, 1212 0x3c2a0, 0xffffffff, 0x00000100, 1213 0x3c208, 0xffffffff, 0x00000100, 1214 0x3c2c0, 0xffffffff, 0x00000100, 1215 0x3c2c8, 0xffffffff, 0x00000100, 1216 0x3c2c4, 0xffffffff, 0x00000100, 1217 0x55e4, 0xffffffff, 0x00600100, 1218 0x3c280, 0xffffffff, 0x00000100, 1219 0x3c214, 0xffffffff, 0x06000100, 1220 0x3c220, 0xffffffff, 0x00000100, 1221 0x3c218, 0xffffffff, 0x06000100, 1222 0x3c204, 0xffffffff, 0x00000100, 1223 0x3c2e0, 0xffffffff, 0x00000100, 1224 0x3c224, 0xffffffff, 0x00000100, 1225 0x3c200, 0xffffffff, 0x00000100, 1226 0x3c230, 0xffffffff, 0x00000100, 1227 0x3c234, 0xffffffff, 0x00000100, 1228 0x3c250, 0xffffffff, 0x00000100, 1229 0x3c254, 0xffffffff, 0x00000100, 1230 0x3c258, 0xffffffff, 0x00000100, 1231 0x3c25c, 0xffffffff, 0x00000100, 1232 0x3c260, 0xffffffff, 0x00000100, 1233 0x3c27c, 0xffffffff, 0x00000100, 1234 0x3c278, 0xffffffff, 0x00000100, 1235 0x3c210, 0xffffffff, 0x06000100, 1236 0x3c290, 0xffffffff, 0x00000100, 1237 0x3c274, 0xffffffff, 0x00000100, 1238 0x3c2b4, 0xffffffff, 0x00000100, 1239 0x3c2b0, 0xffffffff, 0x00000100, 1240 0x3c270, 0xffffffff, 0x00000100, 1241 0x30800, 0xffffffff, 0xe0000000, 1242 0x3c020, 0xffffffff, 0x00010000, 1243 0x3c024, 0xffffffff, 0x00030002, 1244 0x3c028, 0xffffffff, 0x00040007, 1245 0x3c02c, 0xffffffff, 0x00060005, 1246 0x3c030, 0xffffffff, 0x00090008, 1247 0x3c034, 0xffffffff, 0x00010000, 1248 0x3c038, 0xffffffff, 0x00030002, 1249 0x3c03c, 0xffffffff, 0x00040007, 1250 0x3c040, 0xffffffff, 0x00060005, 1251 0x3c044, 0xffffffff, 0x00090008, 1252 0x3c048, 0xffffffff, 0x00010000, 1253 0x3c04c, 0xffffffff, 0x00030002, 1254 0x3c050, 0xffffffff, 0x00040007, 1255 0x3c054, 0xffffffff, 0x00060005, 1256 0x3c058, 0xffffffff, 0x00090008, 1257 0x3c05c, 0xffffffff, 0x00010000, 1258 0x3c060, 0xffffffff, 0x00030002, 1259 0x3c064, 0xffffffff, 0x00040007, 1260 0x3c068, 0xffffffff, 0x00060005, 1261 0x3c06c, 0xffffffff, 0x00090008, 1262 0x3c070, 0xffffffff, 0x00010000, 1263 0x3c074, 0xffffffff, 0x00030002, 1264 0x3c078, 0xffffffff, 0x00040007, 1265 0x3c07c, 0xffffffff, 0x00060005, 1266 0x3c080, 0xffffffff, 0x00090008, 1267 0x3c084, 0xffffffff, 0x00010000, 1268 0x3c088, 0xffffffff, 0x00030002, 1269 0x3c08c, 0xffffffff, 0x00040007, 1270 0x3c090, 0xffffffff, 0x00060005, 1271 0x3c094, 0xffffffff, 0x00090008, 1272 0x3c098, 0xffffffff, 0x00010000, 1273 0x3c09c, 0xffffffff, 0x00030002, 1274 0x3c0a0, 0xffffffff, 0x00040007, 1275 0x3c0a4, 0xffffffff, 0x00060005, 1276 0x3c0a8, 0xffffffff, 0x00090008, 1277 0x3c0ac, 0xffffffff, 0x00010000, 1278 0x3c0b0, 0xffffffff, 0x00030002, 1279 0x3c0b4, 0xffffffff, 0x00040007, 1280 0x3c0b8, 0xffffffff, 0x00060005, 1281 0x3c0bc, 0xffffffff, 0x00090008, 1282 0x3c000, 0xffffffff, 0x96e00200, 1283 0x8708, 0xffffffff, 0x00900100, 1284 0xc424, 0xffffffff, 0x0020003f, 1285 0x38, 0xffffffff, 0x0140001c, 1286 0x3c, 0x000f0000, 0x000f0000, 1287 0x220, 0xffffffff, 0xC060000C, 1288 0x224, 0xc0000fff, 0x00000100, 1289 0xf90, 0xffffffff, 0x00000100, 1290 0xf98, 0x00000101, 0x00000000, 1291 0x20a8, 0xffffffff, 0x00000104, 1292 0x55e4, 0xff000fff, 0x00000100, 1293 0x30cc, 0xc0000fff, 0x00000104, 1294 0xc1e4, 0x00000001, 0x00000001, 1295 0xd00c, 0xff000ff0, 0x00000100, 1296 0xd80c, 0xff000ff0, 0x00000100 1297 }; 1298 1299 static const u32 kalindi_golden_spm_registers[] = 1300 { 1301 0x30800, 0xe0ffffff, 0xe0000000 1302 }; 1303 1304 static const u32 kalindi_golden_common_registers[] = 1305 { 1306 0xc770, 0xffffffff, 0x00000800, 1307 0xc774, 0xffffffff, 0x00000800, 1308 0xc798, 0xffffffff, 0x00007fbf, 1309 0xc79c, 0xffffffff, 0x00007faf 1310 }; 1311 1312 static const u32 kalindi_golden_registers[] = 1313 { 1314 0x3c000, 0xffffdfff, 0x6e944040, 1315 0x55e4, 0xff607fff, 0xfc000100, 1316 0x3c220, 0xff000fff, 0x00000100, 1317 0x3c224, 0xff000fff, 0x00000100, 1318 0x3c200, 0xfffc0fff, 0x00000100, 1319 0x6ed8, 0x00010101, 0x00010000, 1320 0x9830, 0xffffffff, 0x00000000, 1321 0x9834, 0xf00fffff, 0x00000400, 1322 0x5bb0, 0x000000f0, 0x00000070, 1323 0x5bc0, 0xf0311fff, 0x80300000, 1324 0x98f8, 0x73773777, 0x12010001, 1325 0x98fc, 0xffffffff, 0x00000010, 1326 0x9b7c, 0x00ff0000, 0x00fc0000, 1327 0x8030, 0x00001f0f, 0x0000100a, 1328 0x2f48, 0x73773777, 0x12010001, 1329 0x2408, 0x000fffff, 0x000c007f, 1330 0x8a14, 0xf000003f, 0x00000007, 1331 0x8b24, 0x3fff3fff, 0x00ffcfff, 1332 0x30a04, 0x0000ff0f, 0x00000000, 1333 0x28a4c, 0x07ffffff, 0x06000000, 1334 0x4d8, 0x00000fff, 0x00000100, 1335 0x3e78, 0x00000001, 0x00000002, 1336 0xc768, 0x00000008, 0x00000008, 1337 0x8c00, 0x000000ff, 0x00000003, 1338 0x214f8, 0x01ff01ff, 0x00000002, 1339 0x21498, 0x007ff800, 0x00200000, 1340 0x2015c, 0xffffffff, 0x00000f40, 1341 0x88c4, 0x001f3ae3, 0x00000082, 1342 0x88d4, 0x0000001f, 0x00000010, 1343 0x30934, 0xffffffff, 0x00000000 1344 }; 1345 1346 static const u32 kalindi_mgcg_cgcg_init[] = 1347 { 1348 0xc420, 0xffffffff, 0xfffffffc, 1349 0x30800, 0xffffffff, 0xe0000000, 1350 0x3c2a0, 0xffffffff, 0x00000100, 1351 0x3c208, 0xffffffff, 0x00000100, 1352 0x3c2c0, 0xffffffff, 0x00000100, 1353 0x3c2c8, 0xffffffff, 0x00000100, 1354 0x3c2c4, 0xffffffff, 0x00000100, 1355 0x55e4, 0xffffffff, 0x00600100, 1356 0x3c280, 0xffffffff, 0x00000100, 1357 0x3c214, 0xffffffff, 0x06000100, 1358 0x3c220, 0xffffffff, 0x00000100, 1359 0x3c218, 0xffffffff, 0x06000100, 1360 0x3c204, 0xffffffff, 0x00000100, 1361 0x3c2e0, 0xffffffff, 0x00000100, 1362 0x3c224, 0xffffffff, 0x00000100, 1363 0x3c200, 0xffffffff, 0x00000100, 1364 0x3c230, 0xffffffff, 0x00000100, 1365 0x3c234, 0xffffffff, 0x00000100, 1366 0x3c250, 0xffffffff, 0x00000100, 1367 0x3c254, 0xffffffff, 0x00000100, 1368 0x3c258, 0xffffffff, 0x00000100, 1369 0x3c25c, 0xffffffff, 0x00000100, 1370 0x3c260, 0xffffffff, 0x00000100, 1371 0x3c27c, 0xffffffff, 0x00000100, 1372 0x3c278, 0xffffffff, 0x00000100, 1373 0x3c210, 0xffffffff, 0x06000100, 1374 0x3c290, 0xffffffff, 0x00000100, 1375 0x3c274, 0xffffffff, 0x00000100, 1376 0x3c2b4, 0xffffffff, 0x00000100, 1377 0x3c2b0, 0xffffffff, 0x00000100, 1378 0x3c270, 0xffffffff, 0x00000100, 1379 0x30800, 0xffffffff, 0xe0000000, 1380 0x3c020, 0xffffffff, 0x00010000, 1381 0x3c024, 0xffffffff, 0x00030002, 1382 0x3c028, 0xffffffff, 0x00040007, 1383 0x3c02c, 0xffffffff, 0x00060005, 1384 0x3c030, 0xffffffff, 0x00090008, 1385 0x3c034, 0xffffffff, 0x00010000, 1386 0x3c038, 0xffffffff, 0x00030002, 1387 0x3c03c, 0xffffffff, 0x00040007, 1388 0x3c040, 0xffffffff, 0x00060005, 1389 0x3c044, 0xffffffff, 0x00090008, 1390 0x3c000, 0xffffffff, 0x96e00200, 1391 0x8708, 0xffffffff, 0x00900100, 1392 0xc424, 0xffffffff, 0x0020003f, 1393 0x38, 0xffffffff, 0x0140001c, 1394 0x3c, 0x000f0000, 0x000f0000, 1395 0x220, 0xffffffff, 0xC060000C, 1396 0x224, 0xc0000fff, 0x00000100, 1397 0x20a8, 0xffffffff, 0x00000104, 1398 0x55e4, 0xff000fff, 0x00000100, 1399 0x30cc, 0xc0000fff, 0x00000104, 1400 0xc1e4, 0x00000001, 0x00000001, 1401 0xd00c, 0xff000ff0, 0x00000100, 1402 0xd80c, 0xff000ff0, 0x00000100 1403 }; 1404 1405 static const u32 hawaii_golden_spm_registers[] = 1406 { 1407 0x30800, 0xe0ffffff, 0xe0000000 1408 }; 1409 1410 static const u32 hawaii_golden_common_registers[] = 1411 { 1412 0x30800, 0xffffffff, 0xe0000000, 1413 0x28350, 0xffffffff, 0x3a00161a, 1414 0x28354, 0xffffffff, 0x0000002e, 1415 0x9a10, 0xffffffff, 0x00018208, 1416 0x98f8, 0xffffffff, 0x12011003 1417 }; 1418 1419 static const u32 hawaii_golden_registers[] = 1420 { 1421 0x3354, 0x00000333, 0x00000333, 1422 0x9a10, 0x00010000, 0x00058208, 1423 0x9830, 0xffffffff, 0x00000000, 1424 0x9834, 0xf00fffff, 0x00000400, 1425 0x9838, 0x0002021c, 0x00020200, 1426 0xc78, 0x00000080, 0x00000000, 1427 0x5bb0, 0x000000f0, 0x00000070, 1428 0x5bc0, 0xf0311fff, 0x80300000, 1429 0x350c, 0x00810000, 0x408af000, 1430 0x7030, 0x31000111, 0x00000011, 1431 0x2f48, 0x73773777, 0x12010001, 1432 0x2120, 0x0000007f, 0x0000001b, 1433 0x21dc, 0x00007fb6, 0x00002191, 1434 0x3628, 0x0000003f, 0x0000000a, 1435 0x362c, 0x0000003f, 0x0000000a, 1436 0x2ae4, 0x00073ffe, 0x000022a2, 1437 0x240c, 0x000007ff, 0x00000000, 1438 0x8bf0, 0x00002001, 0x00000001, 1439 0x8b24, 0xffffffff, 0x00ffffff, 1440 0x30a04, 0x0000ff0f, 0x00000000, 1441 0x28a4c, 0x07ffffff, 0x06000000, 1442 0x3e78, 0x00000001, 0x00000002, 1443 0xc768, 0x00000008, 0x00000008, 1444 0xc770, 0x00000f00, 0x00000800, 1445 0xc774, 0x00000f00, 0x00000800, 1446 0xc798, 0x00ffffff, 0x00ff7fbf, 1447 0xc79c, 0x00ffffff, 0x00ff7faf, 1448 0x8c00, 0x000000ff, 0x00000800, 1449 0xe40, 0x00001fff, 0x00001fff, 1450 0x9060, 0x0000007f, 0x00000020, 1451 0x9508, 0x00010000, 0x00010000, 1452 0xae00, 0x00100000, 0x000ff07c, 1453 0xac14, 0x000003ff, 0x0000000f, 1454 0xac10, 0xffffffff, 0x7564fdec, 1455 0xac0c, 0xffffffff, 0x3120b9a8, 1456 0xac08, 0x20000000, 0x0f9c0000 1457 }; 1458 1459 static const u32 hawaii_mgcg_cgcg_init[] = 1460 { 1461 0xc420, 0xffffffff, 0xfffffffd, 1462 0x30800, 0xffffffff, 0xe0000000, 1463 0x3c2a0, 0xffffffff, 0x00000100, 1464 0x3c208, 0xffffffff, 0x00000100, 1465 0x3c2c0, 0xffffffff, 0x00000100, 1466 0x3c2c8, 0xffffffff, 0x00000100, 1467 0x3c2c4, 0xffffffff, 0x00000100, 1468 0x55e4, 0xffffffff, 0x00200100, 1469 0x3c280, 0xffffffff, 0x00000100, 1470 0x3c214, 0xffffffff, 0x06000100, 1471 0x3c220, 0xffffffff, 0x00000100, 1472 0x3c218, 0xffffffff, 0x06000100, 1473 0x3c204, 0xffffffff, 0x00000100, 1474 0x3c2e0, 0xffffffff, 0x00000100, 1475 0x3c224, 0xffffffff, 0x00000100, 1476 0x3c200, 0xffffffff, 0x00000100, 1477 0x3c230, 0xffffffff, 0x00000100, 1478 0x3c234, 0xffffffff, 0x00000100, 1479 0x3c250, 0xffffffff, 0x00000100, 1480 0x3c254, 0xffffffff, 0x00000100, 1481 0x3c258, 0xffffffff, 0x00000100, 1482 0x3c25c, 0xffffffff, 0x00000100, 1483 0x3c260, 0xffffffff, 0x00000100, 1484 0x3c27c, 0xffffffff, 0x00000100, 1485 0x3c278, 0xffffffff, 0x00000100, 1486 0x3c210, 0xffffffff, 0x06000100, 1487 0x3c290, 0xffffffff, 0x00000100, 1488 0x3c274, 0xffffffff, 0x00000100, 1489 0x3c2b4, 0xffffffff, 0x00000100, 1490 0x3c2b0, 0xffffffff, 0x00000100, 1491 0x3c270, 0xffffffff, 0x00000100, 1492 0x30800, 0xffffffff, 0xe0000000, 1493 0x3c020, 0xffffffff, 0x00010000, 1494 0x3c024, 0xffffffff, 0x00030002, 1495 0x3c028, 0xffffffff, 0x00040007, 1496 0x3c02c, 0xffffffff, 0x00060005, 1497 0x3c030, 0xffffffff, 0x00090008, 1498 0x3c034, 0xffffffff, 0x00010000, 1499 0x3c038, 0xffffffff, 0x00030002, 1500 0x3c03c, 0xffffffff, 0x00040007, 1501 0x3c040, 0xffffffff, 0x00060005, 1502 0x3c044, 0xffffffff, 0x00090008, 1503 0x3c048, 0xffffffff, 0x00010000, 1504 0x3c04c, 0xffffffff, 0x00030002, 1505 0x3c050, 0xffffffff, 0x00040007, 1506 0x3c054, 0xffffffff, 0x00060005, 1507 0x3c058, 0xffffffff, 0x00090008, 1508 0x3c05c, 0xffffffff, 0x00010000, 1509 0x3c060, 0xffffffff, 0x00030002, 1510 0x3c064, 0xffffffff, 0x00040007, 1511 0x3c068, 0xffffffff, 0x00060005, 1512 0x3c06c, 0xffffffff, 0x00090008, 1513 0x3c070, 0xffffffff, 0x00010000, 1514 0x3c074, 0xffffffff, 0x00030002, 1515 0x3c078, 0xffffffff, 0x00040007, 1516 0x3c07c, 0xffffffff, 0x00060005, 1517 0x3c080, 0xffffffff, 0x00090008, 1518 0x3c084, 0xffffffff, 0x00010000, 1519 0x3c088, 0xffffffff, 0x00030002, 1520 0x3c08c, 0xffffffff, 0x00040007, 1521 0x3c090, 0xffffffff, 0x00060005, 1522 0x3c094, 0xffffffff, 0x00090008, 1523 0x3c098, 0xffffffff, 0x00010000, 1524 0x3c09c, 0xffffffff, 0x00030002, 1525 0x3c0a0, 0xffffffff, 0x00040007, 1526 0x3c0a4, 0xffffffff, 0x00060005, 1527 0x3c0a8, 0xffffffff, 0x00090008, 1528 0x3c0ac, 0xffffffff, 0x00010000, 1529 0x3c0b0, 0xffffffff, 0x00030002, 1530 0x3c0b4, 0xffffffff, 0x00040007, 1531 0x3c0b8, 0xffffffff, 0x00060005, 1532 0x3c0bc, 0xffffffff, 0x00090008, 1533 0x3c0c0, 0xffffffff, 0x00010000, 1534 0x3c0c4, 0xffffffff, 0x00030002, 1535 0x3c0c8, 0xffffffff, 0x00040007, 1536 0x3c0cc, 0xffffffff, 0x00060005, 1537 0x3c0d0, 0xffffffff, 0x00090008, 1538 0x3c0d4, 0xffffffff, 0x00010000, 1539 0x3c0d8, 0xffffffff, 0x00030002, 1540 0x3c0dc, 0xffffffff, 0x00040007, 1541 0x3c0e0, 0xffffffff, 0x00060005, 1542 0x3c0e4, 0xffffffff, 0x00090008, 1543 0x3c0e8, 0xffffffff, 0x00010000, 1544 0x3c0ec, 0xffffffff, 0x00030002, 1545 0x3c0f0, 0xffffffff, 0x00040007, 1546 0x3c0f4, 0xffffffff, 0x00060005, 1547 0x3c0f8, 0xffffffff, 0x00090008, 1548 0xc318, 0xffffffff, 0x00020200, 1549 0x3350, 0xffffffff, 0x00000200, 1550 0x15c0, 0xffffffff, 0x00000400, 1551 0x55e8, 0xffffffff, 0x00000000, 1552 0x2f50, 0xffffffff, 0x00000902, 1553 0x3c000, 0xffffffff, 0x96940200, 1554 0x8708, 0xffffffff, 0x00900100, 1555 0xc424, 0xffffffff, 0x0020003f, 1556 0x38, 0xffffffff, 0x0140001c, 1557 0x3c, 0x000f0000, 0x000f0000, 1558 0x220, 0xffffffff, 0xc060000c, 1559 0x224, 0xc0000fff, 0x00000100, 1560 0xf90, 0xffffffff, 0x00000100, 1561 0xf98, 0x00000101, 0x00000000, 1562 0x20a8, 0xffffffff, 0x00000104, 1563 0x55e4, 0xff000fff, 0x00000100, 1564 0x30cc, 0xc0000fff, 0x00000104, 1565 0xc1e4, 0x00000001, 0x00000001, 1566 0xd00c, 0xff000ff0, 0x00000100, 1567 0xd80c, 0xff000ff0, 0x00000100 1568 }; 1569 1570 static const u32 godavari_golden_registers[] = 1571 { 1572 0x55e4, 0xff607fff, 0xfc000100, 1573 0x6ed8, 0x00010101, 0x00010000, 1574 0x9830, 0xffffffff, 0x00000000, 1575 0x98302, 0xf00fffff, 0x00000400, 1576 0x6130, 0xffffffff, 0x00010000, 1577 0x5bb0, 0x000000f0, 0x00000070, 1578 0x5bc0, 0xf0311fff, 0x80300000, 1579 0x98f8, 0x73773777, 0x12010001, 1580 0x98fc, 0xffffffff, 0x00000010, 1581 0x8030, 0x00001f0f, 0x0000100a, 1582 0x2f48, 0x73773777, 0x12010001, 1583 0x2408, 0x000fffff, 0x000c007f, 1584 0x8a14, 0xf000003f, 0x00000007, 1585 0x8b24, 0xffffffff, 0x00ff0fff, 1586 0x30a04, 0x0000ff0f, 0x00000000, 1587 0x28a4c, 0x07ffffff, 0x06000000, 1588 0x4d8, 0x00000fff, 0x00000100, 1589 0xd014, 0x00010000, 0x00810001, 1590 0xd814, 0x00010000, 0x00810001, 1591 0x3e78, 0x00000001, 0x00000002, 1592 0xc768, 0x00000008, 0x00000008, 1593 0xc770, 0x00000f00, 0x00000800, 1594 0xc774, 0x00000f00, 0x00000800, 1595 0xc798, 0x00ffffff, 0x00ff7fbf, 1596 0xc79c, 0x00ffffff, 0x00ff7faf, 1597 0x8c00, 0x000000ff, 0x00000001, 1598 0x214f8, 0x01ff01ff, 0x00000002, 1599 0x21498, 0x007ff800, 0x00200000, 1600 0x2015c, 0xffffffff, 0x00000f40, 1601 0x88c4, 0x001f3ae3, 0x00000082, 1602 0x88d4, 0x0000001f, 0x00000010, 1603 0x30934, 0xffffffff, 0x00000000 1604 }; 1605 1606 1607 static void cik_init_golden_registers(struct radeon_device *rdev) 1608 { 1609 /* Some of the registers might be dependent on GRBM_GFX_INDEX */ 1610 mutex_lock(&rdev->grbm_idx_mutex); 1611 switch (rdev->family) { 1612 case CHIP_BONAIRE: 1613 radeon_program_register_sequence(rdev, 1614 bonaire_mgcg_cgcg_init, 1615 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init)); 1616 radeon_program_register_sequence(rdev, 1617 bonaire_golden_registers, 1618 (const u32)ARRAY_SIZE(bonaire_golden_registers)); 1619 radeon_program_register_sequence(rdev, 1620 bonaire_golden_common_registers, 1621 (const u32)ARRAY_SIZE(bonaire_golden_common_registers)); 1622 radeon_program_register_sequence(rdev, 1623 bonaire_golden_spm_registers, 1624 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers)); 1625 break; 1626 case CHIP_KABINI: 1627 radeon_program_register_sequence(rdev, 1628 kalindi_mgcg_cgcg_init, 1629 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 1630 radeon_program_register_sequence(rdev, 1631 kalindi_golden_registers, 1632 (const u32)ARRAY_SIZE(kalindi_golden_registers)); 1633 radeon_program_register_sequence(rdev, 1634 kalindi_golden_common_registers, 1635 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 1636 radeon_program_register_sequence(rdev, 1637 kalindi_golden_spm_registers, 1638 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 1639 break; 1640 case CHIP_MULLINS: 1641 radeon_program_register_sequence(rdev, 1642 kalindi_mgcg_cgcg_init, 1643 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 1644 radeon_program_register_sequence(rdev, 1645 godavari_golden_registers, 1646 (const u32)ARRAY_SIZE(godavari_golden_registers)); 1647 radeon_program_register_sequence(rdev, 1648 kalindi_golden_common_registers, 1649 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 1650 radeon_program_register_sequence(rdev, 1651 kalindi_golden_spm_registers, 1652 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 1653 break; 1654 case CHIP_KAVERI: 1655 radeon_program_register_sequence(rdev, 1656 spectre_mgcg_cgcg_init, 1657 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init)); 1658 radeon_program_register_sequence(rdev, 1659 spectre_golden_registers, 1660 (const u32)ARRAY_SIZE(spectre_golden_registers)); 1661 radeon_program_register_sequence(rdev, 1662 spectre_golden_common_registers, 1663 (const u32)ARRAY_SIZE(spectre_golden_common_registers)); 1664 radeon_program_register_sequence(rdev, 1665 spectre_golden_spm_registers, 1666 (const u32)ARRAY_SIZE(spectre_golden_spm_registers)); 1667 break; 1668 case CHIP_HAWAII: 1669 radeon_program_register_sequence(rdev, 1670 hawaii_mgcg_cgcg_init, 1671 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init)); 1672 radeon_program_register_sequence(rdev, 1673 hawaii_golden_registers, 1674 (const u32)ARRAY_SIZE(hawaii_golden_registers)); 1675 radeon_program_register_sequence(rdev, 1676 hawaii_golden_common_registers, 1677 (const u32)ARRAY_SIZE(hawaii_golden_common_registers)); 1678 radeon_program_register_sequence(rdev, 1679 hawaii_golden_spm_registers, 1680 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers)); 1681 break; 1682 default: 1683 break; 1684 } 1685 mutex_unlock(&rdev->grbm_idx_mutex); 1686 } 1687 1688 /** 1689 * cik_get_xclk - get the xclk 1690 * 1691 * @rdev: radeon_device pointer 1692 * 1693 * Returns the reference clock used by the gfx engine 1694 * (CIK). 1695 */ 1696 u32 cik_get_xclk(struct radeon_device *rdev) 1697 { 1698 u32 reference_clock = rdev->clock.spll.reference_freq; 1699 1700 if (rdev->flags & RADEON_IS_IGP) { 1701 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK) 1702 return reference_clock / 2; 1703 } else { 1704 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE) 1705 return reference_clock / 4; 1706 } 1707 return reference_clock; 1708 } 1709 1710 /** 1711 * cik_mm_rdoorbell - read a doorbell dword 1712 * 1713 * @rdev: radeon_device pointer 1714 * @index: doorbell index 1715 * 1716 * Returns the value in the doorbell aperture at the 1717 * requested doorbell index (CIK). 1718 */ 1719 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index) 1720 { 1721 if (index < rdev->doorbell.num_doorbells) { 1722 return readl(rdev->doorbell.ptr + index); 1723 } else { 1724 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 1725 return 0; 1726 } 1727 } 1728 1729 /** 1730 * cik_mm_wdoorbell - write a doorbell dword 1731 * 1732 * @rdev: radeon_device pointer 1733 * @index: doorbell index 1734 * @v: value to write 1735 * 1736 * Writes @v to the doorbell aperture at the 1737 * requested doorbell index (CIK). 1738 */ 1739 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v) 1740 { 1741 if (index < rdev->doorbell.num_doorbells) { 1742 writel(v, rdev->doorbell.ptr + index); 1743 } else { 1744 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 1745 } 1746 } 1747 1748 #define BONAIRE_IO_MC_REGS_SIZE 36 1749 1750 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] = 1751 { 1752 {0x00000070, 0x04400000}, 1753 {0x00000071, 0x80c01803}, 1754 {0x00000072, 0x00004004}, 1755 {0x00000073, 0x00000100}, 1756 {0x00000074, 0x00ff0000}, 1757 {0x00000075, 0x34000000}, 1758 {0x00000076, 0x08000014}, 1759 {0x00000077, 0x00cc08ec}, 1760 {0x00000078, 0x00000400}, 1761 {0x00000079, 0x00000000}, 1762 {0x0000007a, 0x04090000}, 1763 {0x0000007c, 0x00000000}, 1764 {0x0000007e, 0x4408a8e8}, 1765 {0x0000007f, 0x00000304}, 1766 {0x00000080, 0x00000000}, 1767 {0x00000082, 0x00000001}, 1768 {0x00000083, 0x00000002}, 1769 {0x00000084, 0xf3e4f400}, 1770 {0x00000085, 0x052024e3}, 1771 {0x00000087, 0x00000000}, 1772 {0x00000088, 0x01000000}, 1773 {0x0000008a, 0x1c0a0000}, 1774 {0x0000008b, 0xff010000}, 1775 {0x0000008d, 0xffffefff}, 1776 {0x0000008e, 0xfff3efff}, 1777 {0x0000008f, 0xfff3efbf}, 1778 {0x00000092, 0xf7ffffff}, 1779 {0x00000093, 0xffffff7f}, 1780 {0x00000095, 0x00101101}, 1781 {0x00000096, 0x00000fff}, 1782 {0x00000097, 0x00116fff}, 1783 {0x00000098, 0x60010000}, 1784 {0x00000099, 0x10010000}, 1785 {0x0000009a, 0x00006000}, 1786 {0x0000009b, 0x00001000}, 1787 {0x0000009f, 0x00b48000} 1788 }; 1789 1790 #define HAWAII_IO_MC_REGS_SIZE 22 1791 1792 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] = 1793 { 1794 {0x0000007d, 0x40000000}, 1795 {0x0000007e, 0x40180304}, 1796 {0x0000007f, 0x0000ff00}, 1797 {0x00000081, 0x00000000}, 1798 {0x00000083, 0x00000800}, 1799 {0x00000086, 0x00000000}, 1800 {0x00000087, 0x00000100}, 1801 {0x00000088, 0x00020100}, 1802 {0x00000089, 0x00000000}, 1803 {0x0000008b, 0x00040000}, 1804 {0x0000008c, 0x00000100}, 1805 {0x0000008e, 0xff010000}, 1806 {0x00000090, 0xffffefff}, 1807 {0x00000091, 0xfff3efff}, 1808 {0x00000092, 0xfff3efbf}, 1809 {0x00000093, 0xf7ffffff}, 1810 {0x00000094, 0xffffff7f}, 1811 {0x00000095, 0x00000fff}, 1812 {0x00000096, 0x00116fff}, 1813 {0x00000097, 0x60010000}, 1814 {0x00000098, 0x10010000}, 1815 {0x0000009f, 0x00c79000} 1816 }; 1817 1818 1819 /** 1820 * cik_srbm_select - select specific register instances 1821 * 1822 * @rdev: radeon_device pointer 1823 * @me: selected ME (micro engine) 1824 * @pipe: pipe 1825 * @queue: queue 1826 * @vmid: VMID 1827 * 1828 * Switches the currently active registers instances. Some 1829 * registers are instanced per VMID, others are instanced per 1830 * me/pipe/queue combination. 1831 */ 1832 static void cik_srbm_select(struct radeon_device *rdev, 1833 u32 me, u32 pipe, u32 queue, u32 vmid) 1834 { 1835 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) | 1836 MEID(me & 0x3) | 1837 VMID(vmid & 0xf) | 1838 QUEUEID(queue & 0x7)); 1839 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl); 1840 } 1841 1842 /* ucode loading */ 1843 /** 1844 * ci_mc_load_microcode - load MC ucode into the hw 1845 * 1846 * @rdev: radeon_device pointer 1847 * 1848 * Load the GDDR MC ucode into the hw (CIK). 1849 * Returns 0 on success, error on failure. 1850 */ 1851 int ci_mc_load_microcode(struct radeon_device *rdev) 1852 { 1853 const __be32 *fw_data = NULL; 1854 const __le32 *new_fw_data = NULL; 1855 u32 running, blackout = 0, tmp; 1856 u32 *io_mc_regs = NULL; 1857 const __le32 *new_io_mc_regs = NULL; 1858 int i, regs_size, ucode_size; 1859 1860 if (!rdev->mc_fw) 1861 return -EINVAL; 1862 1863 if (rdev->new_fw) { 1864 const struct mc_firmware_header_v1_0 *hdr = 1865 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data; 1866 1867 radeon_ucode_print_mc_hdr(&hdr->header); 1868 1869 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2); 1870 new_io_mc_regs = (const __le32 *) 1871 ((const char *)rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); 1872 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 1873 new_fw_data = (const __le32 *) 1874 ((const char *)rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1875 } else { 1876 ucode_size = rdev->mc_fw->datasize / 4; 1877 1878 switch (rdev->family) { 1879 case CHIP_BONAIRE: 1880 io_mc_regs = (u32 *)&bonaire_io_mc_regs; 1881 regs_size = BONAIRE_IO_MC_REGS_SIZE; 1882 break; 1883 case CHIP_HAWAII: 1884 io_mc_regs = (u32 *)&hawaii_io_mc_regs; 1885 regs_size = HAWAII_IO_MC_REGS_SIZE; 1886 break; 1887 default: 1888 return -EINVAL; 1889 } 1890 fw_data = (const __be32 *)rdev->mc_fw->data; 1891 } 1892 1893 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; 1894 1895 if (running == 0) { 1896 if (running) { 1897 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 1898 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 1899 } 1900 1901 /* reset the engine and set to writable */ 1902 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1903 WREG32(MC_SEQ_SUP_CNTL, 0x00000010); 1904 1905 /* load mc io regs */ 1906 for (i = 0; i < regs_size; i++) { 1907 if (rdev->new_fw) { 1908 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++)); 1909 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++)); 1910 } else { 1911 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); 1912 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); 1913 } 1914 } 1915 1916 tmp = RREG32(MC_SEQ_MISC0); 1917 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) { 1918 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5); 1919 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023); 1920 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9); 1921 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0); 1922 } 1923 1924 /* load the MC ucode */ 1925 for (i = 0; i < ucode_size; i++) { 1926 if (rdev->new_fw) 1927 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++)); 1928 else 1929 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); 1930 } 1931 1932 /* put the engine back into the active state */ 1933 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1934 WREG32(MC_SEQ_SUP_CNTL, 0x00000004); 1935 WREG32(MC_SEQ_SUP_CNTL, 0x00000001); 1936 1937 /* wait for training to complete */ 1938 for (i = 0; i < rdev->usec_timeout; i++) { 1939 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0) 1940 break; 1941 udelay(1); 1942 } 1943 for (i = 0; i < rdev->usec_timeout; i++) { 1944 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1) 1945 break; 1946 udelay(1); 1947 } 1948 1949 if (running) 1950 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout); 1951 } 1952 1953 return 0; 1954 } 1955 1956 /** 1957 * cik_init_microcode - load ucode images from disk 1958 * 1959 * @rdev: radeon_device pointer 1960 * 1961 * Use the firmware interface to load the ucode images into 1962 * the driver (not loaded into hw). 1963 * Returns 0 on success, error on failure. 1964 */ 1965 static int cik_init_microcode(struct radeon_device *rdev) 1966 { 1967 const char *chip_name; 1968 const char *new_chip_name; 1969 size_t pfp_req_size, me_req_size, ce_req_size, 1970 mec_req_size, rlc_req_size, mc_req_size = 0, 1971 sdma_req_size, smc_req_size = 0, mc2_req_size = 0; 1972 char fw_name[30]; 1973 int new_fw = 0; 1974 int err; 1975 int num_fw; 1976 1977 DRM_DEBUG("\n"); 1978 1979 switch (rdev->family) { 1980 case CHIP_BONAIRE: 1981 chip_name = "BONAIRE"; 1982 new_chip_name = "bonaire"; 1983 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1984 me_req_size = CIK_ME_UCODE_SIZE * 4; 1985 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1986 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1987 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; 1988 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4; 1989 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4; 1990 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1991 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4); 1992 num_fw = 8; 1993 break; 1994 case CHIP_HAWAII: 1995 chip_name = "HAWAII"; 1996 new_chip_name = "hawaii"; 1997 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1998 me_req_size = CIK_ME_UCODE_SIZE * 4; 1999 ce_req_size = CIK_CE_UCODE_SIZE * 4; 2000 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 2001 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; 2002 mc_req_size = HAWAII_MC_UCODE_SIZE * 4; 2003 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4; 2004 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 2005 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4); 2006 num_fw = 8; 2007 break; 2008 case CHIP_KAVERI: 2009 chip_name = "KAVERI"; 2010 new_chip_name = "kaveri"; 2011 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 2012 me_req_size = CIK_ME_UCODE_SIZE * 4; 2013 ce_req_size = CIK_CE_UCODE_SIZE * 4; 2014 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 2015 rlc_req_size = KV_RLC_UCODE_SIZE * 4; 2016 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 2017 num_fw = 7; 2018 break; 2019 case CHIP_KABINI: 2020 chip_name = "KABINI"; 2021 new_chip_name = "kabini"; 2022 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 2023 me_req_size = CIK_ME_UCODE_SIZE * 4; 2024 ce_req_size = CIK_CE_UCODE_SIZE * 4; 2025 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 2026 rlc_req_size = KB_RLC_UCODE_SIZE * 4; 2027 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 2028 num_fw = 6; 2029 break; 2030 case CHIP_MULLINS: 2031 chip_name = "MULLINS"; 2032 new_chip_name = "mullins"; 2033 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 2034 me_req_size = CIK_ME_UCODE_SIZE * 4; 2035 ce_req_size = CIK_CE_UCODE_SIZE * 4; 2036 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 2037 rlc_req_size = ML_RLC_UCODE_SIZE * 4; 2038 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 2039 num_fw = 6; 2040 break; 2041 default: BUG(); 2042 } 2043 2044 DRM_INFO("Loading %s Microcode\n", new_chip_name); 2045 2046 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", new_chip_name); 2047 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 2048 if (err) { 2049 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name); 2050 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 2051 if (err) 2052 goto out; 2053 if (rdev->pfp_fw->datasize != pfp_req_size) { 2054 printk(KERN_ERR 2055 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2056 rdev->pfp_fw->datasize, fw_name); 2057 err = -EINVAL; 2058 goto out; 2059 } 2060 } else { 2061 err = radeon_ucode_validate(rdev->pfp_fw); 2062 if (err) { 2063 printk(KERN_ERR 2064 "cik_fw: validation failed for firmware \"%s\"\n", 2065 fw_name); 2066 goto out; 2067 } else { 2068 new_fw++; 2069 } 2070 } 2071 2072 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", new_chip_name); 2073 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 2074 if (err) { 2075 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name); 2076 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 2077 if (err) 2078 goto out; 2079 if (rdev->me_fw->datasize != me_req_size) { 2080 printk(KERN_ERR 2081 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2082 rdev->me_fw->datasize, fw_name); 2083 err = -EINVAL; 2084 } 2085 } else { 2086 err = radeon_ucode_validate(rdev->me_fw); 2087 if (err) { 2088 printk(KERN_ERR 2089 "cik_fw: validation failed for firmware \"%s\"\n", 2090 fw_name); 2091 goto out; 2092 } else { 2093 new_fw++; 2094 } 2095 } 2096 2097 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", new_chip_name); 2098 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 2099 if (err) { 2100 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", chip_name); 2101 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 2102 if (err) 2103 goto out; 2104 if (rdev->ce_fw->datasize != ce_req_size) { 2105 printk(KERN_ERR 2106 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2107 rdev->ce_fw->datasize, fw_name); 2108 err = -EINVAL; 2109 } 2110 } else { 2111 err = radeon_ucode_validate(rdev->ce_fw); 2112 if (err) { 2113 printk(KERN_ERR 2114 "cik_fw: validation failed for firmware \"%s\"\n", 2115 fw_name); 2116 goto out; 2117 } else { 2118 new_fw++; 2119 } 2120 } 2121 2122 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mec", new_chip_name); 2123 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); 2124 if (err) { 2125 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mec", chip_name); 2126 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); 2127 if (err) 2128 goto out; 2129 if (rdev->mec_fw->datasize != mec_req_size) { 2130 printk(KERN_ERR 2131 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2132 rdev->mec_fw->datasize, fw_name); 2133 err = -EINVAL; 2134 } 2135 } else { 2136 err = radeon_ucode_validate(rdev->mec_fw); 2137 if (err) { 2138 printk(KERN_ERR 2139 "cik_fw: validation failed for firmware \"%s\"\n", 2140 fw_name); 2141 goto out; 2142 } else { 2143 new_fw++; 2144 } 2145 } 2146 2147 if (rdev->family == CHIP_KAVERI) { 2148 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mec2", new_chip_name); 2149 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev); 2150 if (err) { 2151 goto out; 2152 } else { 2153 err = radeon_ucode_validate(rdev->mec2_fw); 2154 if (err) { 2155 goto out; 2156 } else { 2157 new_fw++; 2158 } 2159 } 2160 } 2161 2162 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", new_chip_name); 2163 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 2164 if (err) { 2165 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", chip_name); 2166 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 2167 if (err) 2168 goto out; 2169 if (rdev->rlc_fw->datasize != rlc_req_size) { 2170 printk(KERN_ERR 2171 "cik_rlc: Bogus length %zu in firmware \"%s\"\n", 2172 rdev->rlc_fw->datasize, fw_name); 2173 err = -EINVAL; 2174 } 2175 } else { 2176 err = radeon_ucode_validate(rdev->rlc_fw); 2177 if (err) { 2178 printk(KERN_ERR 2179 "cik_fw: validation failed for firmware \"%s\"\n", 2180 fw_name); 2181 goto out; 2182 } else { 2183 new_fw++; 2184 } 2185 } 2186 2187 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_sdma", new_chip_name); 2188 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); 2189 if (err) { 2190 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_sdma", chip_name); 2191 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); 2192 if (err) 2193 goto out; 2194 if (rdev->sdma_fw->datasize != sdma_req_size) { 2195 printk(KERN_ERR 2196 "cik_sdma: Bogus length %zu in firmware \"%s\"\n", 2197 rdev->sdma_fw->datasize, fw_name); 2198 err = -EINVAL; 2199 } 2200 } else { 2201 err = radeon_ucode_validate(rdev->sdma_fw); 2202 if (err) { 2203 printk(KERN_ERR 2204 "cik_fw: validation failed for firmware \"%s\"\n", 2205 fw_name); 2206 goto out; 2207 } else { 2208 new_fw++; 2209 } 2210 } 2211 2212 /* No SMC, MC ucode on APUs */ 2213 if (!(rdev->flags & RADEON_IS_IGP)) { 2214 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", new_chip_name); 2215 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2216 if (err) { 2217 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc2", chip_name); 2218 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2219 if (err) { 2220 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", chip_name); 2221 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2222 if (err) 2223 goto out; 2224 } 2225 if ((rdev->mc_fw->datasize != mc_req_size) && 2226 (rdev->mc_fw->datasize != mc2_req_size)){ 2227 printk(KERN_ERR 2228 "cik_mc: Bogus length %zu in firmware \"%s\"\n", 2229 rdev->mc_fw->datasize, fw_name); 2230 err = -EINVAL; 2231 } 2232 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->datasize); 2233 } else { 2234 err = radeon_ucode_validate(rdev->mc_fw); 2235 if (err) { 2236 printk(KERN_ERR 2237 "cik_fw: validation failed for firmware \"%s\"\n", 2238 fw_name); 2239 goto out; 2240 } else { 2241 new_fw++; 2242 } 2243 } 2244 2245 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", new_chip_name); 2246 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 2247 if (err) { 2248 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", chip_name); 2249 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 2250 if (err) { 2251 printk(KERN_ERR 2252 "smc: error loading firmware \"%s\"\n", 2253 fw_name); 2254 release_firmware(rdev->smc_fw); 2255 rdev->smc_fw = NULL; 2256 err = 0; 2257 } else if (rdev->smc_fw->datasize != smc_req_size) { 2258 printk(KERN_ERR 2259 "cik_smc: Bogus length %zu in firmware \"%s\"\n", 2260 rdev->smc_fw->datasize, fw_name); 2261 err = -EINVAL; 2262 } 2263 } else { 2264 err = radeon_ucode_validate(rdev->smc_fw); 2265 if (err) { 2266 printk(KERN_ERR 2267 "cik_fw: validation failed for firmware \"%s\"\n", 2268 fw_name); 2269 goto out; 2270 } else { 2271 new_fw++; 2272 } 2273 } 2274 } 2275 2276 if (new_fw == 0) { 2277 rdev->new_fw = false; 2278 } else if (new_fw < num_fw) { 2279 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n"); 2280 err = -EINVAL; 2281 } else { 2282 rdev->new_fw = true; 2283 } 2284 2285 out: 2286 if (err) { 2287 if (err != -EINVAL) 2288 printk(KERN_ERR 2289 "cik_cp: Failed to load firmware \"%s\"\n", 2290 fw_name); 2291 release_firmware(rdev->pfp_fw); 2292 rdev->pfp_fw = NULL; 2293 release_firmware(rdev->me_fw); 2294 rdev->me_fw = NULL; 2295 release_firmware(rdev->ce_fw); 2296 rdev->ce_fw = NULL; 2297 release_firmware(rdev->mec_fw); 2298 rdev->mec_fw = NULL; 2299 release_firmware(rdev->mec2_fw); 2300 rdev->mec2_fw = NULL; 2301 release_firmware(rdev->rlc_fw); 2302 rdev->rlc_fw = NULL; 2303 release_firmware(rdev->sdma_fw); 2304 rdev->sdma_fw = NULL; 2305 release_firmware(rdev->mc_fw); 2306 rdev->mc_fw = NULL; 2307 release_firmware(rdev->smc_fw); 2308 rdev->smc_fw = NULL; 2309 } 2310 return err; 2311 } 2312 2313 /** 2314 * cik_fini_microcode - drop the firmwares image references 2315 * 2316 * @rdev: radeon_device pointer 2317 * 2318 * Drop the pfp, me, mec, mec2, rlc, sdma, mc, smc and ce firmware image references. 2319 * Called at driver shutdown. 2320 */ 2321 static void cik_fini_microcode(struct radeon_device *rdev) 2322 { 2323 release_firmware(rdev->pfp_fw); 2324 rdev->pfp_fw = NULL; 2325 release_firmware(rdev->me_fw); 2326 rdev->me_fw = NULL; 2327 release_firmware(rdev->ce_fw); 2328 rdev->ce_fw = NULL; 2329 release_firmware(rdev->mec_fw); 2330 rdev->mec_fw = NULL; 2331 release_firmware(rdev->mec2_fw); 2332 rdev->mec2_fw = NULL; 2333 release_firmware(rdev->rlc_fw); 2334 rdev->rlc_fw = NULL; 2335 release_firmware(rdev->sdma_fw); 2336 rdev->sdma_fw = NULL; 2337 release_firmware(rdev->mc_fw); 2338 rdev->mc_fw = NULL; 2339 release_firmware(rdev->smc_fw); 2340 rdev->smc_fw = NULL; 2341 } 2342 2343 /* 2344 * Core functions 2345 */ 2346 /** 2347 * cik_tiling_mode_table_init - init the hw tiling table 2348 * 2349 * @rdev: radeon_device pointer 2350 * 2351 * Starting with SI, the tiling setup is done globally in a 2352 * set of 32 tiling modes. Rather than selecting each set of 2353 * parameters per surface as on older asics, we just select 2354 * which index in the tiling table we want to use, and the 2355 * surface uses those parameters (CIK). 2356 */ 2357 static void cik_tiling_mode_table_init(struct radeon_device *rdev) 2358 { 2359 u32 *tile = rdev->config.cik.tile_mode_array; 2360 u32 *macrotile = rdev->config.cik.macrotile_mode_array; 2361 const u32 num_tile_mode_states = 2362 ARRAY_SIZE(rdev->config.cik.tile_mode_array); 2363 const u32 num_secondary_tile_mode_states = 2364 ARRAY_SIZE(rdev->config.cik.macrotile_mode_array); 2365 u32 reg_offset, split_equal_to_row_size; 2366 u32 num_pipe_configs; 2367 u32 num_rbs = rdev->config.cik.max_backends_per_se * 2368 rdev->config.cik.max_shader_engines; 2369 2370 switch (rdev->config.cik.mem_row_size_in_kb) { 2371 case 1: 2372 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB; 2373 break; 2374 case 2: 2375 default: 2376 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB; 2377 break; 2378 case 4: 2379 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB; 2380 break; 2381 } 2382 2383 num_pipe_configs = rdev->config.cik.max_tile_pipes; 2384 if (num_pipe_configs > 8) 2385 num_pipe_configs = 16; 2386 2387 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2388 tile[reg_offset] = 0; 2389 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2390 macrotile[reg_offset] = 0; 2391 2392 switch(num_pipe_configs) { 2393 case 16: 2394 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2395 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2396 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2397 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2398 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2399 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2400 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2401 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2402 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2403 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2404 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2405 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2406 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2407 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2408 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2409 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2410 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2411 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2412 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2413 TILE_SPLIT(split_equal_to_row_size)); 2414 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2415 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2416 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2417 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2418 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2419 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2420 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2421 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2422 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2423 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2424 TILE_SPLIT(split_equal_to_row_size)); 2425 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2426 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16)); 2427 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2428 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2429 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2430 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2431 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2432 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2433 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2434 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2435 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2436 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2437 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2438 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2439 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2440 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2441 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2442 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2443 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2444 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2445 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2446 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2447 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2448 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2449 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2450 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2451 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2452 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2453 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2454 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2455 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2456 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2457 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2458 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2459 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2460 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2461 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2462 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2463 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2464 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2465 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2466 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2467 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2468 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2469 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2470 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2471 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2472 2473 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2474 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2475 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2476 NUM_BANKS(ADDR_SURF_16_BANK)); 2477 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2478 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2479 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2480 NUM_BANKS(ADDR_SURF_16_BANK)); 2481 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2484 NUM_BANKS(ADDR_SURF_16_BANK)); 2485 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2488 NUM_BANKS(ADDR_SURF_16_BANK)); 2489 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2490 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2491 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2492 NUM_BANKS(ADDR_SURF_8_BANK)); 2493 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2494 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2495 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2496 NUM_BANKS(ADDR_SURF_4_BANK)); 2497 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2498 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2499 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2500 NUM_BANKS(ADDR_SURF_2_BANK)); 2501 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2502 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2503 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2504 NUM_BANKS(ADDR_SURF_16_BANK)); 2505 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2506 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2507 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2508 NUM_BANKS(ADDR_SURF_16_BANK)); 2509 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2510 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2511 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2512 NUM_BANKS(ADDR_SURF_16_BANK)); 2513 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2514 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2515 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2516 NUM_BANKS(ADDR_SURF_8_BANK)); 2517 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2520 NUM_BANKS(ADDR_SURF_4_BANK)); 2521 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2522 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2523 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2524 NUM_BANKS(ADDR_SURF_2_BANK)); 2525 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2526 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2527 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2528 NUM_BANKS(ADDR_SURF_2_BANK)); 2529 2530 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2531 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]); 2532 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2533 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]); 2534 break; 2535 2536 case 8: 2537 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2538 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2539 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2540 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2541 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2542 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2543 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2544 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2545 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2546 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2547 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2548 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2549 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2550 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2551 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2552 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2553 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2554 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2555 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2556 TILE_SPLIT(split_equal_to_row_size)); 2557 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2558 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2559 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2560 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2561 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2562 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2563 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2564 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2565 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2566 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2567 TILE_SPLIT(split_equal_to_row_size)); 2568 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2569 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2570 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2571 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2572 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2573 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2574 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2575 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2576 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2577 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2578 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2579 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2580 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2581 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2582 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2583 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2584 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2585 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2586 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2587 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2588 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2589 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2590 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2591 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2592 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2593 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2594 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2595 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2596 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2597 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2598 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2599 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2600 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2601 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2602 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2603 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2604 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2605 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2606 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2607 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2608 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2609 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2610 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2611 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2612 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2613 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2614 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2615 2616 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2617 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2618 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2619 NUM_BANKS(ADDR_SURF_16_BANK)); 2620 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2621 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2622 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2623 NUM_BANKS(ADDR_SURF_16_BANK)); 2624 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2625 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2626 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2627 NUM_BANKS(ADDR_SURF_16_BANK)); 2628 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2629 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2630 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2631 NUM_BANKS(ADDR_SURF_16_BANK)); 2632 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2633 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2634 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2635 NUM_BANKS(ADDR_SURF_8_BANK)); 2636 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2637 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2638 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2639 NUM_BANKS(ADDR_SURF_4_BANK)); 2640 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2641 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2642 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2643 NUM_BANKS(ADDR_SURF_2_BANK)); 2644 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2645 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2646 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2647 NUM_BANKS(ADDR_SURF_16_BANK)); 2648 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2649 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2650 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2651 NUM_BANKS(ADDR_SURF_16_BANK)); 2652 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2653 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2654 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2655 NUM_BANKS(ADDR_SURF_16_BANK)); 2656 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2657 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2658 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2659 NUM_BANKS(ADDR_SURF_16_BANK)); 2660 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2661 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2662 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2663 NUM_BANKS(ADDR_SURF_8_BANK)); 2664 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2665 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2666 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2667 NUM_BANKS(ADDR_SURF_4_BANK)); 2668 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2669 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2670 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2671 NUM_BANKS(ADDR_SURF_2_BANK)); 2672 2673 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2674 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]); 2675 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2676 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]); 2677 break; 2678 2679 case 4: 2680 if (num_rbs == 4) { 2681 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2682 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2683 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2684 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2685 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2686 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2687 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2688 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2689 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2690 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2691 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2692 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2693 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2694 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2695 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2696 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2697 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2698 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2699 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2700 TILE_SPLIT(split_equal_to_row_size)); 2701 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2702 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2703 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2704 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2705 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2706 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2707 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2708 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2709 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2710 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2711 TILE_SPLIT(split_equal_to_row_size)); 2712 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2713 PIPE_CONFIG(ADDR_SURF_P4_16x16)); 2714 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2715 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2716 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2717 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2718 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2719 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2720 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2721 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2722 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2723 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2724 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2725 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2726 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2727 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2728 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2729 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2730 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2731 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2732 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2733 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2734 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2735 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2736 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2737 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2738 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2739 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2740 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2741 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2742 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2743 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2744 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2745 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2746 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2747 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2748 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2749 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2750 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2751 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2752 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2753 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2754 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2755 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2756 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2757 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2758 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2759 2760 } else if (num_rbs < 4) { 2761 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2762 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2763 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2764 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2765 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2766 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2767 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2768 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2769 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2770 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2771 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2772 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2773 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2774 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2775 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2776 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2777 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2778 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2779 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2780 TILE_SPLIT(split_equal_to_row_size)); 2781 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2782 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2783 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2784 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2785 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2786 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2787 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2788 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2789 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2790 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2791 TILE_SPLIT(split_equal_to_row_size)); 2792 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2793 PIPE_CONFIG(ADDR_SURF_P4_8x16)); 2794 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2795 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2796 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2797 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2798 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2799 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2800 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2801 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2802 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2803 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2804 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2805 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2806 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2807 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2808 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2809 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2810 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2811 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2812 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2813 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2814 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2815 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2816 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2817 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2818 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2819 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2820 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2821 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2822 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2823 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2824 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2825 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2826 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2827 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2828 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2829 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2830 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2831 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2832 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2833 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2834 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2835 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2836 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2837 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2838 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2839 } 2840 2841 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2842 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2843 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2844 NUM_BANKS(ADDR_SURF_16_BANK)); 2845 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2846 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2847 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2848 NUM_BANKS(ADDR_SURF_16_BANK)); 2849 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2850 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2851 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2852 NUM_BANKS(ADDR_SURF_16_BANK)); 2853 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2854 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2855 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2856 NUM_BANKS(ADDR_SURF_16_BANK)); 2857 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2858 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2859 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2860 NUM_BANKS(ADDR_SURF_16_BANK)); 2861 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2862 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2863 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2864 NUM_BANKS(ADDR_SURF_8_BANK)); 2865 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2866 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2867 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2868 NUM_BANKS(ADDR_SURF_4_BANK)); 2869 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2870 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2871 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2872 NUM_BANKS(ADDR_SURF_16_BANK)); 2873 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2874 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2875 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2876 NUM_BANKS(ADDR_SURF_16_BANK)); 2877 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2878 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2879 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2880 NUM_BANKS(ADDR_SURF_16_BANK)); 2881 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2882 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2883 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2884 NUM_BANKS(ADDR_SURF_16_BANK)); 2885 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2886 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2887 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2888 NUM_BANKS(ADDR_SURF_16_BANK)); 2889 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2890 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2891 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2892 NUM_BANKS(ADDR_SURF_8_BANK)); 2893 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2894 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2895 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2896 NUM_BANKS(ADDR_SURF_4_BANK)); 2897 2898 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2899 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]); 2900 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2901 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]); 2902 break; 2903 2904 case 2: 2905 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2906 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2907 PIPE_CONFIG(ADDR_SURF_P2) | 2908 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2909 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2910 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2911 PIPE_CONFIG(ADDR_SURF_P2) | 2912 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2913 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2914 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2915 PIPE_CONFIG(ADDR_SURF_P2) | 2916 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2917 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2918 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2919 PIPE_CONFIG(ADDR_SURF_P2) | 2920 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2921 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2922 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2923 PIPE_CONFIG(ADDR_SURF_P2) | 2924 TILE_SPLIT(split_equal_to_row_size)); 2925 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2926 PIPE_CONFIG(ADDR_SURF_P2) | 2927 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2928 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2929 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2930 PIPE_CONFIG(ADDR_SURF_P2) | 2931 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2932 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2933 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2934 PIPE_CONFIG(ADDR_SURF_P2) | 2935 TILE_SPLIT(split_equal_to_row_size)); 2936 tile[8] = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2937 PIPE_CONFIG(ADDR_SURF_P2); 2938 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2939 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2940 PIPE_CONFIG(ADDR_SURF_P2)); 2941 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2942 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2943 PIPE_CONFIG(ADDR_SURF_P2) | 2944 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2945 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2946 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2947 PIPE_CONFIG(ADDR_SURF_P2) | 2948 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2949 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2950 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2951 PIPE_CONFIG(ADDR_SURF_P2) | 2952 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2953 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2954 PIPE_CONFIG(ADDR_SURF_P2) | 2955 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2956 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2957 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2958 PIPE_CONFIG(ADDR_SURF_P2) | 2959 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2960 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2961 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2962 PIPE_CONFIG(ADDR_SURF_P2) | 2963 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2964 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2965 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2966 PIPE_CONFIG(ADDR_SURF_P2) | 2967 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2968 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2969 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2970 PIPE_CONFIG(ADDR_SURF_P2)); 2971 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2972 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2973 PIPE_CONFIG(ADDR_SURF_P2) | 2974 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2975 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2976 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2977 PIPE_CONFIG(ADDR_SURF_P2) | 2978 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2979 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2980 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2981 PIPE_CONFIG(ADDR_SURF_P2) | 2982 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2983 2984 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2985 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2986 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2987 NUM_BANKS(ADDR_SURF_16_BANK)); 2988 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2989 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2990 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2991 NUM_BANKS(ADDR_SURF_16_BANK)); 2992 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2993 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2994 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2995 NUM_BANKS(ADDR_SURF_16_BANK)); 2996 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2997 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2998 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2999 NUM_BANKS(ADDR_SURF_16_BANK)); 3000 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3001 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3002 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3003 NUM_BANKS(ADDR_SURF_16_BANK)); 3004 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3005 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3006 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3007 NUM_BANKS(ADDR_SURF_16_BANK)); 3008 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3009 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3010 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3011 NUM_BANKS(ADDR_SURF_8_BANK)); 3012 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3013 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3014 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3015 NUM_BANKS(ADDR_SURF_16_BANK)); 3016 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3017 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3018 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3019 NUM_BANKS(ADDR_SURF_16_BANK)); 3020 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3021 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3022 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3023 NUM_BANKS(ADDR_SURF_16_BANK)); 3024 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3025 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3026 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3027 NUM_BANKS(ADDR_SURF_16_BANK)); 3028 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3029 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3030 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3031 NUM_BANKS(ADDR_SURF_16_BANK)); 3032 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3033 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3034 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3035 NUM_BANKS(ADDR_SURF_16_BANK)); 3036 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3037 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3038 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3039 NUM_BANKS(ADDR_SURF_8_BANK)); 3040 3041 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 3042 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]); 3043 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 3044 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]); 3045 break; 3046 3047 default: 3048 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs); 3049 } 3050 } 3051 3052 /** 3053 * cik_select_se_sh - select which SE, SH to address 3054 * 3055 * @rdev: radeon_device pointer 3056 * @se_num: shader engine to address 3057 * @sh_num: sh block to address 3058 * 3059 * Select which SE, SH combinations to address. Certain 3060 * registers are instanced per SE or SH. 0xffffffff means 3061 * broadcast to all SEs or SHs (CIK). 3062 */ 3063 static void cik_select_se_sh(struct radeon_device *rdev, 3064 u32 se_num, u32 sh_num) 3065 { 3066 u32 data = INSTANCE_BROADCAST_WRITES; 3067 3068 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) 3069 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES; 3070 else if (se_num == 0xffffffff) 3071 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num); 3072 else if (sh_num == 0xffffffff) 3073 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num); 3074 else 3075 data |= SH_INDEX(sh_num) | SE_INDEX(se_num); 3076 WREG32(GRBM_GFX_INDEX, data); 3077 } 3078 3079 /** 3080 * cik_create_bitmask - create a bitmask 3081 * 3082 * @bit_width: length of the mask 3083 * 3084 * create a variable length bit mask (CIK). 3085 * Returns the bitmask. 3086 */ 3087 static u32 cik_create_bitmask(u32 bit_width) 3088 { 3089 u32 i, mask = 0; 3090 3091 for (i = 0; i < bit_width; i++) { 3092 mask <<= 1; 3093 mask |= 1; 3094 } 3095 return mask; 3096 } 3097 3098 /** 3099 * cik_get_rb_disabled - computes the mask of disabled RBs 3100 * 3101 * @rdev: radeon_device pointer 3102 * @max_rb_num: max RBs (render backends) for the asic 3103 * @se_num: number of SEs (shader engines) for the asic 3104 * @sh_per_se: number of SH blocks per SE for the asic 3105 * 3106 * Calculates the bitmask of disabled RBs (CIK). 3107 * Returns the disabled RB bitmask. 3108 */ 3109 static u32 cik_get_rb_disabled(struct radeon_device *rdev, 3110 u32 max_rb_num_per_se, 3111 u32 sh_per_se) 3112 { 3113 u32 data, mask; 3114 3115 data = RREG32(CC_RB_BACKEND_DISABLE); 3116 if (data & 1) 3117 data &= BACKEND_DISABLE_MASK; 3118 else 3119 data = 0; 3120 data |= RREG32(GC_USER_RB_BACKEND_DISABLE); 3121 3122 data >>= BACKEND_DISABLE_SHIFT; 3123 3124 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se); 3125 3126 return data & mask; 3127 } 3128 3129 /** 3130 * cik_setup_rb - setup the RBs on the asic 3131 * 3132 * @rdev: radeon_device pointer 3133 * @se_num: number of SEs (shader engines) for the asic 3134 * @sh_per_se: number of SH blocks per SE for the asic 3135 * @max_rb_num: max RBs (render backends) for the asic 3136 * 3137 * Configures per-SE/SH RB registers (CIK). 3138 */ 3139 static void cik_setup_rb(struct radeon_device *rdev, 3140 u32 se_num, u32 sh_per_se, 3141 u32 max_rb_num_per_se) 3142 { 3143 int i, j; 3144 u32 data, mask; 3145 u32 disabled_rbs = 0; 3146 u32 enabled_rbs = 0; 3147 3148 mutex_lock(&rdev->grbm_idx_mutex); 3149 for (i = 0; i < se_num; i++) { 3150 for (j = 0; j < sh_per_se; j++) { 3151 cik_select_se_sh(rdev, i, j); 3152 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se); 3153 if (rdev->family == CHIP_HAWAII) 3154 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH); 3155 else 3156 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH); 3157 } 3158 } 3159 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3160 mutex_unlock(&rdev->grbm_idx_mutex); 3161 3162 mask = 1; 3163 for (i = 0; i < max_rb_num_per_se * se_num; i++) { 3164 if (!(disabled_rbs & mask)) 3165 enabled_rbs |= mask; 3166 mask <<= 1; 3167 } 3168 3169 rdev->config.cik.backend_enable_mask = enabled_rbs; 3170 3171 mutex_lock(&rdev->grbm_idx_mutex); 3172 for (i = 0; i < se_num; i++) { 3173 cik_select_se_sh(rdev, i, 0xffffffff); 3174 data = 0; 3175 for (j = 0; j < sh_per_se; j++) { 3176 switch (enabled_rbs & 3) { 3177 case 0: 3178 if (j == 0) 3179 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3); 3180 else 3181 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0); 3182 break; 3183 case 1: 3184 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2); 3185 break; 3186 case 2: 3187 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2); 3188 break; 3189 case 3: 3190 default: 3191 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2); 3192 break; 3193 } 3194 enabled_rbs >>= 2; 3195 } 3196 WREG32(PA_SC_RASTER_CONFIG, data); 3197 } 3198 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3199 mutex_unlock(&rdev->grbm_idx_mutex); 3200 } 3201 3202 /** 3203 * cik_gpu_init - setup the 3D engine 3204 * 3205 * @rdev: radeon_device pointer 3206 * 3207 * Configures the 3D engine and tiling configuration 3208 * registers so that the 3D engine is usable. 3209 */ 3210 static void cik_gpu_init(struct radeon_device *rdev) 3211 { 3212 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG); 3213 u32 mc_shared_chmap, mc_arb_ramcfg; 3214 u32 hdp_host_path_cntl; 3215 u32 tmp; 3216 int i, j; 3217 3218 switch (rdev->family) { 3219 case CHIP_BONAIRE: 3220 rdev->config.cik.max_shader_engines = 2; 3221 rdev->config.cik.max_tile_pipes = 4; 3222 rdev->config.cik.max_cu_per_sh = 7; 3223 rdev->config.cik.max_sh_per_se = 1; 3224 rdev->config.cik.max_backends_per_se = 2; 3225 rdev->config.cik.max_texture_channel_caches = 4; 3226 rdev->config.cik.max_gprs = 256; 3227 rdev->config.cik.max_gs_threads = 32; 3228 rdev->config.cik.max_hw_contexts = 8; 3229 3230 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3231 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3232 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3233 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3234 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3235 break; 3236 case CHIP_HAWAII: 3237 rdev->config.cik.max_shader_engines = 4; 3238 rdev->config.cik.max_tile_pipes = 16; 3239 rdev->config.cik.max_cu_per_sh = 11; 3240 rdev->config.cik.max_sh_per_se = 1; 3241 rdev->config.cik.max_backends_per_se = 4; 3242 rdev->config.cik.max_texture_channel_caches = 16; 3243 rdev->config.cik.max_gprs = 256; 3244 rdev->config.cik.max_gs_threads = 32; 3245 rdev->config.cik.max_hw_contexts = 8; 3246 3247 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3248 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3249 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3250 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3251 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN; 3252 break; 3253 case CHIP_KAVERI: 3254 rdev->config.cik.max_shader_engines = 1; 3255 rdev->config.cik.max_tile_pipes = 4; 3256 if ((rdev->pdev->device == 0x1304) || 3257 (rdev->pdev->device == 0x1305) || 3258 (rdev->pdev->device == 0x130C) || 3259 (rdev->pdev->device == 0x130F) || 3260 (rdev->pdev->device == 0x1310) || 3261 (rdev->pdev->device == 0x1311) || 3262 (rdev->pdev->device == 0x131C)) { 3263 rdev->config.cik.max_cu_per_sh = 8; 3264 rdev->config.cik.max_backends_per_se = 2; 3265 } else if ((rdev->pdev->device == 0x1309) || 3266 (rdev->pdev->device == 0x130A) || 3267 (rdev->pdev->device == 0x130D) || 3268 (rdev->pdev->device == 0x1313) || 3269 (rdev->pdev->device == 0x131D)) { 3270 rdev->config.cik.max_cu_per_sh = 6; 3271 rdev->config.cik.max_backends_per_se = 2; 3272 } else if ((rdev->pdev->device == 0x1306) || 3273 (rdev->pdev->device == 0x1307) || 3274 (rdev->pdev->device == 0x130B) || 3275 (rdev->pdev->device == 0x130E) || 3276 (rdev->pdev->device == 0x1315) || 3277 (rdev->pdev->device == 0x1318) || 3278 (rdev->pdev->device == 0x131B)) { 3279 rdev->config.cik.max_cu_per_sh = 4; 3280 rdev->config.cik.max_backends_per_se = 1; 3281 } else { 3282 rdev->config.cik.max_cu_per_sh = 3; 3283 rdev->config.cik.max_backends_per_se = 1; 3284 } 3285 rdev->config.cik.max_sh_per_se = 1; 3286 rdev->config.cik.max_texture_channel_caches = 4; 3287 rdev->config.cik.max_gprs = 256; 3288 rdev->config.cik.max_gs_threads = 16; 3289 rdev->config.cik.max_hw_contexts = 8; 3290 3291 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3292 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3293 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3294 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3295 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3296 break; 3297 case CHIP_KABINI: 3298 case CHIP_MULLINS: 3299 default: 3300 rdev->config.cik.max_shader_engines = 1; 3301 rdev->config.cik.max_tile_pipes = 2; 3302 rdev->config.cik.max_cu_per_sh = 2; 3303 rdev->config.cik.max_sh_per_se = 1; 3304 rdev->config.cik.max_backends_per_se = 1; 3305 rdev->config.cik.max_texture_channel_caches = 2; 3306 rdev->config.cik.max_gprs = 256; 3307 rdev->config.cik.max_gs_threads = 16; 3308 rdev->config.cik.max_hw_contexts = 8; 3309 3310 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3311 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3312 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3313 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3314 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3315 break; 3316 } 3317 3318 /* Initialize HDP */ 3319 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3320 WREG32((0x2c14 + j), 0x00000000); 3321 WREG32((0x2c18 + j), 0x00000000); 3322 WREG32((0x2c1c + j), 0x00000000); 3323 WREG32((0x2c20 + j), 0x00000000); 3324 WREG32((0x2c24 + j), 0x00000000); 3325 } 3326 3327 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 3328 WREG32(SRBM_INT_CNTL, 0x1); 3329 WREG32(SRBM_INT_ACK, 0x1); 3330 3331 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 3332 3333 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 3334 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3335 3336 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes; 3337 rdev->config.cik.mem_max_burst_length_bytes = 256; 3338 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT; 3339 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 3340 if (rdev->config.cik.mem_row_size_in_kb > 4) 3341 rdev->config.cik.mem_row_size_in_kb = 4; 3342 /* XXX use MC settings? */ 3343 rdev->config.cik.shader_engine_tile_size = 32; 3344 rdev->config.cik.num_gpus = 1; 3345 rdev->config.cik.multi_gpu_tile_size = 64; 3346 3347 /* fix up row size */ 3348 gb_addr_config &= ~ROW_SIZE_MASK; 3349 switch (rdev->config.cik.mem_row_size_in_kb) { 3350 case 1: 3351 default: 3352 gb_addr_config |= ROW_SIZE(0); 3353 break; 3354 case 2: 3355 gb_addr_config |= ROW_SIZE(1); 3356 break; 3357 case 4: 3358 gb_addr_config |= ROW_SIZE(2); 3359 break; 3360 } 3361 3362 /* setup tiling info dword. gb_addr_config is not adequate since it does 3363 * not have bank info, so create a custom tiling dword. 3364 * bits 3:0 num_pipes 3365 * bits 7:4 num_banks 3366 * bits 11:8 group_size 3367 * bits 15:12 row_size 3368 */ 3369 rdev->config.cik.tile_config = 0; 3370 switch (rdev->config.cik.num_tile_pipes) { 3371 case 1: 3372 rdev->config.cik.tile_config |= (0 << 0); 3373 break; 3374 case 2: 3375 rdev->config.cik.tile_config |= (1 << 0); 3376 break; 3377 case 4: 3378 rdev->config.cik.tile_config |= (2 << 0); 3379 break; 3380 case 8: 3381 default: 3382 /* XXX what about 12? */ 3383 rdev->config.cik.tile_config |= (3 << 0); 3384 break; 3385 } 3386 rdev->config.cik.tile_config |= 3387 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4; 3388 rdev->config.cik.tile_config |= 3389 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8; 3390 rdev->config.cik.tile_config |= 3391 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12; 3392 3393 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3394 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3395 WREG32(DMIF_ADDR_CALC, gb_addr_config); 3396 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70); 3397 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70); 3398 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3399 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3400 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3401 3402 cik_tiling_mode_table_init(rdev); 3403 3404 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines, 3405 rdev->config.cik.max_sh_per_se, 3406 rdev->config.cik.max_backends_per_se); 3407 3408 rdev->config.cik.active_cus = 0; 3409 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 3410 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 3411 rdev->config.cik.active_cus += 3412 hweight32(cik_get_cu_active_bitmap(rdev, i, j)); 3413 } 3414 } 3415 3416 /* set HW defaults for 3D engine */ 3417 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); 3418 3419 mutex_lock(&rdev->grbm_idx_mutex); 3420 /* 3421 * making sure that the following register writes will be broadcasted 3422 * to all the shaders 3423 */ 3424 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3425 WREG32(SX_DEBUG_1, 0x20); 3426 3427 WREG32(TA_CNTL_AUX, 0x00010000); 3428 3429 tmp = RREG32(SPI_CONFIG_CNTL); 3430 tmp |= 0x03000000; 3431 WREG32(SPI_CONFIG_CNTL, tmp); 3432 3433 WREG32(SQ_CONFIG, 1); 3434 3435 WREG32(DB_DEBUG, 0); 3436 3437 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff; 3438 tmp |= 0x00000400; 3439 WREG32(DB_DEBUG2, tmp); 3440 3441 tmp = RREG32(DB_DEBUG3) & ~0x0002021c; 3442 tmp |= 0x00020200; 3443 WREG32(DB_DEBUG3, tmp); 3444 3445 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000; 3446 tmp |= 0x00018208; 3447 WREG32(CB_HW_CONTROL, tmp); 3448 3449 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3450 3451 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) | 3452 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) | 3453 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) | 3454 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size))); 3455 3456 WREG32(VGT_NUM_INSTANCES, 1); 3457 3458 WREG32(CP_PERFMON_CNTL, 0); 3459 3460 WREG32(SQ_CONFIG, 0); 3461 3462 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3463 FORCE_EOV_MAX_REZ_CNT(255))); 3464 3465 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) | 3466 AUTO_INVLD_EN(ES_AND_GS_AUTO)); 3467 3468 WREG32(VGT_GS_VERTEX_REUSE, 16); 3469 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3470 3471 tmp = RREG32(HDP_MISC_CNTL); 3472 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3473 WREG32(HDP_MISC_CNTL, tmp); 3474 3475 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3476 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3477 3478 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3479 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER); 3480 mutex_unlock(&rdev->grbm_idx_mutex); 3481 3482 udelay(50); 3483 } 3484 3485 /* 3486 * GPU scratch registers helpers function. 3487 */ 3488 /** 3489 * cik_scratch_init - setup driver info for CP scratch regs 3490 * 3491 * @rdev: radeon_device pointer 3492 * 3493 * Set up the number and offset of the CP scratch registers. 3494 * NOTE: use of CP scratch registers is a legacy inferface and 3495 * is not used by default on newer asics (r6xx+). On newer asics, 3496 * memory buffers are used for fences rather than scratch regs. 3497 */ 3498 static void cik_scratch_init(struct radeon_device *rdev) 3499 { 3500 int i; 3501 3502 rdev->scratch.num_reg = 7; 3503 rdev->scratch.reg_base = SCRATCH_REG0; 3504 for (i = 0; i < rdev->scratch.num_reg; i++) { 3505 rdev->scratch.free[i] = true; 3506 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); 3507 } 3508 } 3509 3510 /** 3511 * cik_ring_test - basic gfx ring test 3512 * 3513 * @rdev: radeon_device pointer 3514 * @ring: radeon_ring structure holding ring information 3515 * 3516 * Allocate a scratch register and write to it using the gfx ring (CIK). 3517 * Provides a basic gfx ring test to verify that the ring is working. 3518 * Used by cik_cp_gfx_resume(); 3519 * Returns 0 on success, error on failure. 3520 */ 3521 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) 3522 { 3523 uint32_t scratch; 3524 uint32_t tmp = 0; 3525 unsigned i; 3526 int r; 3527 3528 r = radeon_scratch_get(rdev, &scratch); 3529 if (r) { 3530 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r); 3531 return r; 3532 } 3533 WREG32(scratch, 0xCAFEDEAD); 3534 r = radeon_ring_lock(rdev, ring, 3); 3535 if (r) { 3536 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r); 3537 radeon_scratch_free(rdev, scratch); 3538 return r; 3539 } 3540 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 3541 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2)); 3542 radeon_ring_write(ring, 0xDEADBEEF); 3543 radeon_ring_unlock_commit(rdev, ring, false); 3544 3545 for (i = 0; i < rdev->usec_timeout; i++) { 3546 tmp = RREG32(scratch); 3547 if (tmp == 0xDEADBEEF) 3548 break; 3549 DRM_UDELAY(1); 3550 } 3551 if (i < rdev->usec_timeout) { 3552 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 3553 } else { 3554 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n", 3555 ring->idx, scratch, tmp); 3556 r = -EINVAL; 3557 } 3558 radeon_scratch_free(rdev, scratch); 3559 return r; 3560 } 3561 3562 /** 3563 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp 3564 * 3565 * @rdev: radeon_device pointer 3566 * @ridx: radeon ring index 3567 * 3568 * Emits an hdp flush on the cp. 3569 */ 3570 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev, 3571 int ridx) 3572 { 3573 struct radeon_ring *ring = &rdev->ring[ridx]; 3574 u32 ref_and_mask; 3575 3576 switch (ring->idx) { 3577 case CAYMAN_RING_TYPE_CP1_INDEX: 3578 case CAYMAN_RING_TYPE_CP2_INDEX: 3579 default: 3580 switch (ring->me) { 3581 case 0: 3582 ref_and_mask = CP2 << ring->pipe; 3583 break; 3584 case 1: 3585 ref_and_mask = CP6 << ring->pipe; 3586 break; 3587 default: 3588 return; 3589 } 3590 break; 3591 case RADEON_RING_TYPE_GFX_INDEX: 3592 ref_and_mask = CP0; 3593 break; 3594 } 3595 3596 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 3597 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */ 3598 WAIT_REG_MEM_FUNCTION(3) | /* == */ 3599 WAIT_REG_MEM_ENGINE(1))); /* pfp */ 3600 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2); 3601 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2); 3602 radeon_ring_write(ring, ref_and_mask); 3603 radeon_ring_write(ring, ref_and_mask); 3604 radeon_ring_write(ring, 0x20); /* poll interval */ 3605 } 3606 3607 /** 3608 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring 3609 * 3610 * @rdev: radeon_device pointer 3611 * @fence: radeon fence object 3612 * 3613 * Emits a fence sequnce number on the gfx ring and flushes 3614 * GPU caches. 3615 */ 3616 void cik_fence_gfx_ring_emit(struct radeon_device *rdev, 3617 struct radeon_fence *fence) 3618 { 3619 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3620 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3621 3622 /* Workaround for cache flush problems. First send a dummy EOP 3623 * event down the pipe with seq one below. 3624 */ 3625 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3626 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3627 EOP_TC_ACTION_EN | 3628 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3629 EVENT_INDEX(5))); 3630 radeon_ring_write(ring, addr & 0xfffffffc); 3631 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | 3632 DATA_SEL(1) | INT_SEL(0)); 3633 radeon_ring_write(ring, fence->seq - 1); 3634 radeon_ring_write(ring, 0); 3635 3636 /* Then send the real EOP event down the pipe. */ 3637 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3638 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3639 EOP_TC_ACTION_EN | 3640 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3641 EVENT_INDEX(5))); 3642 radeon_ring_write(ring, addr & 0xfffffffc); 3643 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2)); 3644 radeon_ring_write(ring, fence->seq); 3645 radeon_ring_write(ring, 0); 3646 } 3647 3648 /** 3649 * cik_fence_compute_ring_emit - emit a fence on the compute ring 3650 * 3651 * @rdev: radeon_device pointer 3652 * @fence: radeon fence object 3653 * 3654 * Emits a fence sequnce number on the compute ring and flushes 3655 * GPU caches. 3656 */ 3657 void cik_fence_compute_ring_emit(struct radeon_device *rdev, 3658 struct radeon_fence *fence) 3659 { 3660 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3661 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3662 3663 /* RELEASE_MEM - flush caches, send int */ 3664 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5)); 3665 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3666 EOP_TC_ACTION_EN | 3667 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3668 EVENT_INDEX(5))); 3669 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2)); 3670 radeon_ring_write(ring, addr & 0xfffffffc); 3671 radeon_ring_write(ring, upper_32_bits(addr)); 3672 radeon_ring_write(ring, fence->seq); 3673 radeon_ring_write(ring, 0); 3674 } 3675 3676 /** 3677 * cik_semaphore_ring_emit - emit a semaphore on the CP ring 3678 * 3679 * @rdev: radeon_device pointer 3680 * @ring: radeon ring buffer object 3681 * @semaphore: radeon semaphore object 3682 * @emit_wait: Is this a sempahore wait? 3683 * 3684 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP 3685 * from running ahead of semaphore waits. 3686 */ 3687 bool cik_semaphore_ring_emit(struct radeon_device *rdev, 3688 struct radeon_ring *ring, 3689 struct radeon_semaphore *semaphore, 3690 bool emit_wait) 3691 { 3692 uint64_t addr = semaphore->gpu_addr; 3693 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL; 3694 3695 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1)); 3696 radeon_ring_write(ring, lower_32_bits(addr)); 3697 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel); 3698 3699 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) { 3700 /* Prevent the PFP from running ahead of the semaphore wait */ 3701 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 3702 radeon_ring_write(ring, 0x0); 3703 } 3704 3705 return true; 3706 } 3707 3708 /** 3709 * cik_copy_cpdma - copy pages using the CP DMA engine 3710 * 3711 * @rdev: radeon_device pointer 3712 * @src_offset: src GPU address 3713 * @dst_offset: dst GPU address 3714 * @num_gpu_pages: number of GPU pages to xfer 3715 * @fence: radeon fence object 3716 * 3717 * Copy GPU paging using the CP DMA engine (CIK+). 3718 * Used by the radeon ttm implementation to move pages if 3719 * registered as the asic copy callback. 3720 */ 3721 int cik_copy_cpdma(struct radeon_device *rdev, 3722 uint64_t src_offset, uint64_t dst_offset, 3723 unsigned num_gpu_pages, 3724 struct radeon_fence **fence) 3725 { 3726 struct radeon_semaphore *sem = NULL; 3727 int ring_index = rdev->asic->copy.blit_ring_index; 3728 struct radeon_ring *ring = &rdev->ring[ring_index]; 3729 u32 size_in_bytes, cur_size_in_bytes, control; 3730 int i, num_loops; 3731 int r = 0; 3732 3733 r = radeon_semaphore_create(rdev, &sem); 3734 if (r) { 3735 DRM_ERROR("radeon: moving bo (%d).\n", r); 3736 return r; 3737 } 3738 3739 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); 3740 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); 3741 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18); 3742 if (r) { 3743 DRM_ERROR("radeon: moving bo (%d).\n", r); 3744 radeon_semaphore_free(rdev, &sem, NULL); 3745 return r; 3746 } 3747 3748 radeon_semaphore_sync_to(sem, *fence); 3749 radeon_semaphore_sync_rings(rdev, sem, ring->idx); 3750 3751 for (i = 0; i < num_loops; i++) { 3752 cur_size_in_bytes = size_in_bytes; 3753 if (cur_size_in_bytes > 0x1fffff) 3754 cur_size_in_bytes = 0x1fffff; 3755 size_in_bytes -= cur_size_in_bytes; 3756 control = 0; 3757 if (size_in_bytes == 0) 3758 control |= PACKET3_DMA_DATA_CP_SYNC; 3759 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 3760 radeon_ring_write(ring, control); 3761 radeon_ring_write(ring, lower_32_bits(src_offset)); 3762 radeon_ring_write(ring, upper_32_bits(src_offset)); 3763 radeon_ring_write(ring, lower_32_bits(dst_offset)); 3764 radeon_ring_write(ring, upper_32_bits(dst_offset)); 3765 radeon_ring_write(ring, cur_size_in_bytes); 3766 src_offset += cur_size_in_bytes; 3767 dst_offset += cur_size_in_bytes; 3768 } 3769 3770 r = radeon_fence_emit(rdev, fence, ring->idx); 3771 if (r) { 3772 radeon_ring_unlock_undo(rdev, ring); 3773 radeon_semaphore_free(rdev, &sem, NULL); 3774 return r; 3775 } 3776 3777 radeon_ring_unlock_commit(rdev, ring, false); 3778 radeon_semaphore_free(rdev, &sem, *fence); 3779 3780 return r; 3781 } 3782 3783 /* 3784 * IB stuff 3785 */ 3786 /** 3787 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring 3788 * 3789 * @rdev: radeon_device pointer 3790 * @ib: radeon indirect buffer object 3791 * 3792 * Emits a DE (drawing engine) or CE (constant engine) IB 3793 * on the gfx ring. IBs are usually generated by userspace 3794 * acceleration drivers and submitted to the kernel for 3795 * scheduling on the ring. This function schedules the IB 3796 * on the gfx ring for execution by the GPU. 3797 */ 3798 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 3799 { 3800 struct radeon_ring *ring = &rdev->ring[ib->ring]; 3801 u32 header, control = INDIRECT_BUFFER_VALID; 3802 3803 if (ib->is_const_ib) { 3804 /* set switch buffer packet before const IB */ 3805 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 3806 radeon_ring_write(ring, 0); 3807 3808 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 3809 } else { 3810 u32 next_rptr; 3811 if (ring->rptr_save_reg) { 3812 next_rptr = ring->wptr + 3 + 4; 3813 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 3814 radeon_ring_write(ring, ((ring->rptr_save_reg - 3815 PACKET3_SET_UCONFIG_REG_START) >> 2)); 3816 radeon_ring_write(ring, next_rptr); 3817 } else if (rdev->wb.enabled) { 3818 next_rptr = ring->wptr + 5 + 4; 3819 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 3820 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1)); 3821 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 3822 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr)); 3823 radeon_ring_write(ring, next_rptr); 3824 } 3825 3826 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 3827 } 3828 3829 control |= ib->length_dw | 3830 (ib->vm ? (ib->vm->id << 24) : 0); 3831 3832 radeon_ring_write(ring, header); 3833 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC)); 3834 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 3835 radeon_ring_write(ring, control); 3836 } 3837 3838 /** 3839 * cik_ib_test - basic gfx ring IB test 3840 * 3841 * @rdev: radeon_device pointer 3842 * @ring: radeon_ring structure holding ring information 3843 * 3844 * Allocate an IB and execute it on the gfx ring (CIK). 3845 * Provides a basic gfx ring test to verify that IBs are working. 3846 * Returns 0 on success, error on failure. 3847 */ 3848 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) 3849 { 3850 struct radeon_ib ib; 3851 uint32_t scratch; 3852 uint32_t tmp = 0; 3853 unsigned i; 3854 int r; 3855 3856 r = radeon_scratch_get(rdev, &scratch); 3857 if (r) { 3858 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r); 3859 return r; 3860 } 3861 WREG32(scratch, 0xCAFEDEAD); 3862 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); 3863 if (r) { 3864 DRM_ERROR("radeon: failed to get ib (%d).\n", r); 3865 radeon_scratch_free(rdev, scratch); 3866 return r; 3867 } 3868 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1); 3869 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2); 3870 ib.ptr[2] = 0xDEADBEEF; 3871 ib.length_dw = 3; 3872 r = radeon_ib_schedule(rdev, &ib, NULL, false); 3873 if (r) { 3874 radeon_scratch_free(rdev, scratch); 3875 radeon_ib_free(rdev, &ib); 3876 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); 3877 return r; 3878 } 3879 r = radeon_fence_wait_timeout(ib.fence, false, usecs_to_jiffies( 3880 RADEON_USEC_IB_TEST_TIMEOUT)); 3881 if (r < 0) { 3882 DRM_ERROR("radeon: fence wait failed (%d).\n", r); 3883 radeon_scratch_free(rdev, scratch); 3884 radeon_ib_free(rdev, &ib); 3885 return r; 3886 } else if (r == 0) { 3887 DRM_ERROR("radeon: fence wait timed out.\n"); 3888 #if 0 3889 radeon_scratch_free(rdev, scratch); 3890 radeon_ib_free(rdev, &ib); 3891 return -ETIMEDOUT; 3892 #endif 3893 } 3894 r = 0; 3895 for (i = 0; i < rdev->usec_timeout; i++) { 3896 tmp = RREG32(scratch); 3897 if (tmp == 0xDEADBEEF) 3898 break; 3899 DRM_UDELAY(1); 3900 } 3901 if (i < rdev->usec_timeout) { 3902 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i); 3903 } else { 3904 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n", 3905 scratch, tmp); 3906 r = -EINVAL; 3907 } 3908 radeon_scratch_free(rdev, scratch); 3909 radeon_ib_free(rdev, &ib); 3910 return r; 3911 } 3912 3913 /* 3914 * CP. 3915 * On CIK, gfx and compute now have independant command processors. 3916 * 3917 * GFX 3918 * Gfx consists of a single ring and can process both gfx jobs and 3919 * compute jobs. The gfx CP consists of three microengines (ME): 3920 * PFP - Pre-Fetch Parser 3921 * ME - Micro Engine 3922 * CE - Constant Engine 3923 * The PFP and ME make up what is considered the Drawing Engine (DE). 3924 * The CE is an asynchronous engine used for updating buffer desciptors 3925 * used by the DE so that they can be loaded into cache in parallel 3926 * while the DE is processing state update packets. 3927 * 3928 * Compute 3929 * The compute CP consists of two microengines (ME): 3930 * MEC1 - Compute MicroEngine 1 3931 * MEC2 - Compute MicroEngine 2 3932 * Each MEC supports 4 compute pipes and each pipe supports 8 queues. 3933 * The queues are exposed to userspace and are programmed directly 3934 * by the compute runtime. 3935 */ 3936 /** 3937 * cik_cp_gfx_enable - enable/disable the gfx CP MEs 3938 * 3939 * @rdev: radeon_device pointer 3940 * @enable: enable or disable the MEs 3941 * 3942 * Halts or unhalts the gfx MEs. 3943 */ 3944 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable) 3945 { 3946 if (enable) 3947 WREG32(CP_ME_CNTL, 0); 3948 else { 3949 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 3950 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 3951 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT)); 3952 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 3953 } 3954 udelay(50); 3955 } 3956 3957 /** 3958 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode 3959 * 3960 * @rdev: radeon_device pointer 3961 * 3962 * Loads the gfx PFP, ME, and CE ucode. 3963 * Returns 0 for success, -EINVAL if the ucode is not available. 3964 */ 3965 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev) 3966 { 3967 int i; 3968 3969 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw) 3970 return -EINVAL; 3971 3972 cik_cp_gfx_enable(rdev, false); 3973 3974 if (rdev->new_fw) { 3975 const struct gfx_firmware_header_v1_0 *pfp_hdr = 3976 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; 3977 const struct gfx_firmware_header_v1_0 *ce_hdr = 3978 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; 3979 const struct gfx_firmware_header_v1_0 *me_hdr = 3980 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; 3981 const __le32 *fw_data; 3982 u32 fw_size; 3983 3984 radeon_ucode_print_gfx_hdr(&pfp_hdr->header); 3985 radeon_ucode_print_gfx_hdr(&ce_hdr->header); 3986 radeon_ucode_print_gfx_hdr(&me_hdr->header); 3987 3988 /* PFP */ 3989 fw_data = (const __le32 *) 3990 ((const char *)rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3991 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3992 WREG32(CP_PFP_UCODE_ADDR, 0); 3993 for (i = 0; i < fw_size; i++) 3994 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3995 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version)); 3996 3997 /* CE */ 3998 fw_data = (const __le32 *) 3999 ((const char *)rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 4000 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 4001 WREG32(CP_CE_UCODE_ADDR, 0); 4002 for (i = 0; i < fw_size; i++) 4003 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 4004 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version)); 4005 4006 /* ME */ 4007 fw_data = (const __be32 *) 4008 ((const char *)rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 4009 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 4010 WREG32(CP_ME_RAM_WADDR, 0); 4011 for (i = 0; i < fw_size; i++) 4012 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 4013 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version)); 4014 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version)); 4015 } else { 4016 const __be32 *fw_data; 4017 4018 /* PFP */ 4019 fw_data = (const __be32 *)rdev->pfp_fw->data; 4020 WREG32(CP_PFP_UCODE_ADDR, 0); 4021 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++) 4022 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 4023 WREG32(CP_PFP_UCODE_ADDR, 0); 4024 4025 /* CE */ 4026 fw_data = (const __be32 *)rdev->ce_fw->data; 4027 WREG32(CP_CE_UCODE_ADDR, 0); 4028 for (i = 0; i < CIK_CE_UCODE_SIZE; i++) 4029 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); 4030 WREG32(CP_CE_UCODE_ADDR, 0); 4031 4032 /* ME */ 4033 fw_data = (const __be32 *)rdev->me_fw->data; 4034 WREG32(CP_ME_RAM_WADDR, 0); 4035 for (i = 0; i < CIK_ME_UCODE_SIZE; i++) 4036 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 4037 WREG32(CP_ME_RAM_WADDR, 0); 4038 } 4039 4040 return 0; 4041 } 4042 4043 /** 4044 * cik_cp_gfx_start - start the gfx ring 4045 * 4046 * @rdev: radeon_device pointer 4047 * 4048 * Enables the ring and loads the clear state context and other 4049 * packets required to init the ring. 4050 * Returns 0 for success, error for failure. 4051 */ 4052 static int cik_cp_gfx_start(struct radeon_device *rdev) 4053 { 4054 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4055 int r, i; 4056 4057 /* init the CP */ 4058 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1); 4059 WREG32(CP_ENDIAN_SWAP, 0); 4060 WREG32(CP_DEVICE_ID, 1); 4061 4062 cik_cp_gfx_enable(rdev, true); 4063 4064 r = radeon_ring_lock(rdev, ring, cik_default_size + 17); 4065 if (r) { 4066 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 4067 return r; 4068 } 4069 4070 /* init the CE partitions. CE only used for gfx on CIK */ 4071 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 4072 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 4073 radeon_ring_write(ring, 0x8000); 4074 radeon_ring_write(ring, 0x8000); 4075 4076 /* setup clear context state */ 4077 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4078 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 4079 4080 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 4081 radeon_ring_write(ring, 0x80000000); 4082 radeon_ring_write(ring, 0x80000000); 4083 4084 for (i = 0; i < cik_default_size; i++) 4085 radeon_ring_write(ring, cik_default_state[i]); 4086 4087 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4088 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 4089 4090 /* set clear context state */ 4091 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 4092 radeon_ring_write(ring, 0); 4093 4094 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 4095 radeon_ring_write(ring, 0x00000316); 4096 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 4097 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */ 4098 4099 radeon_ring_unlock_commit(rdev, ring, false); 4100 4101 return 0; 4102 } 4103 4104 /** 4105 * cik_cp_gfx_fini - stop the gfx ring 4106 * 4107 * @rdev: radeon_device pointer 4108 * 4109 * Stop the gfx ring and tear down the driver ring 4110 * info. 4111 */ 4112 static void cik_cp_gfx_fini(struct radeon_device *rdev) 4113 { 4114 cik_cp_gfx_enable(rdev, false); 4115 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 4116 } 4117 4118 /** 4119 * cik_cp_gfx_resume - setup the gfx ring buffer registers 4120 * 4121 * @rdev: radeon_device pointer 4122 * 4123 * Program the location and size of the gfx ring buffer 4124 * and test it to make sure it's working. 4125 * Returns 0 for success, error for failure. 4126 */ 4127 static int cik_cp_gfx_resume(struct radeon_device *rdev) 4128 { 4129 struct radeon_ring *ring; 4130 u32 tmp; 4131 u32 rb_bufsz; 4132 u64 rb_addr; 4133 int r; 4134 4135 WREG32(CP_SEM_WAIT_TIMER, 0x0); 4136 if (rdev->family != CHIP_HAWAII) 4137 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 4138 4139 /* Set the write pointer delay */ 4140 WREG32(CP_RB_WPTR_DELAY, 0); 4141 4142 /* set the RB to use vmid 0 */ 4143 WREG32(CP_RB_VMID, 0); 4144 4145 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 4146 4147 /* ring 0 - compute and gfx */ 4148 /* Set ring buffer size */ 4149 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4150 rb_bufsz = order_base_2(ring->ring_size / 8); 4151 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 4152 #ifdef __BIG_ENDIAN 4153 tmp |= BUF_SWAP_32BIT; 4154 #endif 4155 WREG32(CP_RB0_CNTL, tmp); 4156 4157 /* Initialize the ring buffer's read and write pointers */ 4158 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA); 4159 ring->wptr = 0; 4160 WREG32(CP_RB0_WPTR, ring->wptr); 4161 4162 /* set the wb address wether it's enabled or not */ 4163 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); 4164 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 4165 4166 /* scratch register shadowing is no longer supported */ 4167 WREG32(SCRATCH_UMSK, 0); 4168 4169 if (!rdev->wb.enabled) 4170 tmp |= RB_NO_UPDATE; 4171 4172 mdelay(1); 4173 WREG32(CP_RB0_CNTL, tmp); 4174 4175 rb_addr = ring->gpu_addr >> 8; 4176 WREG32(CP_RB0_BASE, rb_addr); 4177 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr)); 4178 4179 /* start the ring */ 4180 cik_cp_gfx_start(rdev); 4181 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; 4182 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 4183 if (r) { 4184 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 4185 return r; 4186 } 4187 4188 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 4189 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); 4190 4191 return 0; 4192 } 4193 4194 u32 cik_gfx_get_rptr(struct radeon_device *rdev, 4195 struct radeon_ring *ring) 4196 { 4197 u32 rptr; 4198 4199 if (rdev->wb.enabled) 4200 rptr = rdev->wb.wb[ring->rptr_offs/4]; 4201 else 4202 rptr = RREG32(CP_RB0_RPTR); 4203 4204 return rptr; 4205 } 4206 4207 u32 cik_gfx_get_wptr(struct radeon_device *rdev, 4208 struct radeon_ring *ring) 4209 { 4210 u32 wptr; 4211 4212 wptr = RREG32(CP_RB0_WPTR); 4213 4214 return wptr; 4215 } 4216 4217 void cik_gfx_set_wptr(struct radeon_device *rdev, 4218 struct radeon_ring *ring) 4219 { 4220 WREG32(CP_RB0_WPTR, ring->wptr); 4221 (void)RREG32(CP_RB0_WPTR); 4222 } 4223 4224 u32 cik_compute_get_rptr(struct radeon_device *rdev, 4225 struct radeon_ring *ring) 4226 { 4227 u32 rptr; 4228 4229 if (rdev->wb.enabled) { 4230 rptr = rdev->wb.wb[ring->rptr_offs/4]; 4231 } else { 4232 mutex_lock(&rdev->srbm_mutex); 4233 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4234 rptr = RREG32(CP_HQD_PQ_RPTR); 4235 cik_srbm_select(rdev, 0, 0, 0, 0); 4236 mutex_unlock(&rdev->srbm_mutex); 4237 } 4238 4239 return rptr; 4240 } 4241 4242 u32 cik_compute_get_wptr(struct radeon_device *rdev, 4243 struct radeon_ring *ring) 4244 { 4245 u32 wptr; 4246 4247 if (rdev->wb.enabled) { 4248 /* XXX check if swapping is necessary on BE */ 4249 wptr = rdev->wb.wb[ring->wptr_offs/4]; 4250 } else { 4251 mutex_lock(&rdev->srbm_mutex); 4252 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4253 wptr = RREG32(CP_HQD_PQ_WPTR); 4254 cik_srbm_select(rdev, 0, 0, 0, 0); 4255 mutex_unlock(&rdev->srbm_mutex); 4256 } 4257 4258 return wptr; 4259 } 4260 4261 void cik_compute_set_wptr(struct radeon_device *rdev, 4262 struct radeon_ring *ring) 4263 { 4264 /* XXX check if swapping is necessary on BE */ 4265 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr; 4266 WDOORBELL32(ring->doorbell_index, ring->wptr); 4267 } 4268 4269 static void cik_compute_stop(struct radeon_device *rdev, 4270 struct radeon_ring *ring) 4271 { 4272 u32 j, tmp; 4273 4274 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4275 /* Disable wptr polling. */ 4276 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL); 4277 tmp &= ~WPTR_POLL_EN; 4278 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp); 4279 /* Disable HQD. */ 4280 if (RREG32(CP_HQD_ACTIVE) & 1) { 4281 WREG32(CP_HQD_DEQUEUE_REQUEST, 1); 4282 for (j = 0; j < rdev->usec_timeout; j++) { 4283 if (!(RREG32(CP_HQD_ACTIVE) & 1)) 4284 break; 4285 udelay(1); 4286 } 4287 WREG32(CP_HQD_DEQUEUE_REQUEST, 0); 4288 WREG32(CP_HQD_PQ_RPTR, 0); 4289 WREG32(CP_HQD_PQ_WPTR, 0); 4290 } 4291 cik_srbm_select(rdev, 0, 0, 0, 0); 4292 } 4293 4294 /** 4295 * cik_cp_compute_enable - enable/disable the compute CP MEs 4296 * 4297 * @rdev: radeon_device pointer 4298 * @enable: enable or disable the MEs 4299 * 4300 * Halts or unhalts the compute MEs. 4301 */ 4302 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable) 4303 { 4304 if (enable) 4305 WREG32(CP_MEC_CNTL, 0); 4306 else { 4307 /* 4308 * To make hibernation reliable we need to clear compute ring 4309 * configuration before halting the compute ring. 4310 */ 4311 mutex_lock(&rdev->srbm_mutex); 4312 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]); 4313 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]); 4314 mutex_unlock(&rdev->srbm_mutex); 4315 4316 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT)); 4317 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 4318 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 4319 } 4320 udelay(50); 4321 } 4322 4323 /** 4324 * cik_cp_compute_load_microcode - load the compute CP ME ucode 4325 * 4326 * @rdev: radeon_device pointer 4327 * 4328 * Loads the compute MEC1&2 ucode. 4329 * Returns 0 for success, -EINVAL if the ucode is not available. 4330 */ 4331 static int cik_cp_compute_load_microcode(struct radeon_device *rdev) 4332 { 4333 int i; 4334 4335 if (!rdev->mec_fw) 4336 return -EINVAL; 4337 4338 cik_cp_compute_enable(rdev, false); 4339 4340 if (rdev->new_fw) { 4341 const struct gfx_firmware_header_v1_0 *mec_hdr = 4342 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; 4343 const __le32 *fw_data; 4344 u32 fw_size; 4345 4346 radeon_ucode_print_gfx_hdr(&mec_hdr->header); 4347 4348 /* MEC1 */ 4349 fw_data = (const __le32 *) 4350 ((const char *)rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 4351 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4; 4352 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4353 for (i = 0; i < fw_size; i++) 4354 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++)); 4355 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version)); 4356 4357 /* MEC2 */ 4358 if (rdev->family == CHIP_KAVERI) { 4359 const struct gfx_firmware_header_v1_0 *mec2_hdr = 4360 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; 4361 4362 fw_data = (const __le32 *) 4363 ((const char *)rdev->mec2_fw->data + 4364 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes)); 4365 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4; 4366 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4367 for (i = 0; i < fw_size; i++) 4368 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++)); 4369 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version)); 4370 } 4371 } else { 4372 const __be32 *fw_data; 4373 4374 /* MEC1 */ 4375 fw_data = (const __be32 *)rdev->mec_fw->data; 4376 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4377 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) 4378 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++)); 4379 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4380 4381 if (rdev->family == CHIP_KAVERI) { 4382 /* MEC2 */ 4383 fw_data = (const __be32 *)rdev->mec_fw->data; 4384 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4385 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) 4386 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++)); 4387 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4388 } 4389 } 4390 4391 return 0; 4392 } 4393 4394 /** 4395 * cik_cp_compute_start - start the compute queues 4396 * 4397 * @rdev: radeon_device pointer 4398 * 4399 * Enable the compute queues. 4400 * Returns 0 for success, error for failure. 4401 */ 4402 static int cik_cp_compute_start(struct radeon_device *rdev) 4403 { 4404 cik_cp_compute_enable(rdev, true); 4405 4406 return 0; 4407 } 4408 4409 /** 4410 * cik_cp_compute_fini - stop the compute queues 4411 * 4412 * @rdev: radeon_device pointer 4413 * 4414 * Stop the compute queues and tear down the driver queue 4415 * info. 4416 */ 4417 static void cik_cp_compute_fini(struct radeon_device *rdev) 4418 { 4419 int i, idx, r; 4420 4421 cik_cp_compute_enable(rdev, false); 4422 4423 for (i = 0; i < 2; i++) { 4424 if (i == 0) 4425 idx = CAYMAN_RING_TYPE_CP1_INDEX; 4426 else 4427 idx = CAYMAN_RING_TYPE_CP2_INDEX; 4428 4429 if (rdev->ring[idx].mqd_obj) { 4430 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); 4431 if (unlikely(r != 0)) 4432 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r); 4433 4434 radeon_bo_unpin(rdev->ring[idx].mqd_obj); 4435 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); 4436 4437 radeon_bo_unref(&rdev->ring[idx].mqd_obj); 4438 rdev->ring[idx].mqd_obj = NULL; 4439 } 4440 } 4441 } 4442 4443 static void cik_mec_fini(struct radeon_device *rdev) 4444 { 4445 int r; 4446 4447 if (rdev->mec.hpd_eop_obj) { 4448 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); 4449 if (unlikely(r != 0)) 4450 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r); 4451 radeon_bo_unpin(rdev->mec.hpd_eop_obj); 4452 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); 4453 4454 radeon_bo_unref(&rdev->mec.hpd_eop_obj); 4455 rdev->mec.hpd_eop_obj = NULL; 4456 } 4457 } 4458 4459 #define MEC_HPD_SIZE 2048 4460 4461 static int cik_mec_init(struct radeon_device *rdev) 4462 { 4463 int r; 4464 u32 *hpd; 4465 4466 /* 4467 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total 4468 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total 4469 */ 4470 if (rdev->family == CHIP_KAVERI) 4471 rdev->mec.num_mec = 2; 4472 else 4473 rdev->mec.num_mec = 1; 4474 rdev->mec.num_pipe = 4; 4475 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8; 4476 4477 if (rdev->mec.hpd_eop_obj == NULL) { 4478 r = radeon_bo_create(rdev, 4479 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2, 4480 PAGE_SIZE, true, 4481 RADEON_GEM_DOMAIN_GTT, 0, NULL, 4482 &rdev->mec.hpd_eop_obj); 4483 if (r) { 4484 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r); 4485 return r; 4486 } 4487 } 4488 4489 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); 4490 if (unlikely(r != 0)) { 4491 cik_mec_fini(rdev); 4492 return r; 4493 } 4494 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT, 4495 &rdev->mec.hpd_eop_gpu_addr); 4496 if (r) { 4497 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r); 4498 cik_mec_fini(rdev); 4499 return r; 4500 } 4501 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd); 4502 if (r) { 4503 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r); 4504 cik_mec_fini(rdev); 4505 return r; 4506 } 4507 4508 /* clear memory. Not sure if this is required or not */ 4509 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2); 4510 4511 radeon_bo_kunmap(rdev->mec.hpd_eop_obj); 4512 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); 4513 4514 return 0; 4515 } 4516 4517 struct hqd_registers 4518 { 4519 u32 cp_mqd_base_addr; 4520 u32 cp_mqd_base_addr_hi; 4521 u32 cp_hqd_active; 4522 u32 cp_hqd_vmid; 4523 u32 cp_hqd_persistent_state; 4524 u32 cp_hqd_pipe_priority; 4525 u32 cp_hqd_queue_priority; 4526 u32 cp_hqd_quantum; 4527 u32 cp_hqd_pq_base; 4528 u32 cp_hqd_pq_base_hi; 4529 u32 cp_hqd_pq_rptr; 4530 u32 cp_hqd_pq_rptr_report_addr; 4531 u32 cp_hqd_pq_rptr_report_addr_hi; 4532 u32 cp_hqd_pq_wptr_poll_addr; 4533 u32 cp_hqd_pq_wptr_poll_addr_hi; 4534 u32 cp_hqd_pq_doorbell_control; 4535 u32 cp_hqd_pq_wptr; 4536 u32 cp_hqd_pq_control; 4537 u32 cp_hqd_ib_base_addr; 4538 u32 cp_hqd_ib_base_addr_hi; 4539 u32 cp_hqd_ib_rptr; 4540 u32 cp_hqd_ib_control; 4541 u32 cp_hqd_iq_timer; 4542 u32 cp_hqd_iq_rptr; 4543 u32 cp_hqd_dequeue_request; 4544 u32 cp_hqd_dma_offload; 4545 u32 cp_hqd_sema_cmd; 4546 u32 cp_hqd_msg_type; 4547 u32 cp_hqd_atomic0_preop_lo; 4548 u32 cp_hqd_atomic0_preop_hi; 4549 u32 cp_hqd_atomic1_preop_lo; 4550 u32 cp_hqd_atomic1_preop_hi; 4551 u32 cp_hqd_hq_scheduler0; 4552 u32 cp_hqd_hq_scheduler1; 4553 u32 cp_mqd_control; 4554 }; 4555 4556 struct bonaire_mqd 4557 { 4558 u32 header; 4559 u32 dispatch_initiator; 4560 u32 dimensions[3]; 4561 u32 start_idx[3]; 4562 u32 num_threads[3]; 4563 u32 pipeline_stat_enable; 4564 u32 perf_counter_enable; 4565 u32 pgm[2]; 4566 u32 tba[2]; 4567 u32 tma[2]; 4568 u32 pgm_rsrc[2]; 4569 u32 vmid; 4570 u32 resource_limits; 4571 u32 static_thread_mgmt01[2]; 4572 u32 tmp_ring_size; 4573 u32 static_thread_mgmt23[2]; 4574 u32 restart[3]; 4575 u32 thread_trace_enable; 4576 u32 reserved1; 4577 u32 user_data[16]; 4578 u32 vgtcs_invoke_count[2]; 4579 struct hqd_registers queue_state; 4580 u32 dequeue_cntr; 4581 u32 interrupt_queue[64]; 4582 }; 4583 4584 /** 4585 * cik_cp_compute_resume - setup the compute queue registers 4586 * 4587 * @rdev: radeon_device pointer 4588 * 4589 * Program the compute queues and test them to make sure they 4590 * are working. 4591 * Returns 0 for success, error for failure. 4592 */ 4593 static int cik_cp_compute_resume(struct radeon_device *rdev) 4594 { 4595 int r, i, j, idx; 4596 u32 tmp; 4597 bool use_doorbell = true; 4598 u64 hqd_gpu_addr; 4599 u64 mqd_gpu_addr; 4600 u64 eop_gpu_addr; 4601 u64 wb_gpu_addr; 4602 u32 *buf; 4603 struct bonaire_mqd *mqd; 4604 4605 r = cik_cp_compute_start(rdev); 4606 if (r) 4607 return r; 4608 4609 /* fix up chicken bits */ 4610 tmp = RREG32(CP_CPF_DEBUG); 4611 tmp |= (1 << 23); 4612 WREG32(CP_CPF_DEBUG, tmp); 4613 4614 /* init the pipes */ 4615 mutex_lock(&rdev->srbm_mutex); 4616 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) { 4617 int me = (i < 4) ? 1 : 2; 4618 int pipe = (i < 4) ? i : (i - 4); 4619 4620 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2); 4621 4622 cik_srbm_select(rdev, me, pipe, 0, 0); 4623 4624 /* write the EOP addr */ 4625 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8); 4626 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8); 4627 4628 /* set the VMID assigned */ 4629 WREG32(CP_HPD_EOP_VMID, 0); 4630 4631 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4632 tmp = RREG32(CP_HPD_EOP_CONTROL); 4633 tmp &= ~EOP_SIZE_MASK; 4634 tmp |= order_base_2(MEC_HPD_SIZE / 8); 4635 WREG32(CP_HPD_EOP_CONTROL, tmp); 4636 } 4637 cik_srbm_select(rdev, 0, 0, 0, 0); 4638 mutex_unlock(&rdev->srbm_mutex); 4639 4640 /* init the queues. Just two for now. */ 4641 for (i = 0; i < 2; i++) { 4642 if (i == 0) 4643 idx = CAYMAN_RING_TYPE_CP1_INDEX; 4644 else 4645 idx = CAYMAN_RING_TYPE_CP2_INDEX; 4646 4647 if (rdev->ring[idx].mqd_obj == NULL) { 4648 r = radeon_bo_create(rdev, 4649 sizeof(struct bonaire_mqd), 4650 PAGE_SIZE, true, 4651 RADEON_GEM_DOMAIN_GTT, 0, NULL, 4652 &rdev->ring[idx].mqd_obj); 4653 if (r) { 4654 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r); 4655 return r; 4656 } 4657 } 4658 4659 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); 4660 if (unlikely(r != 0)) { 4661 cik_cp_compute_fini(rdev); 4662 return r; 4663 } 4664 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT, 4665 &mqd_gpu_addr); 4666 if (r) { 4667 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r); 4668 cik_cp_compute_fini(rdev); 4669 return r; 4670 } 4671 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf); 4672 if (r) { 4673 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r); 4674 cik_cp_compute_fini(rdev); 4675 return r; 4676 } 4677 4678 /* init the mqd struct */ 4679 memset(buf, 0, sizeof(struct bonaire_mqd)); 4680 4681 mqd = (struct bonaire_mqd *)buf; 4682 mqd->header = 0xC0310800; 4683 mqd->static_thread_mgmt01[0] = 0xffffffff; 4684 mqd->static_thread_mgmt01[1] = 0xffffffff; 4685 mqd->static_thread_mgmt23[0] = 0xffffffff; 4686 mqd->static_thread_mgmt23[1] = 0xffffffff; 4687 4688 mutex_lock(&rdev->srbm_mutex); 4689 cik_srbm_select(rdev, rdev->ring[idx].me, 4690 rdev->ring[idx].pipe, 4691 rdev->ring[idx].queue, 0); 4692 4693 /* disable wptr polling */ 4694 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL); 4695 tmp &= ~WPTR_POLL_EN; 4696 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp); 4697 4698 /* enable doorbell? */ 4699 mqd->queue_state.cp_hqd_pq_doorbell_control = 4700 RREG32(CP_HQD_PQ_DOORBELL_CONTROL); 4701 if (use_doorbell) 4702 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; 4703 else 4704 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN; 4705 WREG32(CP_HQD_PQ_DOORBELL_CONTROL, 4706 mqd->queue_state.cp_hqd_pq_doorbell_control); 4707 4708 /* disable the queue if it's active */ 4709 mqd->queue_state.cp_hqd_dequeue_request = 0; 4710 mqd->queue_state.cp_hqd_pq_rptr = 0; 4711 mqd->queue_state.cp_hqd_pq_wptr= 0; 4712 if (RREG32(CP_HQD_ACTIVE) & 1) { 4713 WREG32(CP_HQD_DEQUEUE_REQUEST, 1); 4714 for (j = 0; j < rdev->usec_timeout; j++) { 4715 if (!(RREG32(CP_HQD_ACTIVE) & 1)) 4716 break; 4717 udelay(1); 4718 } 4719 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request); 4720 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr); 4721 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); 4722 } 4723 4724 /* set the pointer to the MQD */ 4725 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc; 4726 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr); 4727 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr); 4728 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi); 4729 /* set MQD vmid to 0 */ 4730 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL); 4731 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK; 4732 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control); 4733 4734 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4735 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8; 4736 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr; 4737 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4738 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base); 4739 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi); 4740 4741 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4742 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL); 4743 mqd->queue_state.cp_hqd_pq_control &= 4744 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK); 4745 4746 mqd->queue_state.cp_hqd_pq_control |= 4747 order_base_2(rdev->ring[idx].ring_size / 8); 4748 mqd->queue_state.cp_hqd_pq_control |= 4749 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8); 4750 #ifdef __BIG_ENDIAN 4751 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT; 4752 #endif 4753 mqd->queue_state.cp_hqd_pq_control &= 4754 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE); 4755 mqd->queue_state.cp_hqd_pq_control |= 4756 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */ 4757 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control); 4758 4759 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */ 4760 if (i == 0) 4761 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET; 4762 else 4763 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET; 4764 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc; 4765 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4766 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr); 4767 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI, 4768 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi); 4769 4770 /* set the wb address wether it's enabled or not */ 4771 if (i == 0) 4772 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET; 4773 else 4774 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET; 4775 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc; 4776 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi = 4777 upper_32_bits(wb_gpu_addr) & 0xffff; 4778 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR, 4779 mqd->queue_state.cp_hqd_pq_rptr_report_addr); 4780 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI, 4781 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi); 4782 4783 /* enable the doorbell if requested */ 4784 if (use_doorbell) { 4785 mqd->queue_state.cp_hqd_pq_doorbell_control = 4786 RREG32(CP_HQD_PQ_DOORBELL_CONTROL); 4787 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK; 4788 mqd->queue_state.cp_hqd_pq_doorbell_control |= 4789 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index); 4790 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; 4791 mqd->queue_state.cp_hqd_pq_doorbell_control &= 4792 ~(DOORBELL_SOURCE | DOORBELL_HIT); 4793 4794 } else { 4795 mqd->queue_state.cp_hqd_pq_doorbell_control = 0; 4796 } 4797 WREG32(CP_HQD_PQ_DOORBELL_CONTROL, 4798 mqd->queue_state.cp_hqd_pq_doorbell_control); 4799 4800 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4801 rdev->ring[idx].wptr = 0; 4802 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr; 4803 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); 4804 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR); 4805 4806 /* set the vmid for the queue */ 4807 mqd->queue_state.cp_hqd_vmid = 0; 4808 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid); 4809 4810 /* activate the queue */ 4811 mqd->queue_state.cp_hqd_active = 1; 4812 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active); 4813 4814 cik_srbm_select(rdev, 0, 0, 0, 0); 4815 mutex_unlock(&rdev->srbm_mutex); 4816 4817 radeon_bo_kunmap(rdev->ring[idx].mqd_obj); 4818 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); 4819 4820 rdev->ring[idx].ready = true; 4821 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]); 4822 if (r) 4823 rdev->ring[idx].ready = false; 4824 } 4825 4826 return 0; 4827 } 4828 4829 static void cik_cp_enable(struct radeon_device *rdev, bool enable) 4830 { 4831 cik_cp_gfx_enable(rdev, enable); 4832 cik_cp_compute_enable(rdev, enable); 4833 } 4834 4835 static int cik_cp_load_microcode(struct radeon_device *rdev) 4836 { 4837 int r; 4838 4839 r = cik_cp_gfx_load_microcode(rdev); 4840 if (r) 4841 return r; 4842 r = cik_cp_compute_load_microcode(rdev); 4843 if (r) 4844 return r; 4845 4846 return 0; 4847 } 4848 4849 static void cik_cp_fini(struct radeon_device *rdev) 4850 { 4851 cik_cp_gfx_fini(rdev); 4852 cik_cp_compute_fini(rdev); 4853 } 4854 4855 static int cik_cp_resume(struct radeon_device *rdev) 4856 { 4857 int r; 4858 4859 cik_enable_gui_idle_interrupt(rdev, false); 4860 4861 r = cik_cp_load_microcode(rdev); 4862 if (r) 4863 return r; 4864 4865 r = cik_cp_gfx_resume(rdev); 4866 if (r) 4867 return r; 4868 r = cik_cp_compute_resume(rdev); 4869 if (r) 4870 return r; 4871 4872 cik_enable_gui_idle_interrupt(rdev, true); 4873 4874 return 0; 4875 } 4876 4877 static void cik_print_gpu_status_regs(struct radeon_device *rdev) 4878 { 4879 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", 4880 RREG32(GRBM_STATUS)); 4881 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n", 4882 RREG32(GRBM_STATUS2)); 4883 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", 4884 RREG32(GRBM_STATUS_SE0)); 4885 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", 4886 RREG32(GRBM_STATUS_SE1)); 4887 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n", 4888 RREG32(GRBM_STATUS_SE2)); 4889 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n", 4890 RREG32(GRBM_STATUS_SE3)); 4891 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", 4892 RREG32(SRBM_STATUS)); 4893 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n", 4894 RREG32(SRBM_STATUS2)); 4895 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n", 4896 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET)); 4897 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n", 4898 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET)); 4899 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT)); 4900 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n", 4901 RREG32(CP_STALLED_STAT1)); 4902 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n", 4903 RREG32(CP_STALLED_STAT2)); 4904 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n", 4905 RREG32(CP_STALLED_STAT3)); 4906 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n", 4907 RREG32(CP_CPF_BUSY_STAT)); 4908 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n", 4909 RREG32(CP_CPF_STALLED_STAT1)); 4910 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS)); 4911 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT)); 4912 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n", 4913 RREG32(CP_CPC_STALLED_STAT1)); 4914 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS)); 4915 } 4916 4917 /** 4918 * cik_gpu_check_soft_reset - check which blocks are busy 4919 * 4920 * @rdev: radeon_device pointer 4921 * 4922 * Check which blocks are busy and return the relevant reset 4923 * mask to be used by cik_gpu_soft_reset(). 4924 * Returns a mask of the blocks to be reset. 4925 */ 4926 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev) 4927 { 4928 u32 reset_mask = 0; 4929 u32 tmp; 4930 4931 /* GRBM_STATUS */ 4932 tmp = RREG32(GRBM_STATUS); 4933 if (tmp & (PA_BUSY | SC_BUSY | 4934 BCI_BUSY | SX_BUSY | 4935 TA_BUSY | VGT_BUSY | 4936 DB_BUSY | CB_BUSY | 4937 GDS_BUSY | SPI_BUSY | 4938 IA_BUSY | IA_BUSY_NO_DMA)) 4939 reset_mask |= RADEON_RESET_GFX; 4940 4941 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY)) 4942 reset_mask |= RADEON_RESET_CP; 4943 4944 /* GRBM_STATUS2 */ 4945 tmp = RREG32(GRBM_STATUS2); 4946 if (tmp & RLC_BUSY) 4947 reset_mask |= RADEON_RESET_RLC; 4948 4949 /* SDMA0_STATUS_REG */ 4950 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET); 4951 if (!(tmp & SDMA_IDLE)) 4952 reset_mask |= RADEON_RESET_DMA; 4953 4954 /* SDMA1_STATUS_REG */ 4955 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET); 4956 if (!(tmp & SDMA_IDLE)) 4957 reset_mask |= RADEON_RESET_DMA1; 4958 4959 /* SRBM_STATUS2 */ 4960 tmp = RREG32(SRBM_STATUS2); 4961 if (tmp & SDMA_BUSY) 4962 reset_mask |= RADEON_RESET_DMA; 4963 4964 if (tmp & SDMA1_BUSY) 4965 reset_mask |= RADEON_RESET_DMA1; 4966 4967 /* SRBM_STATUS */ 4968 tmp = RREG32(SRBM_STATUS); 4969 4970 if (tmp & IH_BUSY) 4971 reset_mask |= RADEON_RESET_IH; 4972 4973 if (tmp & SEM_BUSY) 4974 reset_mask |= RADEON_RESET_SEM; 4975 4976 if (tmp & GRBM_RQ_PENDING) 4977 reset_mask |= RADEON_RESET_GRBM; 4978 4979 if (tmp & VMC_BUSY) 4980 reset_mask |= RADEON_RESET_VMC; 4981 4982 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 4983 MCC_BUSY | MCD_BUSY)) 4984 reset_mask |= RADEON_RESET_MC; 4985 4986 if (evergreen_is_display_hung(rdev)) 4987 reset_mask |= RADEON_RESET_DISPLAY; 4988 4989 /* Skip MC reset as it's mostly likely not hung, just busy */ 4990 if (reset_mask & RADEON_RESET_MC) { 4991 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 4992 reset_mask &= ~RADEON_RESET_MC; 4993 } 4994 4995 return reset_mask; 4996 } 4997 4998 /** 4999 * cik_gpu_soft_reset - soft reset GPU 5000 * 5001 * @rdev: radeon_device pointer 5002 * @reset_mask: mask of which blocks to reset 5003 * 5004 * Soft reset the blocks specified in @reset_mask. 5005 */ 5006 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 5007 { 5008 struct evergreen_mc_save save; 5009 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 5010 u32 tmp; 5011 5012 if (reset_mask == 0) 5013 return; 5014 5015 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 5016 5017 cik_print_gpu_status_regs(rdev); 5018 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 5019 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); 5020 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 5021 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); 5022 5023 /* disable CG/PG */ 5024 cik_fini_pg(rdev); 5025 cik_fini_cg(rdev); 5026 5027 /* stop the rlc */ 5028 cik_rlc_stop(rdev); 5029 5030 /* Disable GFX parsing/prefetching */ 5031 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 5032 5033 /* Disable MEC parsing/prefetching */ 5034 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); 5035 5036 if (reset_mask & RADEON_RESET_DMA) { 5037 /* sdma0 */ 5038 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); 5039 tmp |= SDMA_HALT; 5040 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); 5041 } 5042 if (reset_mask & RADEON_RESET_DMA1) { 5043 /* sdma1 */ 5044 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); 5045 tmp |= SDMA_HALT; 5046 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); 5047 } 5048 5049 evergreen_mc_stop(rdev, &save); 5050 if (evergreen_mc_wait_for_idle(rdev)) { 5051 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5052 } 5053 5054 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) 5055 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX; 5056 5057 if (reset_mask & RADEON_RESET_CP) { 5058 grbm_soft_reset |= SOFT_RESET_CP; 5059 5060 srbm_soft_reset |= SOFT_RESET_GRBM; 5061 } 5062 5063 if (reset_mask & RADEON_RESET_DMA) 5064 srbm_soft_reset |= SOFT_RESET_SDMA; 5065 5066 if (reset_mask & RADEON_RESET_DMA1) 5067 srbm_soft_reset |= SOFT_RESET_SDMA1; 5068 5069 if (reset_mask & RADEON_RESET_DISPLAY) 5070 srbm_soft_reset |= SOFT_RESET_DC; 5071 5072 if (reset_mask & RADEON_RESET_RLC) 5073 grbm_soft_reset |= SOFT_RESET_RLC; 5074 5075 if (reset_mask & RADEON_RESET_SEM) 5076 srbm_soft_reset |= SOFT_RESET_SEM; 5077 5078 if (reset_mask & RADEON_RESET_IH) 5079 srbm_soft_reset |= SOFT_RESET_IH; 5080 5081 if (reset_mask & RADEON_RESET_GRBM) 5082 srbm_soft_reset |= SOFT_RESET_GRBM; 5083 5084 if (reset_mask & RADEON_RESET_VMC) 5085 srbm_soft_reset |= SOFT_RESET_VMC; 5086 5087 if (!(rdev->flags & RADEON_IS_IGP)) { 5088 if (reset_mask & RADEON_RESET_MC) 5089 srbm_soft_reset |= SOFT_RESET_MC; 5090 } 5091 5092 if (grbm_soft_reset) { 5093 tmp = RREG32(GRBM_SOFT_RESET); 5094 tmp |= grbm_soft_reset; 5095 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 5096 WREG32(GRBM_SOFT_RESET, tmp); 5097 tmp = RREG32(GRBM_SOFT_RESET); 5098 5099 udelay(50); 5100 5101 tmp &= ~grbm_soft_reset; 5102 WREG32(GRBM_SOFT_RESET, tmp); 5103 tmp = RREG32(GRBM_SOFT_RESET); 5104 } 5105 5106 if (srbm_soft_reset) { 5107 tmp = RREG32(SRBM_SOFT_RESET); 5108 tmp |= srbm_soft_reset; 5109 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 5110 WREG32(SRBM_SOFT_RESET, tmp); 5111 tmp = RREG32(SRBM_SOFT_RESET); 5112 5113 udelay(50); 5114 5115 tmp &= ~srbm_soft_reset; 5116 WREG32(SRBM_SOFT_RESET, tmp); 5117 tmp = RREG32(SRBM_SOFT_RESET); 5118 } 5119 5120 /* Wait a little for things to settle down */ 5121 udelay(50); 5122 5123 evergreen_mc_resume(rdev, &save); 5124 udelay(50); 5125 5126 cik_print_gpu_status_regs(rdev); 5127 } 5128 5129 struct kv_reset_save_regs { 5130 u32 gmcon_reng_execute; 5131 u32 gmcon_misc; 5132 u32 gmcon_misc3; 5133 }; 5134 5135 static void kv_save_regs_for_reset(struct radeon_device *rdev, 5136 struct kv_reset_save_regs *save) 5137 { 5138 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE); 5139 save->gmcon_misc = RREG32(GMCON_MISC); 5140 save->gmcon_misc3 = RREG32(GMCON_MISC3); 5141 5142 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP); 5143 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE | 5144 STCTRL_STUTTER_EN)); 5145 } 5146 5147 static void kv_restore_regs_for_reset(struct radeon_device *rdev, 5148 struct kv_reset_save_regs *save) 5149 { 5150 int i; 5151 5152 WREG32(GMCON_PGFSM_WRITE, 0); 5153 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff); 5154 5155 for (i = 0; i < 5; i++) 5156 WREG32(GMCON_PGFSM_WRITE, 0); 5157 5158 WREG32(GMCON_PGFSM_WRITE, 0); 5159 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff); 5160 5161 for (i = 0; i < 5; i++) 5162 WREG32(GMCON_PGFSM_WRITE, 0); 5163 5164 WREG32(GMCON_PGFSM_WRITE, 0x210000); 5165 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff); 5166 5167 for (i = 0; i < 5; i++) 5168 WREG32(GMCON_PGFSM_WRITE, 0); 5169 5170 WREG32(GMCON_PGFSM_WRITE, 0x21003); 5171 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff); 5172 5173 for (i = 0; i < 5; i++) 5174 WREG32(GMCON_PGFSM_WRITE, 0); 5175 5176 WREG32(GMCON_PGFSM_WRITE, 0x2b00); 5177 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff); 5178 5179 for (i = 0; i < 5; i++) 5180 WREG32(GMCON_PGFSM_WRITE, 0); 5181 5182 WREG32(GMCON_PGFSM_WRITE, 0); 5183 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff); 5184 5185 for (i = 0; i < 5; i++) 5186 WREG32(GMCON_PGFSM_WRITE, 0); 5187 5188 WREG32(GMCON_PGFSM_WRITE, 0x420000); 5189 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff); 5190 5191 for (i = 0; i < 5; i++) 5192 WREG32(GMCON_PGFSM_WRITE, 0); 5193 5194 WREG32(GMCON_PGFSM_WRITE, 0x120202); 5195 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff); 5196 5197 for (i = 0; i < 5; i++) 5198 WREG32(GMCON_PGFSM_WRITE, 0); 5199 5200 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36); 5201 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff); 5202 5203 for (i = 0; i < 5; i++) 5204 WREG32(GMCON_PGFSM_WRITE, 0); 5205 5206 WREG32(GMCON_PGFSM_WRITE, 0x373f3e); 5207 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff); 5208 5209 for (i = 0; i < 5; i++) 5210 WREG32(GMCON_PGFSM_WRITE, 0); 5211 5212 WREG32(GMCON_PGFSM_WRITE, 0x3e1332); 5213 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff); 5214 5215 WREG32(GMCON_MISC3, save->gmcon_misc3); 5216 WREG32(GMCON_MISC, save->gmcon_misc); 5217 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute); 5218 } 5219 5220 static void cik_gpu_pci_config_reset(struct radeon_device *rdev) 5221 { 5222 struct evergreen_mc_save save; 5223 struct kv_reset_save_regs kv_save = { 0 }; 5224 u32 tmp, i; 5225 5226 dev_info(rdev->dev, "GPU pci config reset\n"); 5227 5228 /* disable dpm? */ 5229 5230 /* disable cg/pg */ 5231 cik_fini_pg(rdev); 5232 cik_fini_cg(rdev); 5233 5234 /* Disable GFX parsing/prefetching */ 5235 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 5236 5237 /* Disable MEC parsing/prefetching */ 5238 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); 5239 5240 /* sdma0 */ 5241 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); 5242 tmp |= SDMA_HALT; 5243 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); 5244 /* sdma1 */ 5245 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); 5246 tmp |= SDMA_HALT; 5247 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); 5248 /* XXX other engines? */ 5249 5250 /* halt the rlc, disable cp internal ints */ 5251 cik_rlc_stop(rdev); 5252 5253 udelay(50); 5254 5255 /* disable mem access */ 5256 evergreen_mc_stop(rdev, &save); 5257 if (evergreen_mc_wait_for_idle(rdev)) { 5258 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 5259 } 5260 5261 if (rdev->flags & RADEON_IS_IGP) 5262 kv_save_regs_for_reset(rdev, &kv_save); 5263 5264 /* disable BM */ 5265 pci_disable_busmaster(rdev->pdev->dev.bsddev); 5266 /* reset */ 5267 radeon_pci_config_reset(rdev); 5268 5269 udelay(100); 5270 5271 /* wait for asic to come out of reset */ 5272 for (i = 0; i < rdev->usec_timeout; i++) { 5273 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 5274 break; 5275 udelay(1); 5276 } 5277 5278 /* does asic init need to be run first??? */ 5279 if (rdev->flags & RADEON_IS_IGP) 5280 kv_restore_regs_for_reset(rdev, &kv_save); 5281 } 5282 5283 /** 5284 * cik_asic_reset - soft reset GPU 5285 * 5286 * @rdev: radeon_device pointer 5287 * @hard: force hard reset 5288 * 5289 * Look up which blocks are hung and attempt 5290 * to reset them. 5291 * Returns 0 for success. 5292 */ 5293 int cik_asic_reset(struct radeon_device *rdev, bool hard) 5294 { 5295 u32 reset_mask; 5296 5297 if (hard) { 5298 cik_gpu_pci_config_reset(rdev); 5299 return 0; 5300 } 5301 5302 reset_mask = cik_gpu_check_soft_reset(rdev); 5303 5304 if (reset_mask) 5305 r600_set_bios_scratch_engine_hung(rdev, true); 5306 5307 /* try soft reset */ 5308 cik_gpu_soft_reset(rdev, reset_mask); 5309 5310 reset_mask = cik_gpu_check_soft_reset(rdev); 5311 5312 /* try pci config reset */ 5313 if (reset_mask && radeon_hard_reset) 5314 cik_gpu_pci_config_reset(rdev); 5315 5316 reset_mask = cik_gpu_check_soft_reset(rdev); 5317 5318 if (!reset_mask) 5319 r600_set_bios_scratch_engine_hung(rdev, false); 5320 5321 return 0; 5322 } 5323 5324 /** 5325 * cik_gfx_is_lockup - check if the 3D engine is locked up 5326 * 5327 * @rdev: radeon_device pointer 5328 * @ring: radeon_ring structure holding ring information 5329 * 5330 * Check if the 3D engine is locked up (CIK). 5331 * Returns true if the engine is locked, false if not. 5332 */ 5333 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 5334 { 5335 u32 reset_mask = cik_gpu_check_soft_reset(rdev); 5336 5337 if (!(reset_mask & (RADEON_RESET_GFX | 5338 RADEON_RESET_COMPUTE | 5339 RADEON_RESET_CP))) { 5340 radeon_ring_lockup_update(rdev, ring); 5341 return false; 5342 } 5343 return radeon_ring_test_lockup(rdev, ring); 5344 } 5345 5346 /* MC */ 5347 /** 5348 * cik_mc_program - program the GPU memory controller 5349 * 5350 * @rdev: radeon_device pointer 5351 * 5352 * Set the location of vram, gart, and AGP in the GPU's 5353 * physical address space (CIK). 5354 */ 5355 static void cik_mc_program(struct radeon_device *rdev) 5356 { 5357 struct evergreen_mc_save save; 5358 u32 tmp; 5359 int i, j; 5360 5361 /* Initialize HDP */ 5362 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 5363 WREG32((0x2c14 + j), 0x00000000); 5364 WREG32((0x2c18 + j), 0x00000000); 5365 WREG32((0x2c1c + j), 0x00000000); 5366 WREG32((0x2c20 + j), 0x00000000); 5367 WREG32((0x2c24 + j), 0x00000000); 5368 } 5369 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 5370 5371 evergreen_mc_stop(rdev, &save); 5372 if (radeon_mc_wait_for_idle(rdev)) { 5373 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5374 } 5375 /* Lockout access through VGA aperture*/ 5376 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 5377 /* Update configuration */ 5378 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 5379 rdev->mc.vram_start >> 12); 5380 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 5381 rdev->mc.vram_end >> 12); 5382 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 5383 rdev->vram_scratch.gpu_addr >> 12); 5384 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 5385 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 5386 WREG32(MC_VM_FB_LOCATION, tmp); 5387 /* XXX double check these! */ 5388 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 5389 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 5390 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 5391 WREG32(MC_VM_AGP_BASE, 0); 5392 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 5393 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 5394 if (radeon_mc_wait_for_idle(rdev)) { 5395 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5396 } 5397 evergreen_mc_resume(rdev, &save); 5398 /* we need to own VRAM, so turn off the VGA renderer here 5399 * to stop it overwriting our objects */ 5400 rv515_vga_render_disable(rdev); 5401 } 5402 5403 /** 5404 * cik_mc_init - initialize the memory controller driver params 5405 * 5406 * @rdev: radeon_device pointer 5407 * 5408 * Look up the amount of vram, vram width, and decide how to place 5409 * vram and gart within the GPU's physical address space (CIK). 5410 * Returns 0 for success. 5411 */ 5412 static int cik_mc_init(struct radeon_device *rdev) 5413 { 5414 u32 tmp; 5415 int chansize, numchan; 5416 5417 /* Get VRAM informations */ 5418 rdev->mc.vram_is_ddr = true; 5419 tmp = RREG32(MC_ARB_RAMCFG); 5420 if (tmp & CHANSIZE_MASK) { 5421 chansize = 64; 5422 } else { 5423 chansize = 32; 5424 } 5425 tmp = RREG32(MC_SHARED_CHMAP); 5426 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 5427 case 0: 5428 default: 5429 numchan = 1; 5430 break; 5431 case 1: 5432 numchan = 2; 5433 break; 5434 case 2: 5435 numchan = 4; 5436 break; 5437 case 3: 5438 numchan = 8; 5439 break; 5440 case 4: 5441 numchan = 3; 5442 break; 5443 case 5: 5444 numchan = 6; 5445 break; 5446 case 6: 5447 numchan = 10; 5448 break; 5449 case 7: 5450 numchan = 12; 5451 break; 5452 case 8: 5453 numchan = 16; 5454 break; 5455 } 5456 rdev->mc.vram_width = numchan * chansize; 5457 /* Could aper size report 0 ? */ 5458 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 5459 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 5460 /* size in MB on si */ 5461 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 5462 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 5463 rdev->mc.visible_vram_size = rdev->mc.aper_size; 5464 si_vram_gtt_location(rdev, &rdev->mc); 5465 radeon_update_bandwidth_info(rdev); 5466 5467 return 0; 5468 } 5469 5470 /* 5471 * GART 5472 * VMID 0 is the physical GPU addresses as used by the kernel. 5473 * VMIDs 1-15 are used for userspace clients and are handled 5474 * by the radeon vm/hsa code. 5475 */ 5476 /** 5477 * cik_pcie_gart_tlb_flush - gart tlb flush callback 5478 * 5479 * @rdev: radeon_device pointer 5480 * 5481 * Flush the TLB for the VMID 0 page table (CIK). 5482 */ 5483 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev) 5484 { 5485 /* flush hdp cache */ 5486 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0); 5487 5488 /* bits 0-15 are the VM contexts0-15 */ 5489 WREG32(VM_INVALIDATE_REQUEST, 0x1); 5490 } 5491 5492 static void cik_pcie_init_compute_vmid(struct radeon_device *rdev) 5493 { 5494 int i; 5495 uint32_t sh_mem_bases, sh_mem_config; 5496 5497 sh_mem_bases = 0x6000 | 0x6000 << 16; 5498 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED); 5499 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED); 5500 5501 mutex_lock(&rdev->srbm_mutex); 5502 for (i = 8; i < 16; i++) { 5503 cik_srbm_select(rdev, 0, 0, 0, i); 5504 /* CP and shaders */ 5505 WREG32(SH_MEM_CONFIG, sh_mem_config); 5506 WREG32(SH_MEM_APE1_BASE, 1); 5507 WREG32(SH_MEM_APE1_LIMIT, 0); 5508 WREG32(SH_MEM_BASES, sh_mem_bases); 5509 } 5510 cik_srbm_select(rdev, 0, 0, 0, 0); 5511 mutex_unlock(&rdev->srbm_mutex); 5512 } 5513 5514 /** 5515 * cik_pcie_gart_enable - gart enable 5516 * 5517 * @rdev: radeon_device pointer 5518 * 5519 * This sets up the TLBs, programs the page tables for VMID0, 5520 * sets up the hw for VMIDs 1-15 which are allocated on 5521 * demand, and sets up the global locations for the LDS, GDS, 5522 * and GPUVM for FSA64 clients (CIK). 5523 * Returns 0 for success, errors for failure. 5524 */ 5525 static int cik_pcie_gart_enable(struct radeon_device *rdev) 5526 { 5527 int r, i; 5528 5529 if (rdev->gart.robj == NULL) { 5530 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 5531 return -EINVAL; 5532 } 5533 r = radeon_gart_table_vram_pin(rdev); 5534 if (r) 5535 return r; 5536 /* Setup TLB control */ 5537 WREG32(MC_VM_MX_L1_TLB_CNTL, 5538 (0xA << 7) | 5539 ENABLE_L1_TLB | 5540 ENABLE_L1_FRAGMENT_PROCESSING | 5541 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 5542 ENABLE_ADVANCED_DRIVER_MODEL | 5543 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 5544 /* Setup L2 cache */ 5545 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | 5546 ENABLE_L2_FRAGMENT_PROCESSING | 5547 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 5548 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 5549 EFFECTIVE_L2_QUEUE_SIZE(7) | 5550 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 5551 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); 5552 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 5553 BANK_SELECT(4) | 5554 L2_CACHE_BIGK_FRAGMENT_SIZE(4)); 5555 /* setup context0 */ 5556 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 5557 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 5558 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 5559 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 5560 (u32)(rdev->dummy_page.addr >> 12)); 5561 WREG32(VM_CONTEXT0_CNTL2, 0); 5562 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 5563 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT)); 5564 5565 WREG32(0x15D4, 0); 5566 WREG32(0x15D8, 0); 5567 WREG32(0x15DC, 0); 5568 5569 /* restore context1-15 */ 5570 /* set vm size, must be a multiple of 4 */ 5571 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); 5572 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1); 5573 for (i = 1; i < 16; i++) { 5574 if (i < 8) 5575 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), 5576 rdev->vm_manager.saved_table_addr[i]); 5577 else 5578 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2), 5579 rdev->vm_manager.saved_table_addr[i]); 5580 } 5581 5582 /* enable context1-15 */ 5583 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, 5584 (u32)(rdev->dummy_page.addr >> 12)); 5585 WREG32(VM_CONTEXT1_CNTL2, 4); 5586 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | 5587 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) | 5588 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5589 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | 5590 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5591 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT | 5592 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT | 5593 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT | 5594 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT | 5595 VALID_PROTECTION_FAULT_ENABLE_DEFAULT | 5596 READ_PROTECTION_FAULT_ENABLE_INTERRUPT | 5597 READ_PROTECTION_FAULT_ENABLE_DEFAULT | 5598 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5599 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT); 5600 5601 if (rdev->family == CHIP_KAVERI) { 5602 u32 tmp = RREG32(CHUB_CONTROL); 5603 tmp &= ~BYPASS_VM; 5604 WREG32(CHUB_CONTROL, tmp); 5605 } 5606 5607 /* XXX SH_MEM regs */ 5608 /* where to put LDS, scratch, GPUVM in FSA64 space */ 5609 mutex_lock(&rdev->srbm_mutex); 5610 for (i = 0; i < 16; i++) { 5611 cik_srbm_select(rdev, 0, 0, 0, i); 5612 /* CP and shaders */ 5613 WREG32(SH_MEM_CONFIG, 0); 5614 WREG32(SH_MEM_APE1_BASE, 1); 5615 WREG32(SH_MEM_APE1_LIMIT, 0); 5616 WREG32(SH_MEM_BASES, 0); 5617 /* SDMA GFX */ 5618 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0); 5619 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0); 5620 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0); 5621 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0); 5622 /* XXX SDMA RLC - todo */ 5623 } 5624 cik_srbm_select(rdev, 0, 0, 0, 0); 5625 mutex_unlock(&rdev->srbm_mutex); 5626 5627 cik_pcie_init_compute_vmid(rdev); 5628 5629 cik_pcie_gart_tlb_flush(rdev); 5630 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 5631 (unsigned)(rdev->mc.gtt_size >> 20), 5632 (unsigned long long)rdev->gart.table_addr); 5633 rdev->gart.ready = true; 5634 return 0; 5635 } 5636 5637 /** 5638 * cik_pcie_gart_disable - gart disable 5639 * 5640 * @rdev: radeon_device pointer 5641 * 5642 * This disables all VM page table (CIK). 5643 */ 5644 static void cik_pcie_gart_disable(struct radeon_device *rdev) 5645 { 5646 unsigned i; 5647 5648 for (i = 1; i < 16; ++i) { 5649 uint32_t reg; 5650 if (i < 8) 5651 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2); 5652 else 5653 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2); 5654 rdev->vm_manager.saved_table_addr[i] = RREG32(reg); 5655 } 5656 5657 /* Disable all tables */ 5658 WREG32(VM_CONTEXT0_CNTL, 0); 5659 WREG32(VM_CONTEXT1_CNTL, 0); 5660 /* Setup TLB control */ 5661 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS | 5662 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 5663 /* Setup L2 cache */ 5664 WREG32(VM_L2_CNTL, 5665 ENABLE_L2_FRAGMENT_PROCESSING | 5666 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 5667 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 5668 EFFECTIVE_L2_QUEUE_SIZE(7) | 5669 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 5670 WREG32(VM_L2_CNTL2, 0); 5671 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 5672 L2_CACHE_BIGK_FRAGMENT_SIZE(6)); 5673 radeon_gart_table_vram_unpin(rdev); 5674 } 5675 5676 /** 5677 * cik_pcie_gart_fini - vm fini callback 5678 * 5679 * @rdev: radeon_device pointer 5680 * 5681 * Tears down the driver GART/VM setup (CIK). 5682 */ 5683 static void cik_pcie_gart_fini(struct radeon_device *rdev) 5684 { 5685 cik_pcie_gart_disable(rdev); 5686 radeon_gart_table_vram_free(rdev); 5687 radeon_gart_fini(rdev); 5688 } 5689 5690 /* vm parser */ 5691 /** 5692 * cik_ib_parse - vm ib_parse callback 5693 * 5694 * @rdev: radeon_device pointer 5695 * @ib: indirect buffer pointer 5696 * 5697 * CIK uses hw IB checking so this is a nop (CIK). 5698 */ 5699 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) 5700 { 5701 return 0; 5702 } 5703 5704 /* 5705 * vm 5706 * VMID 0 is the physical GPU addresses as used by the kernel. 5707 * VMIDs 1-15 are used for userspace clients and are handled 5708 * by the radeon vm/hsa code. 5709 */ 5710 /** 5711 * cik_vm_init - cik vm init callback 5712 * 5713 * @rdev: radeon_device pointer 5714 * 5715 * Inits cik specific vm parameters (number of VMs, base of vram for 5716 * VMIDs 1-15) (CIK). 5717 * Returns 0 for success. 5718 */ 5719 int cik_vm_init(struct radeon_device *rdev) 5720 { 5721 /* number of VMs */ 5722 rdev->vm_manager.nvm = 16; 5723 /* base offset of vram pages */ 5724 if (rdev->flags & RADEON_IS_IGP) { 5725 u64 tmp = RREG32(MC_VM_FB_OFFSET); 5726 tmp <<= 22; 5727 rdev->vm_manager.vram_base_offset = tmp; 5728 } else 5729 rdev->vm_manager.vram_base_offset = 0; 5730 5731 return 0; 5732 } 5733 5734 /** 5735 * cik_vm_fini - cik vm fini callback 5736 * 5737 * @rdev: radeon_device pointer 5738 * 5739 * Tear down any asic specific VM setup (CIK). 5740 */ 5741 void cik_vm_fini(struct radeon_device *rdev) 5742 { 5743 } 5744 5745 /** 5746 * cik_vm_decode_fault - print human readable fault info 5747 * 5748 * @rdev: radeon_device pointer 5749 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value 5750 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value 5751 * 5752 * Print human readable fault information (CIK). 5753 */ 5754 static void cik_vm_decode_fault(struct radeon_device *rdev, 5755 u32 status, u32 addr, u32 mc_client) 5756 { 5757 u32 mc_id; 5758 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; 5759 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; 5760 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff, 5761 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 }; 5762 5763 if (rdev->family == CHIP_HAWAII) 5764 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 5765 else 5766 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 5767 5768 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n", 5769 protections, vmid, addr, 5770 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", 5771 block, mc_client, mc_id); 5772 } 5773 5774 /** 5775 * cik_vm_flush - cik vm flush using the CP 5776 * 5777 * @rdev: radeon_device pointer 5778 * 5779 * Update the page table base and flush the VM TLB 5780 * using the CP (CIK). 5781 */ 5782 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, 5783 unsigned vm_id, uint64_t pd_addr) 5784 { 5785 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX); 5786 5787 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5788 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5789 WRITE_DATA_DST_SEL(0))); 5790 if (vm_id < 8) { 5791 radeon_ring_write(ring, 5792 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2); 5793 } else { 5794 radeon_ring_write(ring, 5795 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2); 5796 } 5797 radeon_ring_write(ring, 0); 5798 radeon_ring_write(ring, pd_addr >> 12); 5799 5800 /* update SH_MEM_* regs */ 5801 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5802 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5803 WRITE_DATA_DST_SEL(0))); 5804 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); 5805 radeon_ring_write(ring, 0); 5806 radeon_ring_write(ring, VMID(vm_id)); 5807 5808 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6)); 5809 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5810 WRITE_DATA_DST_SEL(0))); 5811 radeon_ring_write(ring, SH_MEM_BASES >> 2); 5812 radeon_ring_write(ring, 0); 5813 5814 radeon_ring_write(ring, 0); /* SH_MEM_BASES */ 5815 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */ 5816 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */ 5817 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */ 5818 5819 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5820 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5821 WRITE_DATA_DST_SEL(0))); 5822 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); 5823 radeon_ring_write(ring, 0); 5824 radeon_ring_write(ring, VMID(0)); 5825 5826 /* HDP flush */ 5827 cik_hdp_flush_cp_ring_emit(rdev, ring->idx); 5828 5829 /* bits 0-15 are the VM contexts0-15 */ 5830 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5831 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5832 WRITE_DATA_DST_SEL(0))); 5833 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 5834 radeon_ring_write(ring, 0); 5835 radeon_ring_write(ring, 1 << vm_id); 5836 5837 /* wait for the invalidate to complete */ 5838 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 5839 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */ 5840 WAIT_REG_MEM_FUNCTION(0) | /* always */ 5841 WAIT_REG_MEM_ENGINE(0))); /* me */ 5842 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 5843 radeon_ring_write(ring, 0); 5844 radeon_ring_write(ring, 0); /* ref */ 5845 radeon_ring_write(ring, 0); /* mask */ 5846 radeon_ring_write(ring, 0x20); /* poll interval */ 5847 5848 /* compute doesn't have PFP */ 5849 if (usepfp) { 5850 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5851 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5852 radeon_ring_write(ring, 0x0); 5853 } 5854 } 5855 5856 /* 5857 * RLC 5858 * The RLC is a multi-purpose microengine that handles a 5859 * variety of functions, the most important of which is 5860 * the interrupt controller. 5861 */ 5862 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, 5863 bool enable) 5864 { 5865 u32 tmp = RREG32(CP_INT_CNTL_RING0); 5866 5867 if (enable) 5868 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 5869 else 5870 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 5871 WREG32(CP_INT_CNTL_RING0, tmp); 5872 } 5873 5874 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable) 5875 { 5876 u32 tmp; 5877 5878 tmp = RREG32(RLC_LB_CNTL); 5879 if (enable) 5880 tmp |= LOAD_BALANCE_ENABLE; 5881 else 5882 tmp &= ~LOAD_BALANCE_ENABLE; 5883 WREG32(RLC_LB_CNTL, tmp); 5884 } 5885 5886 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) 5887 { 5888 u32 i, j, k; 5889 u32 mask; 5890 5891 mutex_lock(&rdev->grbm_idx_mutex); 5892 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 5893 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 5894 cik_select_se_sh(rdev, i, j); 5895 for (k = 0; k < rdev->usec_timeout; k++) { 5896 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0) 5897 break; 5898 udelay(1); 5899 } 5900 } 5901 } 5902 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5903 mutex_unlock(&rdev->grbm_idx_mutex); 5904 5905 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY; 5906 for (k = 0; k < rdev->usec_timeout; k++) { 5907 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 5908 break; 5909 udelay(1); 5910 } 5911 } 5912 5913 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc) 5914 { 5915 u32 tmp; 5916 5917 tmp = RREG32(RLC_CNTL); 5918 if (tmp != rlc) 5919 WREG32(RLC_CNTL, rlc); 5920 } 5921 5922 static u32 cik_halt_rlc(struct radeon_device *rdev) 5923 { 5924 u32 data, orig; 5925 5926 orig = data = RREG32(RLC_CNTL); 5927 5928 if (data & RLC_ENABLE) { 5929 u32 i; 5930 5931 data &= ~RLC_ENABLE; 5932 WREG32(RLC_CNTL, data); 5933 5934 for (i = 0; i < rdev->usec_timeout; i++) { 5935 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0) 5936 break; 5937 udelay(1); 5938 } 5939 5940 cik_wait_for_rlc_serdes(rdev); 5941 } 5942 5943 return orig; 5944 } 5945 5946 void cik_enter_rlc_safe_mode(struct radeon_device *rdev) 5947 { 5948 u32 tmp, i, mask; 5949 5950 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE); 5951 WREG32(RLC_GPR_REG2, tmp); 5952 5953 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS; 5954 for (i = 0; i < rdev->usec_timeout; i++) { 5955 if ((RREG32(RLC_GPM_STAT) & mask) == mask) 5956 break; 5957 udelay(1); 5958 } 5959 5960 for (i = 0; i < rdev->usec_timeout; i++) { 5961 if ((RREG32(RLC_GPR_REG2) & REQ) == 0) 5962 break; 5963 udelay(1); 5964 } 5965 } 5966 5967 void cik_exit_rlc_safe_mode(struct radeon_device *rdev) 5968 { 5969 u32 tmp; 5970 5971 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE); 5972 WREG32(RLC_GPR_REG2, tmp); 5973 } 5974 5975 /** 5976 * cik_rlc_stop - stop the RLC ME 5977 * 5978 * @rdev: radeon_device pointer 5979 * 5980 * Halt the RLC ME (MicroEngine) (CIK). 5981 */ 5982 static void cik_rlc_stop(struct radeon_device *rdev) 5983 { 5984 WREG32(RLC_CNTL, 0); 5985 5986 cik_enable_gui_idle_interrupt(rdev, false); 5987 5988 cik_wait_for_rlc_serdes(rdev); 5989 } 5990 5991 /** 5992 * cik_rlc_start - start the RLC ME 5993 * 5994 * @rdev: radeon_device pointer 5995 * 5996 * Unhalt the RLC ME (MicroEngine) (CIK). 5997 */ 5998 static void cik_rlc_start(struct radeon_device *rdev) 5999 { 6000 WREG32(RLC_CNTL, RLC_ENABLE); 6001 6002 cik_enable_gui_idle_interrupt(rdev, true); 6003 6004 udelay(50); 6005 } 6006 6007 /** 6008 * cik_rlc_resume - setup the RLC hw 6009 * 6010 * @rdev: radeon_device pointer 6011 * 6012 * Initialize the RLC registers, load the ucode, 6013 * and start the RLC (CIK). 6014 * Returns 0 for success, -EINVAL if the ucode is not available. 6015 */ 6016 static int cik_rlc_resume(struct radeon_device *rdev) 6017 { 6018 u32 i, size, tmp; 6019 6020 if (!rdev->rlc_fw) 6021 return -EINVAL; 6022 6023 cik_rlc_stop(rdev); 6024 6025 /* disable CG */ 6026 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc; 6027 WREG32(RLC_CGCG_CGLS_CTRL, tmp); 6028 6029 si_rlc_reset(rdev); 6030 6031 cik_init_pg(rdev); 6032 6033 cik_init_cg(rdev); 6034 6035 WREG32(RLC_LB_CNTR_INIT, 0); 6036 WREG32(RLC_LB_CNTR_MAX, 0x00008000); 6037 6038 mutex_lock(&rdev->grbm_idx_mutex); 6039 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6040 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); 6041 WREG32(RLC_LB_PARAMS, 0x00600408); 6042 WREG32(RLC_LB_CNTL, 0x80000004); 6043 mutex_unlock(&rdev->grbm_idx_mutex); 6044 6045 WREG32(RLC_MC_CNTL, 0); 6046 WREG32(RLC_UCODE_CNTL, 0); 6047 6048 if (rdev->new_fw) { 6049 const struct rlc_firmware_header_v1_0 *hdr = 6050 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data; 6051 const __le32 *fw_data = (const __le32 *) 6052 ((const char *)rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6053 6054 radeon_ucode_print_rlc_hdr(&hdr->header); 6055 6056 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 6057 WREG32(RLC_GPM_UCODE_ADDR, 0); 6058 for (i = 0; i < size; i++) 6059 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 6060 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version)); 6061 } else { 6062 const __be32 *fw_data; 6063 6064 switch (rdev->family) { 6065 case CHIP_BONAIRE: 6066 case CHIP_HAWAII: 6067 default: 6068 size = BONAIRE_RLC_UCODE_SIZE; 6069 break; 6070 case CHIP_KAVERI: 6071 size = KV_RLC_UCODE_SIZE; 6072 break; 6073 case CHIP_KABINI: 6074 size = KB_RLC_UCODE_SIZE; 6075 break; 6076 case CHIP_MULLINS: 6077 size = ML_RLC_UCODE_SIZE; 6078 break; 6079 } 6080 6081 fw_data = (const __be32 *)rdev->rlc_fw->data; 6082 WREG32(RLC_GPM_UCODE_ADDR, 0); 6083 for (i = 0; i < size; i++) 6084 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++)); 6085 WREG32(RLC_GPM_UCODE_ADDR, 0); 6086 } 6087 6088 /* XXX - find out what chips support lbpw */ 6089 cik_enable_lbpw(rdev, false); 6090 6091 if (rdev->family == CHIP_BONAIRE) 6092 WREG32(RLC_DRIVER_DMA_STATUS, 0); 6093 6094 cik_rlc_start(rdev); 6095 6096 return 0; 6097 } 6098 6099 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) 6100 { 6101 u32 data, orig, tmp, tmp2; 6102 6103 orig = data = RREG32(RLC_CGCG_CGLS_CTRL); 6104 6105 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { 6106 cik_enable_gui_idle_interrupt(rdev, true); 6107 6108 tmp = cik_halt_rlc(rdev); 6109 6110 mutex_lock(&rdev->grbm_idx_mutex); 6111 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6112 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6113 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6114 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE; 6115 WREG32(RLC_SERDES_WR_CTRL, tmp2); 6116 mutex_unlock(&rdev->grbm_idx_mutex); 6117 6118 cik_update_rlc(rdev, tmp); 6119 6120 data |= CGCG_EN | CGLS_EN; 6121 } else { 6122 cik_enable_gui_idle_interrupt(rdev, false); 6123 6124 RREG32(CB_CGTT_SCLK_CTRL); 6125 RREG32(CB_CGTT_SCLK_CTRL); 6126 RREG32(CB_CGTT_SCLK_CTRL); 6127 RREG32(CB_CGTT_SCLK_CTRL); 6128 6129 data &= ~(CGCG_EN | CGLS_EN); 6130 } 6131 6132 if (orig != data) 6133 WREG32(RLC_CGCG_CGLS_CTRL, data); 6134 6135 } 6136 6137 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) 6138 { 6139 u32 data, orig, tmp = 0; 6140 6141 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { 6142 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) { 6143 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { 6144 orig = data = RREG32(CP_MEM_SLP_CNTL); 6145 data |= CP_MEM_LS_EN; 6146 if (orig != data) 6147 WREG32(CP_MEM_SLP_CNTL, data); 6148 } 6149 } 6150 6151 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 6152 data |= 0x00000001; 6153 data &= 0xfffffffd; 6154 if (orig != data) 6155 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 6156 6157 tmp = cik_halt_rlc(rdev); 6158 6159 mutex_lock(&rdev->grbm_idx_mutex); 6160 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6161 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6162 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6163 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0; 6164 WREG32(RLC_SERDES_WR_CTRL, data); 6165 mutex_unlock(&rdev->grbm_idx_mutex); 6166 6167 cik_update_rlc(rdev, tmp); 6168 6169 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) { 6170 orig = data = RREG32(CGTS_SM_CTRL_REG); 6171 data &= ~SM_MODE_MASK; 6172 data |= SM_MODE(0x2); 6173 data |= SM_MODE_ENABLE; 6174 data &= ~CGTS_OVERRIDE; 6175 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) && 6176 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS)) 6177 data &= ~CGTS_LS_OVERRIDE; 6178 data &= ~ON_MONITOR_ADD_MASK; 6179 data |= ON_MONITOR_ADD_EN; 6180 data |= ON_MONITOR_ADD(0x96); 6181 if (orig != data) 6182 WREG32(CGTS_SM_CTRL_REG, data); 6183 } 6184 } else { 6185 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 6186 data |= 0x00000003; 6187 if (orig != data) 6188 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 6189 6190 data = RREG32(RLC_MEM_SLP_CNTL); 6191 if (data & RLC_MEM_LS_EN) { 6192 data &= ~RLC_MEM_LS_EN; 6193 WREG32(RLC_MEM_SLP_CNTL, data); 6194 } 6195 6196 data = RREG32(CP_MEM_SLP_CNTL); 6197 if (data & CP_MEM_LS_EN) { 6198 data &= ~CP_MEM_LS_EN; 6199 WREG32(CP_MEM_SLP_CNTL, data); 6200 } 6201 6202 orig = data = RREG32(CGTS_SM_CTRL_REG); 6203 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE; 6204 if (orig != data) 6205 WREG32(CGTS_SM_CTRL_REG, data); 6206 6207 tmp = cik_halt_rlc(rdev); 6208 6209 mutex_lock(&rdev->grbm_idx_mutex); 6210 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6211 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6212 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6213 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1; 6214 WREG32(RLC_SERDES_WR_CTRL, data); 6215 mutex_unlock(&rdev->grbm_idx_mutex); 6216 6217 cik_update_rlc(rdev, tmp); 6218 } 6219 } 6220 6221 static const u32 mc_cg_registers[] = 6222 { 6223 MC_HUB_MISC_HUB_CG, 6224 MC_HUB_MISC_SIP_CG, 6225 MC_HUB_MISC_VM_CG, 6226 MC_XPB_CLK_GAT, 6227 ATC_MISC_CG, 6228 MC_CITF_MISC_WR_CG, 6229 MC_CITF_MISC_RD_CG, 6230 MC_CITF_MISC_VM_CG, 6231 VM_L2_CG, 6232 }; 6233 6234 static void cik_enable_mc_ls(struct radeon_device *rdev, 6235 bool enable) 6236 { 6237 int i; 6238 u32 orig, data; 6239 6240 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 6241 orig = data = RREG32(mc_cg_registers[i]); 6242 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) 6243 data |= MC_LS_ENABLE; 6244 else 6245 data &= ~MC_LS_ENABLE; 6246 if (data != orig) 6247 WREG32(mc_cg_registers[i], data); 6248 } 6249 } 6250 6251 static void cik_enable_mc_mgcg(struct radeon_device *rdev, 6252 bool enable) 6253 { 6254 int i; 6255 u32 orig, data; 6256 6257 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 6258 orig = data = RREG32(mc_cg_registers[i]); 6259 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) 6260 data |= MC_CG_ENABLE; 6261 else 6262 data &= ~MC_CG_ENABLE; 6263 if (data != orig) 6264 WREG32(mc_cg_registers[i], data); 6265 } 6266 } 6267 6268 static void cik_enable_sdma_mgcg(struct radeon_device *rdev, 6269 bool enable) 6270 { 6271 u32 orig, data; 6272 6273 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { 6274 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100); 6275 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100); 6276 } else { 6277 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET); 6278 data |= 0xff000000; 6279 if (data != orig) 6280 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data); 6281 6282 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET); 6283 data |= 0xff000000; 6284 if (data != orig) 6285 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data); 6286 } 6287 } 6288 6289 static void cik_enable_sdma_mgls(struct radeon_device *rdev, 6290 bool enable) 6291 { 6292 u32 orig, data; 6293 6294 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) { 6295 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); 6296 data |= 0x100; 6297 if (orig != data) 6298 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); 6299 6300 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); 6301 data |= 0x100; 6302 if (orig != data) 6303 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); 6304 } else { 6305 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); 6306 data &= ~0x100; 6307 if (orig != data) 6308 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); 6309 6310 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); 6311 data &= ~0x100; 6312 if (orig != data) 6313 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); 6314 } 6315 } 6316 6317 static void cik_enable_uvd_mgcg(struct radeon_device *rdev, 6318 bool enable) 6319 { 6320 u32 orig, data; 6321 6322 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { 6323 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 6324 data = 0xfff; 6325 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); 6326 6327 orig = data = RREG32(UVD_CGC_CTRL); 6328 data |= DCM; 6329 if (orig != data) 6330 WREG32(UVD_CGC_CTRL, data); 6331 } else { 6332 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 6333 data &= ~0xfff; 6334 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); 6335 6336 orig = data = RREG32(UVD_CGC_CTRL); 6337 data &= ~DCM; 6338 if (orig != data) 6339 WREG32(UVD_CGC_CTRL, data); 6340 } 6341 } 6342 6343 static void cik_enable_bif_mgls(struct radeon_device *rdev, 6344 bool enable) 6345 { 6346 u32 orig, data; 6347 6348 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); 6349 6350 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) 6351 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | 6352 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN; 6353 else 6354 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN | 6355 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN); 6356 6357 if (orig != data) 6358 WREG32_PCIE_PORT(PCIE_CNTL2, data); 6359 } 6360 6361 static void cik_enable_hdp_mgcg(struct radeon_device *rdev, 6362 bool enable) 6363 { 6364 u32 orig, data; 6365 6366 orig = data = RREG32(HDP_HOST_PATH_CNTL); 6367 6368 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) 6369 data &= ~CLOCK_GATING_DIS; 6370 else 6371 data |= CLOCK_GATING_DIS; 6372 6373 if (orig != data) 6374 WREG32(HDP_HOST_PATH_CNTL, data); 6375 } 6376 6377 static void cik_enable_hdp_ls(struct radeon_device *rdev, 6378 bool enable) 6379 { 6380 u32 orig, data; 6381 6382 orig = data = RREG32(HDP_MEM_POWER_LS); 6383 6384 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) 6385 data |= HDP_LS_ENABLE; 6386 else 6387 data &= ~HDP_LS_ENABLE; 6388 6389 if (orig != data) 6390 WREG32(HDP_MEM_POWER_LS, data); 6391 } 6392 6393 void cik_update_cg(struct radeon_device *rdev, 6394 u32 block, bool enable) 6395 { 6396 6397 if (block & RADEON_CG_BLOCK_GFX) { 6398 cik_enable_gui_idle_interrupt(rdev, false); 6399 /* order matters! */ 6400 if (enable) { 6401 cik_enable_mgcg(rdev, true); 6402 cik_enable_cgcg(rdev, true); 6403 } else { 6404 cik_enable_cgcg(rdev, false); 6405 cik_enable_mgcg(rdev, false); 6406 } 6407 cik_enable_gui_idle_interrupt(rdev, true); 6408 } 6409 6410 if (block & RADEON_CG_BLOCK_MC) { 6411 if (!(rdev->flags & RADEON_IS_IGP)) { 6412 cik_enable_mc_mgcg(rdev, enable); 6413 cik_enable_mc_ls(rdev, enable); 6414 } 6415 } 6416 6417 if (block & RADEON_CG_BLOCK_SDMA) { 6418 cik_enable_sdma_mgcg(rdev, enable); 6419 cik_enable_sdma_mgls(rdev, enable); 6420 } 6421 6422 if (block & RADEON_CG_BLOCK_BIF) { 6423 cik_enable_bif_mgls(rdev, enable); 6424 } 6425 6426 if (block & RADEON_CG_BLOCK_UVD) { 6427 if (rdev->has_uvd) 6428 cik_enable_uvd_mgcg(rdev, enable); 6429 } 6430 6431 if (block & RADEON_CG_BLOCK_HDP) { 6432 cik_enable_hdp_mgcg(rdev, enable); 6433 cik_enable_hdp_ls(rdev, enable); 6434 } 6435 6436 if (block & RADEON_CG_BLOCK_VCE) { 6437 vce_v2_0_enable_mgcg(rdev, enable); 6438 } 6439 } 6440 6441 static void cik_init_cg(struct radeon_device *rdev) 6442 { 6443 6444 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true); 6445 6446 if (rdev->has_uvd) 6447 si_init_uvd_internal_cg(rdev); 6448 6449 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | 6450 RADEON_CG_BLOCK_SDMA | 6451 RADEON_CG_BLOCK_BIF | 6452 RADEON_CG_BLOCK_UVD | 6453 RADEON_CG_BLOCK_HDP), true); 6454 } 6455 6456 static void cik_fini_cg(struct radeon_device *rdev) 6457 { 6458 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | 6459 RADEON_CG_BLOCK_SDMA | 6460 RADEON_CG_BLOCK_BIF | 6461 RADEON_CG_BLOCK_UVD | 6462 RADEON_CG_BLOCK_HDP), false); 6463 6464 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false); 6465 } 6466 6467 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev, 6468 bool enable) 6469 { 6470 u32 data, orig; 6471 6472 orig = data = RREG32(RLC_PG_CNTL); 6473 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) 6474 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE; 6475 else 6476 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE; 6477 if (orig != data) 6478 WREG32(RLC_PG_CNTL, data); 6479 } 6480 6481 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev, 6482 bool enable) 6483 { 6484 u32 data, orig; 6485 6486 orig = data = RREG32(RLC_PG_CNTL); 6487 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) 6488 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE; 6489 else 6490 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE; 6491 if (orig != data) 6492 WREG32(RLC_PG_CNTL, data); 6493 } 6494 6495 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable) 6496 { 6497 u32 data, orig; 6498 6499 orig = data = RREG32(RLC_PG_CNTL); 6500 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP)) 6501 data &= ~DISABLE_CP_PG; 6502 else 6503 data |= DISABLE_CP_PG; 6504 if (orig != data) 6505 WREG32(RLC_PG_CNTL, data); 6506 } 6507 6508 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable) 6509 { 6510 u32 data, orig; 6511 6512 orig = data = RREG32(RLC_PG_CNTL); 6513 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS)) 6514 data &= ~DISABLE_GDS_PG; 6515 else 6516 data |= DISABLE_GDS_PG; 6517 if (orig != data) 6518 WREG32(RLC_PG_CNTL, data); 6519 } 6520 6521 #define CP_ME_TABLE_SIZE 96 6522 #define CP_ME_TABLE_OFFSET 2048 6523 #define CP_MEC_TABLE_OFFSET 4096 6524 6525 void cik_init_cp_pg_table(struct radeon_device *rdev) 6526 { 6527 volatile u32 *dst_ptr; 6528 int me, i, max_me = 4; 6529 u32 bo_offset = 0; 6530 u32 table_offset, table_size; 6531 6532 if (rdev->family == CHIP_KAVERI) 6533 max_me = 5; 6534 6535 if (rdev->rlc.cp_table_ptr == NULL) 6536 return; 6537 6538 /* write the cp table buffer */ 6539 dst_ptr = rdev->rlc.cp_table_ptr; 6540 for (me = 0; me < max_me; me++) { 6541 if (rdev->new_fw) { 6542 const __le32 *fw_data; 6543 const struct gfx_firmware_header_v1_0 *hdr; 6544 6545 if (me == 0) { 6546 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; 6547 fw_data = (const __le32 *) 6548 ((const char *)rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6549 table_offset = le32_to_cpu(hdr->jt_offset); 6550 table_size = le32_to_cpu(hdr->jt_size); 6551 } else if (me == 1) { 6552 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; 6553 fw_data = (const __le32 *) 6554 ((const char *)rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6555 table_offset = le32_to_cpu(hdr->jt_offset); 6556 table_size = le32_to_cpu(hdr->jt_size); 6557 } else if (me == 2) { 6558 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; 6559 fw_data = (const __le32 *) 6560 ((const char *)rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6561 table_offset = le32_to_cpu(hdr->jt_offset); 6562 table_size = le32_to_cpu(hdr->jt_size); 6563 } else if (me == 3) { 6564 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; 6565 fw_data = (const __le32 *) 6566 ((const char *)rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6567 table_offset = le32_to_cpu(hdr->jt_offset); 6568 table_size = le32_to_cpu(hdr->jt_size); 6569 } else { 6570 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; 6571 fw_data = (const __le32 *) 6572 ((const char *)rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6573 table_offset = le32_to_cpu(hdr->jt_offset); 6574 table_size = le32_to_cpu(hdr->jt_size); 6575 } 6576 6577 for (i = 0; i < table_size; i ++) { 6578 dst_ptr[bo_offset + i] = 6579 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i])); 6580 } 6581 bo_offset += table_size; 6582 } else { 6583 const __be32 *fw_data; 6584 table_size = CP_ME_TABLE_SIZE; 6585 6586 if (me == 0) { 6587 fw_data = (const __be32 *)rdev->ce_fw->data; 6588 table_offset = CP_ME_TABLE_OFFSET; 6589 } else if (me == 1) { 6590 fw_data = (const __be32 *)rdev->pfp_fw->data; 6591 table_offset = CP_ME_TABLE_OFFSET; 6592 } else if (me == 2) { 6593 fw_data = (const __be32 *)rdev->me_fw->data; 6594 table_offset = CP_ME_TABLE_OFFSET; 6595 } else { 6596 fw_data = (const __be32 *)rdev->mec_fw->data; 6597 table_offset = CP_MEC_TABLE_OFFSET; 6598 } 6599 6600 for (i = 0; i < table_size; i ++) { 6601 dst_ptr[bo_offset + i] = 6602 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i])); 6603 } 6604 bo_offset += table_size; 6605 } 6606 } 6607 } 6608 6609 static void cik_enable_gfx_cgpg(struct radeon_device *rdev, 6610 bool enable) 6611 { 6612 u32 data, orig; 6613 6614 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) { 6615 orig = data = RREG32(RLC_PG_CNTL); 6616 data |= GFX_PG_ENABLE; 6617 if (orig != data) 6618 WREG32(RLC_PG_CNTL, data); 6619 6620 orig = data = RREG32(RLC_AUTO_PG_CTRL); 6621 data |= AUTO_PG_EN; 6622 if (orig != data) 6623 WREG32(RLC_AUTO_PG_CTRL, data); 6624 } else { 6625 orig = data = RREG32(RLC_PG_CNTL); 6626 data &= ~GFX_PG_ENABLE; 6627 if (orig != data) 6628 WREG32(RLC_PG_CNTL, data); 6629 6630 orig = data = RREG32(RLC_AUTO_PG_CTRL); 6631 data &= ~AUTO_PG_EN; 6632 if (orig != data) 6633 WREG32(RLC_AUTO_PG_CTRL, data); 6634 6635 data = RREG32(DB_RENDER_CONTROL); 6636 } 6637 } 6638 6639 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) 6640 { 6641 u32 mask = 0, tmp, tmp1; 6642 int i; 6643 6644 mutex_lock(&rdev->grbm_idx_mutex); 6645 cik_select_se_sh(rdev, se, sh); 6646 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 6647 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); 6648 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6649 mutex_unlock(&rdev->grbm_idx_mutex); 6650 6651 tmp &= 0xffff0000; 6652 6653 tmp |= tmp1; 6654 tmp >>= 16; 6655 6656 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) { 6657 mask <<= 1; 6658 mask |= 1; 6659 } 6660 6661 return (~tmp) & mask; 6662 } 6663 6664 static void cik_init_ao_cu_mask(struct radeon_device *rdev) 6665 { 6666 u32 i, j, k, active_cu_number = 0; 6667 u32 mask, counter, cu_bitmap; 6668 u32 tmp = 0; 6669 6670 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 6671 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 6672 mask = 1; 6673 cu_bitmap = 0; 6674 counter = 0; 6675 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) { 6676 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) { 6677 if (counter < 2) 6678 cu_bitmap |= mask; 6679 counter ++; 6680 } 6681 mask <<= 1; 6682 } 6683 6684 active_cu_number += counter; 6685 tmp |= (cu_bitmap << (i * 16 + j * 8)); 6686 } 6687 } 6688 6689 WREG32(RLC_PG_AO_CU_MASK, tmp); 6690 6691 tmp = RREG32(RLC_MAX_PG_CU); 6692 tmp &= ~MAX_PU_CU_MASK; 6693 tmp |= MAX_PU_CU(active_cu_number); 6694 WREG32(RLC_MAX_PG_CU, tmp); 6695 } 6696 6697 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev, 6698 bool enable) 6699 { 6700 u32 data, orig; 6701 6702 orig = data = RREG32(RLC_PG_CNTL); 6703 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG)) 6704 data |= STATIC_PER_CU_PG_ENABLE; 6705 else 6706 data &= ~STATIC_PER_CU_PG_ENABLE; 6707 if (orig != data) 6708 WREG32(RLC_PG_CNTL, data); 6709 } 6710 6711 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev, 6712 bool enable) 6713 { 6714 u32 data, orig; 6715 6716 orig = data = RREG32(RLC_PG_CNTL); 6717 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG)) 6718 data |= DYN_PER_CU_PG_ENABLE; 6719 else 6720 data &= ~DYN_PER_CU_PG_ENABLE; 6721 if (orig != data) 6722 WREG32(RLC_PG_CNTL, data); 6723 } 6724 6725 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90 6726 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D 6727 6728 static void cik_init_gfx_cgpg(struct radeon_device *rdev) 6729 { 6730 u32 data, orig; 6731 u32 i; 6732 6733 if (rdev->rlc.cs_data) { 6734 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); 6735 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr)); 6736 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr)); 6737 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size); 6738 } else { 6739 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); 6740 for (i = 0; i < 3; i++) 6741 WREG32(RLC_GPM_SCRATCH_DATA, 0); 6742 } 6743 if (rdev->rlc.reg_list) { 6744 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET); 6745 for (i = 0; i < rdev->rlc.reg_list_size; i++) 6746 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]); 6747 } 6748 6749 orig = data = RREG32(RLC_PG_CNTL); 6750 data |= GFX_PG_SRC; 6751 if (orig != data) 6752 WREG32(RLC_PG_CNTL, data); 6753 6754 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 6755 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8); 6756 6757 data = RREG32(CP_RB_WPTR_POLL_CNTL); 6758 data &= ~IDLE_POLL_COUNT_MASK; 6759 data |= IDLE_POLL_COUNT(0x60); 6760 WREG32(CP_RB_WPTR_POLL_CNTL, data); 6761 6762 data = 0x10101010; 6763 WREG32(RLC_PG_DELAY, data); 6764 6765 data = RREG32(RLC_PG_DELAY_2); 6766 data &= ~0xff; 6767 data |= 0x3; 6768 WREG32(RLC_PG_DELAY_2, data); 6769 6770 data = RREG32(RLC_AUTO_PG_CTRL); 6771 data &= ~GRBM_REG_SGIT_MASK; 6772 data |= GRBM_REG_SGIT(0x700); 6773 WREG32(RLC_AUTO_PG_CTRL, data); 6774 6775 } 6776 6777 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable) 6778 { 6779 cik_enable_gfx_cgpg(rdev, enable); 6780 cik_enable_gfx_static_mgpg(rdev, enable); 6781 cik_enable_gfx_dynamic_mgpg(rdev, enable); 6782 } 6783 6784 u32 cik_get_csb_size(struct radeon_device *rdev) 6785 { 6786 u32 count = 0; 6787 const struct cs_section_def *sect = NULL; 6788 const struct cs_extent_def *ext = NULL; 6789 6790 if (rdev->rlc.cs_data == NULL) 6791 return 0; 6792 6793 /* begin clear state */ 6794 count += 2; 6795 /* context control state */ 6796 count += 3; 6797 6798 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 6799 for (ext = sect->section; ext->extent != NULL; ++ext) { 6800 if (sect->id == SECT_CONTEXT) 6801 count += 2 + ext->reg_count; 6802 else 6803 return 0; 6804 } 6805 } 6806 /* pa_sc_raster_config/pa_sc_raster_config1 */ 6807 count += 4; 6808 /* end clear state */ 6809 count += 2; 6810 /* clear state */ 6811 count += 2; 6812 6813 return count; 6814 } 6815 6816 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) 6817 { 6818 u32 count = 0, i; 6819 const struct cs_section_def *sect = NULL; 6820 const struct cs_extent_def *ext = NULL; 6821 6822 if (rdev->rlc.cs_data == NULL) 6823 return; 6824 if (buffer == NULL) 6825 return; 6826 6827 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 6828 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 6829 6830 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 6831 buffer[count++] = cpu_to_le32(0x80000000); 6832 buffer[count++] = cpu_to_le32(0x80000000); 6833 6834 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 6835 for (ext = sect->section; ext->extent != NULL; ++ext) { 6836 if (sect->id == SECT_CONTEXT) { 6837 buffer[count++] = 6838 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 6839 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000); 6840 for (i = 0; i < ext->reg_count; i++) 6841 buffer[count++] = cpu_to_le32(ext->extent[i]); 6842 } else { 6843 return; 6844 } 6845 } 6846 } 6847 6848 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 6849 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 6850 switch (rdev->family) { 6851 case CHIP_BONAIRE: 6852 buffer[count++] = cpu_to_le32(0x16000012); 6853 buffer[count++] = cpu_to_le32(0x00000000); 6854 break; 6855 case CHIP_KAVERI: 6856 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ 6857 buffer[count++] = cpu_to_le32(0x00000000); 6858 break; 6859 case CHIP_KABINI: 6860 case CHIP_MULLINS: 6861 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ 6862 buffer[count++] = cpu_to_le32(0x00000000); 6863 break; 6864 case CHIP_HAWAII: 6865 buffer[count++] = cpu_to_le32(0x3a00161a); 6866 buffer[count++] = cpu_to_le32(0x0000002e); 6867 break; 6868 default: 6869 buffer[count++] = cpu_to_le32(0x00000000); 6870 buffer[count++] = cpu_to_le32(0x00000000); 6871 break; 6872 } 6873 6874 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 6875 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 6876 6877 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 6878 buffer[count++] = cpu_to_le32(0); 6879 } 6880 6881 static void cik_init_pg(struct radeon_device *rdev) 6882 { 6883 if (rdev->pg_flags) { 6884 cik_enable_sck_slowdown_on_pu(rdev, true); 6885 cik_enable_sck_slowdown_on_pd(rdev, true); 6886 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 6887 cik_init_gfx_cgpg(rdev); 6888 cik_enable_cp_pg(rdev, true); 6889 cik_enable_gds_pg(rdev, true); 6890 } 6891 cik_init_ao_cu_mask(rdev); 6892 cik_update_gfx_pg(rdev, true); 6893 } 6894 } 6895 6896 static void cik_fini_pg(struct radeon_device *rdev) 6897 { 6898 if (rdev->pg_flags) { 6899 cik_update_gfx_pg(rdev, false); 6900 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 6901 cik_enable_cp_pg(rdev, false); 6902 cik_enable_gds_pg(rdev, false); 6903 } 6904 } 6905 } 6906 6907 /* 6908 * Interrupts 6909 * Starting with r6xx, interrupts are handled via a ring buffer. 6910 * Ring buffers are areas of GPU accessible memory that the GPU 6911 * writes interrupt vectors into and the host reads vectors out of. 6912 * There is a rptr (read pointer) that determines where the 6913 * host is currently reading, and a wptr (write pointer) 6914 * which determines where the GPU has written. When the 6915 * pointers are equal, the ring is idle. When the GPU 6916 * writes vectors to the ring buffer, it increments the 6917 * wptr. When there is an interrupt, the host then starts 6918 * fetching commands and processing them until the pointers are 6919 * equal again at which point it updates the rptr. 6920 */ 6921 6922 /** 6923 * cik_enable_interrupts - Enable the interrupt ring buffer 6924 * 6925 * @rdev: radeon_device pointer 6926 * 6927 * Enable the interrupt ring buffer (CIK). 6928 */ 6929 static void cik_enable_interrupts(struct radeon_device *rdev) 6930 { 6931 u32 ih_cntl = RREG32(IH_CNTL); 6932 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 6933 6934 ih_cntl |= ENABLE_INTR; 6935 ih_rb_cntl |= IH_RB_ENABLE; 6936 WREG32(IH_CNTL, ih_cntl); 6937 WREG32(IH_RB_CNTL, ih_rb_cntl); 6938 rdev->ih.enabled = true; 6939 } 6940 6941 /** 6942 * cik_disable_interrupts - Disable the interrupt ring buffer 6943 * 6944 * @rdev: radeon_device pointer 6945 * 6946 * Disable the interrupt ring buffer (CIK). 6947 */ 6948 static void cik_disable_interrupts(struct radeon_device *rdev) 6949 { 6950 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 6951 u32 ih_cntl = RREG32(IH_CNTL); 6952 6953 ih_rb_cntl &= ~IH_RB_ENABLE; 6954 ih_cntl &= ~ENABLE_INTR; 6955 WREG32(IH_RB_CNTL, ih_rb_cntl); 6956 WREG32(IH_CNTL, ih_cntl); 6957 /* set rptr, wptr to 0 */ 6958 WREG32(IH_RB_RPTR, 0); 6959 WREG32(IH_RB_WPTR, 0); 6960 rdev->ih.enabled = false; 6961 rdev->ih.rptr = 0; 6962 } 6963 6964 /** 6965 * cik_disable_interrupt_state - Disable all interrupt sources 6966 * 6967 * @rdev: radeon_device pointer 6968 * 6969 * Clear all interrupt enable bits used by the driver (CIK). 6970 */ 6971 static void cik_disable_interrupt_state(struct radeon_device *rdev) 6972 { 6973 u32 tmp; 6974 6975 /* gfx ring */ 6976 tmp = RREG32(CP_INT_CNTL_RING0) & 6977 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 6978 WREG32(CP_INT_CNTL_RING0, tmp); 6979 /* sdma */ 6980 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 6981 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp); 6982 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 6983 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp); 6984 /* compute queues */ 6985 WREG32(CP_ME1_PIPE0_INT_CNTL, 0); 6986 WREG32(CP_ME1_PIPE1_INT_CNTL, 0); 6987 WREG32(CP_ME1_PIPE2_INT_CNTL, 0); 6988 WREG32(CP_ME1_PIPE3_INT_CNTL, 0); 6989 WREG32(CP_ME2_PIPE0_INT_CNTL, 0); 6990 WREG32(CP_ME2_PIPE1_INT_CNTL, 0); 6991 WREG32(CP_ME2_PIPE2_INT_CNTL, 0); 6992 WREG32(CP_ME2_PIPE3_INT_CNTL, 0); 6993 /* grbm */ 6994 WREG32(GRBM_INT_CNTL, 0); 6995 /* SRBM */ 6996 WREG32(SRBM_INT_CNTL, 0); 6997 /* vline/vblank, etc. */ 6998 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 6999 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 7000 if (rdev->num_crtc >= 4) { 7001 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 7002 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 7003 } 7004 if (rdev->num_crtc >= 6) { 7005 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 7006 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 7007 } 7008 /* pflip */ 7009 if (rdev->num_crtc >= 2) { 7010 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 7011 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 7012 } 7013 if (rdev->num_crtc >= 4) { 7014 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 7015 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 7016 } 7017 if (rdev->num_crtc >= 6) { 7018 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 7019 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 7020 } 7021 7022 /* dac hotplug */ 7023 WREG32(DAC_AUTODETECT_INT_CONTROL, 0); 7024 7025 /* digital hotplug */ 7026 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7027 WREG32(DC_HPD1_INT_CONTROL, tmp); 7028 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7029 WREG32(DC_HPD2_INT_CONTROL, tmp); 7030 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7031 WREG32(DC_HPD3_INT_CONTROL, tmp); 7032 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7033 WREG32(DC_HPD4_INT_CONTROL, tmp); 7034 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7035 WREG32(DC_HPD5_INT_CONTROL, tmp); 7036 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7037 WREG32(DC_HPD6_INT_CONTROL, tmp); 7038 7039 } 7040 7041 /** 7042 * cik_irq_init - init and enable the interrupt ring 7043 * 7044 * @rdev: radeon_device pointer 7045 * 7046 * Allocate a ring buffer for the interrupt controller, 7047 * enable the RLC, disable interrupts, enable the IH 7048 * ring buffer and enable it (CIK). 7049 * Called at device load and reume. 7050 * Returns 0 for success, errors for failure. 7051 */ 7052 static int cik_irq_init(struct radeon_device *rdev) 7053 { 7054 int ret = 0; 7055 int rb_bufsz; 7056 u32 interrupt_cntl, ih_cntl, ih_rb_cntl; 7057 7058 /* allocate ring */ 7059 ret = r600_ih_ring_alloc(rdev); 7060 if (ret) 7061 return ret; 7062 7063 /* disable irqs */ 7064 cik_disable_interrupts(rdev); 7065 7066 /* init rlc */ 7067 ret = cik_rlc_resume(rdev); 7068 if (ret) { 7069 r600_ih_ring_fini(rdev); 7070 return ret; 7071 } 7072 7073 /* setup interrupt control */ 7074 /* XXX this should actually be a bus address, not an MC address. same on older asics */ 7075 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8); 7076 interrupt_cntl = RREG32(INTERRUPT_CNTL); 7077 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi 7078 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN 7079 */ 7080 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE; 7081 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */ 7082 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN; 7083 WREG32(INTERRUPT_CNTL, interrupt_cntl); 7084 7085 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); 7086 rb_bufsz = order_base_2(rdev->ih.ring_size / 4); 7087 7088 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | 7089 IH_WPTR_OVERFLOW_CLEAR | 7090 (rb_bufsz << 1)); 7091 7092 if (rdev->wb.enabled) 7093 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE; 7094 7095 /* set the writeback address whether it's enabled or not */ 7096 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); 7097 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); 7098 7099 WREG32(IH_RB_CNTL, ih_rb_cntl); 7100 7101 /* set rptr, wptr to 0 */ 7102 WREG32(IH_RB_RPTR, 0); 7103 WREG32(IH_RB_WPTR, 0); 7104 7105 /* Default settings for IH_CNTL (disabled at first) */ 7106 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0); 7107 /* RPTR_REARM only works if msi's are enabled */ 7108 if (rdev->msi_enabled) 7109 ih_cntl |= RPTR_REARM; 7110 WREG32(IH_CNTL, ih_cntl); 7111 7112 /* force the active interrupt state to all disabled */ 7113 cik_disable_interrupt_state(rdev); 7114 7115 pci_enable_busmaster(rdev->pdev->dev.bsddev); 7116 7117 /* enable irqs */ 7118 cik_enable_interrupts(rdev); 7119 7120 return ret; 7121 } 7122 7123 /** 7124 * cik_irq_set - enable/disable interrupt sources 7125 * 7126 * @rdev: radeon_device pointer 7127 * 7128 * Enable interrupt sources on the GPU (vblanks, hpd, 7129 * etc.) (CIK). 7130 * Returns 0 for success, errors for failure. 7131 */ 7132 int cik_irq_set(struct radeon_device *rdev) 7133 { 7134 u32 cp_int_cntl; 7135 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3; 7136 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3; 7137 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 7138 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 7139 u32 grbm_int_cntl = 0; 7140 u32 dma_cntl, dma_cntl1; 7141 7142 if (!rdev->irq.installed) { 7143 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 7144 return -EINVAL; 7145 } 7146 /* don't enable anything if the ih is disabled */ 7147 if (!rdev->ih.enabled) { 7148 cik_disable_interrupts(rdev); 7149 /* force the active interrupt state to all disabled */ 7150 cik_disable_interrupt_state(rdev); 7151 return 0; 7152 } 7153 7154 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) & 7155 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 7156 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE; 7157 7158 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 7159 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 7160 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 7161 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 7162 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 7163 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 7164 7165 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 7166 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 7167 7168 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7169 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7170 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7171 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7172 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7173 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7174 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7175 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7176 7177 /* enable CP interrupts on all rings */ 7178 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 7179 DRM_DEBUG("cik_irq_set: sw int gfx\n"); 7180 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 7181 } 7182 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 7183 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 7184 DRM_DEBUG("si_irq_set: sw int cp1\n"); 7185 if (ring->me == 1) { 7186 switch (ring->pipe) { 7187 case 0: 7188 cp_m1p0 |= TIME_STAMP_INT_ENABLE; 7189 break; 7190 case 1: 7191 cp_m1p1 |= TIME_STAMP_INT_ENABLE; 7192 break; 7193 case 2: 7194 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 7195 break; 7196 case 3: 7197 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 7198 break; 7199 default: 7200 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); 7201 break; 7202 } 7203 } else if (ring->me == 2) { 7204 switch (ring->pipe) { 7205 case 0: 7206 cp_m2p0 |= TIME_STAMP_INT_ENABLE; 7207 break; 7208 case 1: 7209 cp_m2p1 |= TIME_STAMP_INT_ENABLE; 7210 break; 7211 case 2: 7212 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 7213 break; 7214 case 3: 7215 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 7216 break; 7217 default: 7218 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); 7219 break; 7220 } 7221 } else { 7222 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me); 7223 } 7224 } 7225 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 7226 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 7227 DRM_DEBUG("si_irq_set: sw int cp2\n"); 7228 if (ring->me == 1) { 7229 switch (ring->pipe) { 7230 case 0: 7231 cp_m1p0 |= TIME_STAMP_INT_ENABLE; 7232 break; 7233 case 1: 7234 cp_m1p1 |= TIME_STAMP_INT_ENABLE; 7235 break; 7236 case 2: 7237 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 7238 break; 7239 case 3: 7240 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 7241 break; 7242 default: 7243 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); 7244 break; 7245 } 7246 } else if (ring->me == 2) { 7247 switch (ring->pipe) { 7248 case 0: 7249 cp_m2p0 |= TIME_STAMP_INT_ENABLE; 7250 break; 7251 case 1: 7252 cp_m2p1 |= TIME_STAMP_INT_ENABLE; 7253 break; 7254 case 2: 7255 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 7256 break; 7257 case 3: 7258 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 7259 break; 7260 default: 7261 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); 7262 break; 7263 } 7264 } else { 7265 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me); 7266 } 7267 } 7268 7269 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 7270 DRM_DEBUG("cik_irq_set: sw int dma\n"); 7271 dma_cntl |= TRAP_ENABLE; 7272 } 7273 7274 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 7275 DRM_DEBUG("cik_irq_set: sw int dma1\n"); 7276 dma_cntl1 |= TRAP_ENABLE; 7277 } 7278 7279 if (rdev->irq.crtc_vblank_int[0] || 7280 atomic_read(&rdev->irq.pflip[0])) { 7281 DRM_DEBUG("cik_irq_set: vblank 0\n"); 7282 crtc1 |= VBLANK_INTERRUPT_MASK; 7283 } 7284 if (rdev->irq.crtc_vblank_int[1] || 7285 atomic_read(&rdev->irq.pflip[1])) { 7286 DRM_DEBUG("cik_irq_set: vblank 1\n"); 7287 crtc2 |= VBLANK_INTERRUPT_MASK; 7288 } 7289 if (rdev->irq.crtc_vblank_int[2] || 7290 atomic_read(&rdev->irq.pflip[2])) { 7291 DRM_DEBUG("cik_irq_set: vblank 2\n"); 7292 crtc3 |= VBLANK_INTERRUPT_MASK; 7293 } 7294 if (rdev->irq.crtc_vblank_int[3] || 7295 atomic_read(&rdev->irq.pflip[3])) { 7296 DRM_DEBUG("cik_irq_set: vblank 3\n"); 7297 crtc4 |= VBLANK_INTERRUPT_MASK; 7298 } 7299 if (rdev->irq.crtc_vblank_int[4] || 7300 atomic_read(&rdev->irq.pflip[4])) { 7301 DRM_DEBUG("cik_irq_set: vblank 4\n"); 7302 crtc5 |= VBLANK_INTERRUPT_MASK; 7303 } 7304 if (rdev->irq.crtc_vblank_int[5] || 7305 atomic_read(&rdev->irq.pflip[5])) { 7306 DRM_DEBUG("cik_irq_set: vblank 5\n"); 7307 crtc6 |= VBLANK_INTERRUPT_MASK; 7308 } 7309 if (rdev->irq.hpd[0]) { 7310 DRM_DEBUG("cik_irq_set: hpd 1\n"); 7311 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 7312 } 7313 if (rdev->irq.hpd[1]) { 7314 DRM_DEBUG("cik_irq_set: hpd 2\n"); 7315 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 7316 } 7317 if (rdev->irq.hpd[2]) { 7318 DRM_DEBUG("cik_irq_set: hpd 3\n"); 7319 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 7320 } 7321 if (rdev->irq.hpd[3]) { 7322 DRM_DEBUG("cik_irq_set: hpd 4\n"); 7323 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 7324 } 7325 if (rdev->irq.hpd[4]) { 7326 DRM_DEBUG("cik_irq_set: hpd 5\n"); 7327 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 7328 } 7329 if (rdev->irq.hpd[5]) { 7330 DRM_DEBUG("cik_irq_set: hpd 6\n"); 7331 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 7332 } 7333 7334 WREG32(CP_INT_CNTL_RING0, cp_int_cntl); 7335 7336 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl); 7337 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1); 7338 7339 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0); 7340 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1); 7341 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2); 7342 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3); 7343 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0); 7344 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1); 7345 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2); 7346 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3); 7347 7348 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 7349 7350 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 7351 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 7352 if (rdev->num_crtc >= 4) { 7353 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 7354 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 7355 } 7356 if (rdev->num_crtc >= 6) { 7357 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 7358 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 7359 } 7360 7361 if (rdev->num_crtc >= 2) { 7362 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 7363 GRPH_PFLIP_INT_MASK); 7364 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 7365 GRPH_PFLIP_INT_MASK); 7366 } 7367 if (rdev->num_crtc >= 4) { 7368 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 7369 GRPH_PFLIP_INT_MASK); 7370 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 7371 GRPH_PFLIP_INT_MASK); 7372 } 7373 if (rdev->num_crtc >= 6) { 7374 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 7375 GRPH_PFLIP_INT_MASK); 7376 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 7377 GRPH_PFLIP_INT_MASK); 7378 } 7379 7380 WREG32(DC_HPD1_INT_CONTROL, hpd1); 7381 WREG32(DC_HPD2_INT_CONTROL, hpd2); 7382 WREG32(DC_HPD3_INT_CONTROL, hpd3); 7383 WREG32(DC_HPD4_INT_CONTROL, hpd4); 7384 WREG32(DC_HPD5_INT_CONTROL, hpd5); 7385 WREG32(DC_HPD6_INT_CONTROL, hpd6); 7386 7387 /* posting read */ 7388 RREG32(SRBM_STATUS); 7389 7390 return 0; 7391 } 7392 7393 /** 7394 * cik_irq_ack - ack interrupt sources 7395 * 7396 * @rdev: radeon_device pointer 7397 * 7398 * Ack interrupt sources on the GPU (vblanks, hpd, 7399 * etc.) (CIK). Certain interrupts sources are sw 7400 * generated and do not require an explicit ack. 7401 */ 7402 static inline void cik_irq_ack(struct radeon_device *rdev) 7403 { 7404 u32 tmp; 7405 7406 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS); 7407 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 7408 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 7409 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 7410 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 7411 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 7412 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6); 7413 7414 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS + 7415 EVERGREEN_CRTC0_REGISTER_OFFSET); 7416 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS + 7417 EVERGREEN_CRTC1_REGISTER_OFFSET); 7418 if (rdev->num_crtc >= 4) { 7419 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS + 7420 EVERGREEN_CRTC2_REGISTER_OFFSET); 7421 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS + 7422 EVERGREEN_CRTC3_REGISTER_OFFSET); 7423 } 7424 if (rdev->num_crtc >= 6) { 7425 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS + 7426 EVERGREEN_CRTC4_REGISTER_OFFSET); 7427 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS + 7428 EVERGREEN_CRTC5_REGISTER_OFFSET); 7429 } 7430 7431 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 7432 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, 7433 GRPH_PFLIP_INT_CLEAR); 7434 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 7435 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, 7436 GRPH_PFLIP_INT_CLEAR); 7437 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) 7438 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 7439 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) 7440 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 7441 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 7442 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 7443 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) 7444 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 7445 7446 if (rdev->num_crtc >= 4) { 7447 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 7448 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, 7449 GRPH_PFLIP_INT_CLEAR); 7450 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 7451 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, 7452 GRPH_PFLIP_INT_CLEAR); 7453 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 7454 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 7455 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 7456 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 7457 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 7458 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 7459 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 7460 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 7461 } 7462 7463 if (rdev->num_crtc >= 6) { 7464 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 7465 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, 7466 GRPH_PFLIP_INT_CLEAR); 7467 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 7468 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, 7469 GRPH_PFLIP_INT_CLEAR); 7470 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 7471 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 7472 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 7473 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 7474 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 7475 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 7476 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 7477 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 7478 } 7479 7480 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { 7481 tmp = RREG32(DC_HPD1_INT_CONTROL); 7482 tmp |= DC_HPDx_INT_ACK; 7483 WREG32(DC_HPD1_INT_CONTROL, tmp); 7484 } 7485 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { 7486 tmp = RREG32(DC_HPD2_INT_CONTROL); 7487 tmp |= DC_HPDx_INT_ACK; 7488 WREG32(DC_HPD2_INT_CONTROL, tmp); 7489 } 7490 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { 7491 tmp = RREG32(DC_HPD3_INT_CONTROL); 7492 tmp |= DC_HPDx_INT_ACK; 7493 WREG32(DC_HPD3_INT_CONTROL, tmp); 7494 } 7495 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { 7496 tmp = RREG32(DC_HPD4_INT_CONTROL); 7497 tmp |= DC_HPDx_INT_ACK; 7498 WREG32(DC_HPD4_INT_CONTROL, tmp); 7499 } 7500 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { 7501 tmp = RREG32(DC_HPD5_INT_CONTROL); 7502 tmp |= DC_HPDx_INT_ACK; 7503 WREG32(DC_HPD5_INT_CONTROL, tmp); 7504 } 7505 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { 7506 tmp = RREG32(DC_HPD5_INT_CONTROL); 7507 tmp |= DC_HPDx_INT_ACK; 7508 WREG32(DC_HPD6_INT_CONTROL, tmp); 7509 } 7510 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) { 7511 tmp = RREG32(DC_HPD1_INT_CONTROL); 7512 tmp |= DC_HPDx_RX_INT_ACK; 7513 WREG32(DC_HPD1_INT_CONTROL, tmp); 7514 } 7515 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) { 7516 tmp = RREG32(DC_HPD2_INT_CONTROL); 7517 tmp |= DC_HPDx_RX_INT_ACK; 7518 WREG32(DC_HPD2_INT_CONTROL, tmp); 7519 } 7520 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) { 7521 tmp = RREG32(DC_HPD3_INT_CONTROL); 7522 tmp |= DC_HPDx_RX_INT_ACK; 7523 WREG32(DC_HPD3_INT_CONTROL, tmp); 7524 } 7525 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) { 7526 tmp = RREG32(DC_HPD4_INT_CONTROL); 7527 tmp |= DC_HPDx_RX_INT_ACK; 7528 WREG32(DC_HPD4_INT_CONTROL, tmp); 7529 } 7530 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) { 7531 tmp = RREG32(DC_HPD5_INT_CONTROL); 7532 tmp |= DC_HPDx_RX_INT_ACK; 7533 WREG32(DC_HPD5_INT_CONTROL, tmp); 7534 } 7535 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) { 7536 tmp = RREG32(DC_HPD5_INT_CONTROL); 7537 tmp |= DC_HPDx_RX_INT_ACK; 7538 WREG32(DC_HPD6_INT_CONTROL, tmp); 7539 } 7540 } 7541 7542 /** 7543 * cik_irq_disable - disable interrupts 7544 * 7545 * @rdev: radeon_device pointer 7546 * 7547 * Disable interrupts on the hw (CIK). 7548 */ 7549 static void cik_irq_disable(struct radeon_device *rdev) 7550 { 7551 cik_disable_interrupts(rdev); 7552 /* Wait and acknowledge irq */ 7553 mdelay(1); 7554 cik_irq_ack(rdev); 7555 cik_disable_interrupt_state(rdev); 7556 } 7557 7558 /** 7559 * cik_irq_disable - disable interrupts for suspend 7560 * 7561 * @rdev: radeon_device pointer 7562 * 7563 * Disable interrupts and stop the RLC (CIK). 7564 * Used for suspend. 7565 */ 7566 static void cik_irq_suspend(struct radeon_device *rdev) 7567 { 7568 cik_irq_disable(rdev); 7569 cik_rlc_stop(rdev); 7570 } 7571 7572 /** 7573 * cik_irq_fini - tear down interrupt support 7574 * 7575 * @rdev: radeon_device pointer 7576 * 7577 * Disable interrupts on the hw and free the IH ring 7578 * buffer (CIK). 7579 * Used for driver unload. 7580 */ 7581 static void cik_irq_fini(struct radeon_device *rdev) 7582 { 7583 cik_irq_suspend(rdev); 7584 r600_ih_ring_fini(rdev); 7585 } 7586 7587 /** 7588 * cik_get_ih_wptr - get the IH ring buffer wptr 7589 * 7590 * @rdev: radeon_device pointer 7591 * 7592 * Get the IH ring buffer wptr from either the register 7593 * or the writeback memory buffer (CIK). Also check for 7594 * ring buffer overflow and deal with it. 7595 * Used by cik_irq_process(). 7596 * Returns the value of the wptr. 7597 */ 7598 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev) 7599 { 7600 u32 wptr, tmp; 7601 7602 if (rdev->wb.enabled) 7603 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 7604 else 7605 wptr = RREG32(IH_RB_WPTR); 7606 7607 if (wptr & RB_OVERFLOW) { 7608 wptr &= ~RB_OVERFLOW; 7609 /* When a ring buffer overflow happen start parsing interrupt 7610 * from the last not overwritten vector (wptr + 16). Hopefully 7611 * this should allow us to catchup. 7612 */ 7613 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", 7614 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); 7615 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 7616 tmp = RREG32(IH_RB_CNTL); 7617 tmp |= IH_WPTR_OVERFLOW_CLEAR; 7618 WREG32(IH_RB_CNTL, tmp); 7619 } 7620 return (wptr & rdev->ih.ptr_mask); 7621 } 7622 7623 /* CIK IV Ring 7624 * Each IV ring entry is 128 bits: 7625 * [7:0] - interrupt source id 7626 * [31:8] - reserved 7627 * [59:32] - interrupt source data 7628 * [63:60] - reserved 7629 * [71:64] - RINGID 7630 * CP: 7631 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0] 7632 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher 7633 * - for gfx, hw shader state (0=PS...5=LS, 6=CS) 7634 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes 7635 * PIPE_ID - ME0 0=3D 7636 * - ME1&2 compute dispatcher (4 pipes each) 7637 * SDMA: 7638 * INSTANCE_ID [1:0], QUEUE_ID[1:0] 7639 * INSTANCE_ID - 0 = sdma0, 1 = sdma1 7640 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1 7641 * [79:72] - VMID 7642 * [95:80] - PASID 7643 * [127:96] - reserved 7644 */ 7645 /** 7646 * cik_irq_process - interrupt handler 7647 * 7648 * @rdev: radeon_device pointer 7649 * 7650 * Interrupt hander (CIK). Walk the IH ring, 7651 * ack interrupts and schedule work to handle 7652 * interrupt events. 7653 * Returns irq process return code. 7654 */ 7655 irqreturn_t cik_irq_process(struct radeon_device *rdev) 7656 { 7657 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 7658 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 7659 u32 wptr; 7660 u32 rptr; 7661 u32 src_id, src_data, ring_id; 7662 u8 me_id, pipe_id, queue_id; 7663 u32 ring_index; 7664 bool queue_hotplug = false; 7665 bool queue_dp = false; 7666 bool queue_reset = false; 7667 u32 addr, status, mc_client; 7668 bool queue_thermal = false; 7669 7670 if (!rdev->ih.enabled || rdev->shutdown) 7671 return IRQ_NONE; 7672 7673 wptr = cik_get_ih_wptr(rdev); 7674 7675 restart_ih: 7676 /* is somebody else already processing irqs? */ 7677 if (atomic_xchg(&rdev->ih.lock, 1)) 7678 return IRQ_NONE; 7679 7680 rptr = rdev->ih.rptr; 7681 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 7682 7683 /* Order reading of wptr vs. reading of IH ring data */ 7684 rmb(); 7685 7686 /* display interrupts */ 7687 cik_irq_ack(rdev); 7688 7689 while (rptr != wptr) { 7690 /* wptr/rptr are in bytes! */ 7691 ring_index = rptr / 4; 7692 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 7693 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 7694 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; 7695 7696 switch (src_id) { 7697 case 1: /* D1 vblank/vline */ 7698 switch (src_data) { 7699 case 0: /* D1 vblank */ 7700 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)) 7701 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7702 7703 if (rdev->irq.crtc_vblank_int[0]) { 7704 drm_handle_vblank(rdev->ddev, 0); 7705 rdev->pm.vblank_sync = true; 7706 wake_up(&rdev->irq.vblank_queue); 7707 } 7708 if (atomic_read(&rdev->irq.pflip[0])) 7709 radeon_crtc_handle_vblank(rdev, 0); 7710 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 7711 DRM_DEBUG("IH: D1 vblank\n"); 7712 7713 break; 7714 case 1: /* D1 vline */ 7715 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)) 7716 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7717 7718 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT; 7719 DRM_DEBUG("IH: D1 vline\n"); 7720 7721 break; 7722 default: 7723 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7724 break; 7725 } 7726 break; 7727 case 2: /* D2 vblank/vline */ 7728 switch (src_data) { 7729 case 0: /* D2 vblank */ 7730 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)) 7731 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7732 7733 if (rdev->irq.crtc_vblank_int[1]) { 7734 drm_handle_vblank(rdev->ddev, 1); 7735 rdev->pm.vblank_sync = true; 7736 wake_up(&rdev->irq.vblank_queue); 7737 } 7738 if (atomic_read(&rdev->irq.pflip[1])) 7739 radeon_crtc_handle_vblank(rdev, 1); 7740 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 7741 DRM_DEBUG("IH: D2 vblank\n"); 7742 7743 break; 7744 case 1: /* D2 vline */ 7745 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)) 7746 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7747 7748 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 7749 DRM_DEBUG("IH: D2 vline\n"); 7750 7751 break; 7752 default: 7753 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7754 break; 7755 } 7756 break; 7757 case 3: /* D3 vblank/vline */ 7758 switch (src_data) { 7759 case 0: /* D3 vblank */ 7760 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)) 7761 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7762 7763 if (rdev->irq.crtc_vblank_int[2]) { 7764 drm_handle_vblank(rdev->ddev, 2); 7765 rdev->pm.vblank_sync = true; 7766 wake_up(&rdev->irq.vblank_queue); 7767 } 7768 if (atomic_read(&rdev->irq.pflip[2])) 7769 radeon_crtc_handle_vblank(rdev, 2); 7770 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 7771 DRM_DEBUG("IH: D3 vblank\n"); 7772 7773 break; 7774 case 1: /* D3 vline */ 7775 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)) 7776 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7777 7778 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 7779 DRM_DEBUG("IH: D3 vline\n"); 7780 7781 break; 7782 default: 7783 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7784 break; 7785 } 7786 break; 7787 case 4: /* D4 vblank/vline */ 7788 switch (src_data) { 7789 case 0: /* D4 vblank */ 7790 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)) 7791 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7792 7793 if (rdev->irq.crtc_vblank_int[3]) { 7794 drm_handle_vblank(rdev->ddev, 3); 7795 rdev->pm.vblank_sync = true; 7796 wake_up(&rdev->irq.vblank_queue); 7797 } 7798 if (atomic_read(&rdev->irq.pflip[3])) 7799 radeon_crtc_handle_vblank(rdev, 3); 7800 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 7801 DRM_DEBUG("IH: D4 vblank\n"); 7802 7803 break; 7804 case 1: /* D4 vline */ 7805 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)) 7806 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7807 7808 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 7809 DRM_DEBUG("IH: D4 vline\n"); 7810 7811 break; 7812 default: 7813 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7814 break; 7815 } 7816 break; 7817 case 5: /* D5 vblank/vline */ 7818 switch (src_data) { 7819 case 0: /* D5 vblank */ 7820 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)) 7821 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7822 7823 if (rdev->irq.crtc_vblank_int[4]) { 7824 drm_handle_vblank(rdev->ddev, 4); 7825 rdev->pm.vblank_sync = true; 7826 wake_up(&rdev->irq.vblank_queue); 7827 } 7828 if (atomic_read(&rdev->irq.pflip[4])) 7829 radeon_crtc_handle_vblank(rdev, 4); 7830 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 7831 DRM_DEBUG("IH: D5 vblank\n"); 7832 7833 break; 7834 case 1: /* D5 vline */ 7835 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)) 7836 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7837 7838 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 7839 DRM_DEBUG("IH: D5 vline\n"); 7840 7841 break; 7842 default: 7843 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7844 break; 7845 } 7846 break; 7847 case 6: /* D6 vblank/vline */ 7848 switch (src_data) { 7849 case 0: /* D6 vblank */ 7850 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)) 7851 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7852 7853 if (rdev->irq.crtc_vblank_int[5]) { 7854 drm_handle_vblank(rdev->ddev, 5); 7855 rdev->pm.vblank_sync = true; 7856 wake_up(&rdev->irq.vblank_queue); 7857 } 7858 if (atomic_read(&rdev->irq.pflip[5])) 7859 radeon_crtc_handle_vblank(rdev, 5); 7860 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 7861 DRM_DEBUG("IH: D6 vblank\n"); 7862 7863 break; 7864 case 1: /* D6 vline */ 7865 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)) 7866 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7867 7868 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 7869 DRM_DEBUG("IH: D6 vline\n"); 7870 7871 break; 7872 default: 7873 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7874 break; 7875 } 7876 break; 7877 case 8: /* D1 page flip */ 7878 case 10: /* D2 page flip */ 7879 case 12: /* D3 page flip */ 7880 case 14: /* D4 page flip */ 7881 case 16: /* D5 page flip */ 7882 case 18: /* D6 page flip */ 7883 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); 7884 if (radeon_use_pflipirq > 0) 7885 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); 7886 break; 7887 case 42: /* HPD hotplug */ 7888 switch (src_data) { 7889 case 0: 7890 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT)) 7891 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7892 7893 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT; 7894 queue_hotplug = true; 7895 DRM_DEBUG("IH: HPD1\n"); 7896 7897 break; 7898 case 1: 7899 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT)) 7900 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7901 7902 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT; 7903 queue_hotplug = true; 7904 DRM_DEBUG("IH: HPD2\n"); 7905 7906 break; 7907 case 2: 7908 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT)) 7909 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7910 7911 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 7912 queue_hotplug = true; 7913 DRM_DEBUG("IH: HPD3\n"); 7914 7915 break; 7916 case 3: 7917 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT)) 7918 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7919 7920 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 7921 queue_hotplug = true; 7922 DRM_DEBUG("IH: HPD4\n"); 7923 7924 break; 7925 case 4: 7926 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT)) 7927 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7928 7929 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 7930 queue_hotplug = true; 7931 DRM_DEBUG("IH: HPD5\n"); 7932 7933 break; 7934 case 5: 7935 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT)) 7936 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7937 7938 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 7939 queue_hotplug = true; 7940 DRM_DEBUG("IH: HPD6\n"); 7941 7942 break; 7943 case 6: 7944 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT)) 7945 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7946 7947 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT; 7948 queue_dp = true; 7949 DRM_DEBUG("IH: HPD_RX 1\n"); 7950 7951 break; 7952 case 7: 7953 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT)) 7954 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7955 7956 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT; 7957 queue_dp = true; 7958 DRM_DEBUG("IH: HPD_RX 2\n"); 7959 7960 break; 7961 case 8: 7962 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT)) 7963 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7964 7965 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT; 7966 queue_dp = true; 7967 DRM_DEBUG("IH: HPD_RX 3\n"); 7968 7969 break; 7970 case 9: 7971 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT)) 7972 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7973 7974 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT; 7975 queue_dp = true; 7976 DRM_DEBUG("IH: HPD_RX 4\n"); 7977 7978 break; 7979 case 10: 7980 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT)) 7981 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7982 7983 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT; 7984 queue_dp = true; 7985 DRM_DEBUG("IH: HPD_RX 5\n"); 7986 7987 break; 7988 case 11: 7989 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT)) 7990 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7991 7992 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT; 7993 queue_dp = true; 7994 DRM_DEBUG("IH: HPD_RX 6\n"); 7995 7996 break; 7997 default: 7998 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7999 break; 8000 } 8001 break; 8002 case 96: 8003 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR)); 8004 WREG32(SRBM_INT_ACK, 0x1); 8005 break; 8006 case 124: /* UVD */ 8007 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 8008 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 8009 break; 8010 case 146: 8011 case 147: 8012 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 8013 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 8014 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT); 8015 /* reset addr and status */ 8016 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 8017 if (addr == 0x0 && status == 0x0) 8018 break; 8019 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 8020 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 8021 addr); 8022 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 8023 status); 8024 cik_vm_decode_fault(rdev, status, addr, mc_client); 8025 break; 8026 case 167: /* VCE */ 8027 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data); 8028 switch (src_data) { 8029 case 0: 8030 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX); 8031 break; 8032 case 1: 8033 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX); 8034 break; 8035 default: 8036 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); 8037 break; 8038 } 8039 break; 8040 case 176: /* GFX RB CP_INT */ 8041 case 177: /* GFX IB CP_INT */ 8042 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 8043 break; 8044 case 181: /* CP EOP event */ 8045 DRM_DEBUG("IH: CP EOP\n"); 8046 /* XXX check the bitfield order! */ 8047 me_id = (ring_id & 0x60) >> 5; 8048 pipe_id = (ring_id & 0x18) >> 3; 8049 queue_id = (ring_id & 0x7) >> 0; 8050 switch (me_id) { 8051 case 0: 8052 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 8053 break; 8054 case 1: 8055 case 2: 8056 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id)) 8057 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 8058 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id)) 8059 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 8060 break; 8061 } 8062 break; 8063 case 184: /* CP Privileged reg access */ 8064 DRM_ERROR("Illegal register access in command stream\n"); 8065 /* XXX check the bitfield order! */ 8066 me_id = (ring_id & 0x60) >> 5; 8067 pipe_id = (ring_id & 0x18) >> 3; 8068 queue_id = (ring_id & 0x7) >> 0; 8069 switch (me_id) { 8070 case 0: 8071 /* This results in a full GPU reset, but all we need to do is soft 8072 * reset the CP for gfx 8073 */ 8074 queue_reset = true; 8075 break; 8076 case 1: 8077 /* XXX compute */ 8078 queue_reset = true; 8079 break; 8080 case 2: 8081 /* XXX compute */ 8082 queue_reset = true; 8083 break; 8084 } 8085 break; 8086 case 185: /* CP Privileged inst */ 8087 DRM_ERROR("Illegal instruction in command stream\n"); 8088 /* XXX check the bitfield order! */ 8089 me_id = (ring_id & 0x60) >> 5; 8090 pipe_id = (ring_id & 0x18) >> 3; 8091 queue_id = (ring_id & 0x7) >> 0; 8092 switch (me_id) { 8093 case 0: 8094 /* This results in a full GPU reset, but all we need to do is soft 8095 * reset the CP for gfx 8096 */ 8097 queue_reset = true; 8098 break; 8099 case 1: 8100 /* XXX compute */ 8101 queue_reset = true; 8102 break; 8103 case 2: 8104 /* XXX compute */ 8105 queue_reset = true; 8106 break; 8107 } 8108 break; 8109 case 224: /* SDMA trap event */ 8110 /* XXX check the bitfield order! */ 8111 me_id = (ring_id & 0x3) >> 0; 8112 queue_id = (ring_id & 0xc) >> 2; 8113 DRM_DEBUG("IH: SDMA trap\n"); 8114 switch (me_id) { 8115 case 0: 8116 switch (queue_id) { 8117 case 0: 8118 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 8119 break; 8120 case 1: 8121 /* XXX compute */ 8122 break; 8123 case 2: 8124 /* XXX compute */ 8125 break; 8126 } 8127 break; 8128 case 1: 8129 switch (queue_id) { 8130 case 0: 8131 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 8132 break; 8133 case 1: 8134 /* XXX compute */ 8135 break; 8136 case 2: 8137 /* XXX compute */ 8138 break; 8139 } 8140 break; 8141 } 8142 break; 8143 case 230: /* thermal low to high */ 8144 DRM_DEBUG("IH: thermal low to high\n"); 8145 rdev->pm.dpm.thermal.high_to_low = false; 8146 queue_thermal = true; 8147 break; 8148 case 231: /* thermal high to low */ 8149 DRM_DEBUG("IH: thermal high to low\n"); 8150 rdev->pm.dpm.thermal.high_to_low = true; 8151 queue_thermal = true; 8152 break; 8153 case 233: /* GUI IDLE */ 8154 DRM_DEBUG("IH: GUI idle\n"); 8155 break; 8156 case 241: /* SDMA Privileged inst */ 8157 case 247: /* SDMA Privileged inst */ 8158 DRM_ERROR("Illegal instruction in SDMA command stream\n"); 8159 /* XXX check the bitfield order! */ 8160 me_id = (ring_id & 0x3) >> 0; 8161 queue_id = (ring_id & 0xc) >> 2; 8162 switch (me_id) { 8163 case 0: 8164 switch (queue_id) { 8165 case 0: 8166 queue_reset = true; 8167 break; 8168 case 1: 8169 /* XXX compute */ 8170 queue_reset = true; 8171 break; 8172 case 2: 8173 /* XXX compute */ 8174 queue_reset = true; 8175 break; 8176 } 8177 break; 8178 case 1: 8179 switch (queue_id) { 8180 case 0: 8181 queue_reset = true; 8182 break; 8183 case 1: 8184 /* XXX compute */ 8185 queue_reset = true; 8186 break; 8187 case 2: 8188 /* XXX compute */ 8189 queue_reset = true; 8190 break; 8191 } 8192 break; 8193 } 8194 break; 8195 default: 8196 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 8197 break; 8198 } 8199 8200 /* wptr/rptr are in bytes! */ 8201 rptr += 16; 8202 rptr &= rdev->ih.ptr_mask; 8203 WREG32(IH_RB_RPTR, rptr); 8204 } 8205 if (queue_dp) 8206 schedule_work(&rdev->dp_work); 8207 if (queue_hotplug) 8208 taskqueue_enqueue(rdev->tq, &rdev->hotplug_work); 8209 if (queue_reset) { 8210 rdev->needs_reset = true; 8211 wake_up_all(&rdev->fence_queue); 8212 } 8213 if (queue_thermal) 8214 taskqueue_enqueue(rdev->tq, &rdev->pm.dpm.thermal.work); 8215 rdev->ih.rptr = rptr; 8216 atomic_set(&rdev->ih.lock, 0); 8217 8218 /* make sure wptr hasn't changed while processing */ 8219 wptr = cik_get_ih_wptr(rdev); 8220 if (wptr != rptr) 8221 goto restart_ih; 8222 8223 return IRQ_HANDLED; 8224 } 8225 8226 /* 8227 * startup/shutdown callbacks 8228 */ 8229 static void cik_uvd_init(struct radeon_device *rdev) 8230 { 8231 int r; 8232 8233 if (!rdev->has_uvd) 8234 return; 8235 8236 r = radeon_uvd_init(rdev); 8237 if (r) { 8238 dev_err(rdev->dev, "failed UVD (%d) init.\n", r); 8239 /* 8240 * At this point rdev->uvd.vcpu_bo is NULL which trickles down 8241 * to early fails cik_uvd_start() and thus nothing happens 8242 * there. So it is pointless to try to go through that code 8243 * hence why we disable uvd here. 8244 */ 8245 rdev->has_uvd = 0; 8246 return; 8247 } 8248 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL; 8249 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096); 8250 } 8251 8252 static void cik_uvd_start(struct radeon_device *rdev) 8253 { 8254 int r; 8255 8256 if (!rdev->has_uvd) 8257 return; 8258 8259 r = radeon_uvd_resume(rdev); 8260 if (r) { 8261 dev_err(rdev->dev, "failed UVD resume (%d).\n", r); 8262 goto error; 8263 } 8264 r = uvd_v4_2_resume(rdev); 8265 if (r) { 8266 dev_err(rdev->dev, "failed UVD 4.2 resume (%d).\n", r); 8267 goto error; 8268 } 8269 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX); 8270 if (r) { 8271 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r); 8272 goto error; 8273 } 8274 return; 8275 8276 error: 8277 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 8278 } 8279 8280 static void cik_uvd_resume(struct radeon_device *rdev) 8281 { 8282 struct radeon_ring *ring; 8283 int r; 8284 8285 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size) 8286 return; 8287 8288 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 8289 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, RADEON_CP_PACKET2); 8290 if (r) { 8291 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r); 8292 return; 8293 } 8294 r = uvd_v1_0_init(rdev); 8295 if (r) { 8296 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r); 8297 return; 8298 } 8299 } 8300 8301 static void cik_vce_init(struct radeon_device *rdev) 8302 { 8303 int r; 8304 8305 if (!rdev->has_vce) 8306 return; 8307 8308 r = radeon_vce_init(rdev); 8309 if (r) { 8310 dev_err(rdev->dev, "failed VCE (%d) init.\n", r); 8311 /* 8312 * At this point rdev->vce.vcpu_bo is NULL which trickles down 8313 * to early fails cik_vce_start() and thus nothing happens 8314 * there. So it is pointless to try to go through that code 8315 * hence why we disable vce here. 8316 */ 8317 rdev->has_vce = 0; 8318 return; 8319 } 8320 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL; 8321 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096); 8322 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL; 8323 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096); 8324 } 8325 8326 static void cik_vce_start(struct radeon_device *rdev) 8327 { 8328 int r; 8329 8330 if (!rdev->has_vce) 8331 return; 8332 8333 r = radeon_vce_resume(rdev); 8334 if (r) { 8335 dev_err(rdev->dev, "failed VCE resume (%d).\n", r); 8336 goto error; 8337 } 8338 r = vce_v2_0_resume(rdev); 8339 if (r) { 8340 dev_err(rdev->dev, "failed VCE resume (%d).\n", r); 8341 goto error; 8342 } 8343 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX); 8344 if (r) { 8345 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r); 8346 goto error; 8347 } 8348 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX); 8349 if (r) { 8350 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r); 8351 goto error; 8352 } 8353 return; 8354 8355 error: 8356 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0; 8357 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0; 8358 } 8359 8360 static void cik_vce_resume(struct radeon_device *rdev) 8361 { 8362 struct radeon_ring *ring; 8363 int r; 8364 8365 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size) 8366 return; 8367 8368 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; 8369 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP); 8370 if (r) { 8371 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r); 8372 return; 8373 } 8374 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; 8375 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP); 8376 if (r) { 8377 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r); 8378 return; 8379 } 8380 r = vce_v1_0_init(rdev); 8381 if (r) { 8382 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r); 8383 return; 8384 } 8385 } 8386 8387 /** 8388 * cik_startup - program the asic to a functional state 8389 * 8390 * @rdev: radeon_device pointer 8391 * 8392 * Programs the asic to a functional state (CIK). 8393 * Called by cik_init() and cik_resume(). 8394 * Returns 0 for success, error for failure. 8395 */ 8396 static int cik_startup(struct radeon_device *rdev) 8397 { 8398 struct radeon_ring *ring; 8399 u32 nop; 8400 int r; 8401 8402 /* enable pcie gen2/3 link */ 8403 cik_pcie_gen3_enable(rdev); 8404 /* enable aspm */ 8405 cik_program_aspm(rdev); 8406 8407 /* scratch needs to be initialized before MC */ 8408 r = r600_vram_scratch_init(rdev); 8409 if (r) 8410 return r; 8411 8412 cik_mc_program(rdev); 8413 8414 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) { 8415 r = ci_mc_load_microcode(rdev); 8416 if (r) { 8417 DRM_ERROR("Failed to load MC firmware!\n"); 8418 return r; 8419 } 8420 } 8421 8422 r = cik_pcie_gart_enable(rdev); 8423 if (r) 8424 return r; 8425 cik_gpu_init(rdev); 8426 8427 /* allocate rlc buffers */ 8428 if (rdev->flags & RADEON_IS_IGP) { 8429 if (rdev->family == CHIP_KAVERI) { 8430 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list; 8431 rdev->rlc.reg_list_size = 8432 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list); 8433 } else { 8434 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list; 8435 rdev->rlc.reg_list_size = 8436 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list); 8437 } 8438 } 8439 rdev->rlc.cs_data = ci_cs_data; 8440 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4; 8441 r = sumo_rlc_init(rdev); 8442 if (r) { 8443 DRM_ERROR("Failed to init rlc BOs!\n"); 8444 return r; 8445 } 8446 8447 /* allocate wb buffer */ 8448 r = radeon_wb_init(rdev); 8449 if (r) 8450 return r; 8451 8452 /* allocate mec buffers */ 8453 r = cik_mec_init(rdev); 8454 if (r) { 8455 DRM_ERROR("Failed to init MEC BOs!\n"); 8456 return r; 8457 } 8458 8459 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 8460 if (r) { 8461 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8462 return r; 8463 } 8464 8465 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 8466 if (r) { 8467 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8468 return r; 8469 } 8470 8471 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 8472 if (r) { 8473 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8474 return r; 8475 } 8476 8477 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 8478 if (r) { 8479 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 8480 return r; 8481 } 8482 8483 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 8484 if (r) { 8485 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 8486 return r; 8487 } 8488 8489 cik_uvd_start(rdev); 8490 cik_vce_start(rdev); 8491 8492 /* Enable IRQ */ 8493 if (!rdev->irq.installed) { 8494 r = radeon_irq_kms_init(rdev); 8495 if (r) 8496 return r; 8497 } 8498 8499 r = cik_irq_init(rdev); 8500 if (r) { 8501 DRM_ERROR("radeon: IH init failed (%d).\n", r); 8502 radeon_irq_kms_fini(rdev); 8503 return r; 8504 } 8505 cik_irq_set(rdev); 8506 8507 if (rdev->family == CHIP_HAWAII) { 8508 if (rdev->new_fw) 8509 nop = PACKET3(PACKET3_NOP, 0x3FFF); 8510 else 8511 nop = RADEON_CP_PACKET2; 8512 } else { 8513 nop = PACKET3(PACKET3_NOP, 0x3FFF); 8514 } 8515 8516 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 8517 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 8518 nop); 8519 if (r) 8520 return r; 8521 8522 /* set up the compute queues */ 8523 /* type-2 packets are deprecated on MEC, use type-3 instead */ 8524 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 8525 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, 8526 nop); 8527 if (r) 8528 return r; 8529 ring->me = 1; /* first MEC */ 8530 ring->pipe = 0; /* first pipe */ 8531 ring->queue = 0; /* first queue */ 8532 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET; 8533 8534 /* type-2 packets are deprecated on MEC, use type-3 instead */ 8535 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 8536 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, 8537 nop); 8538 if (r) 8539 return r; 8540 /* dGPU only have 1 MEC */ 8541 ring->me = 1; /* first MEC */ 8542 ring->pipe = 0; /* first pipe */ 8543 ring->queue = 1; /* second queue */ 8544 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET; 8545 8546 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 8547 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 8548 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); 8549 if (r) 8550 return r; 8551 8552 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 8553 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, 8554 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); 8555 if (r) 8556 return r; 8557 8558 r = cik_cp_resume(rdev); 8559 if (r) 8560 return r; 8561 8562 r = cik_sdma_resume(rdev); 8563 if (r) 8564 return r; 8565 8566 cik_uvd_resume(rdev); 8567 cik_vce_resume(rdev); 8568 8569 r = radeon_ib_pool_init(rdev); 8570 if (r) { 8571 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 8572 return r; 8573 } 8574 8575 r = radeon_vm_manager_init(rdev); 8576 if (r) { 8577 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); 8578 return r; 8579 } 8580 8581 r = radeon_audio_init(rdev); 8582 if (r) 8583 return r; 8584 8585 return 0; 8586 } 8587 8588 /** 8589 * cik_resume - resume the asic to a functional state 8590 * 8591 * @rdev: radeon_device pointer 8592 * 8593 * Programs the asic to a functional state (CIK). 8594 * Called at resume. 8595 * Returns 0 for success, error for failure. 8596 */ 8597 int cik_resume(struct radeon_device *rdev) 8598 { 8599 int r; 8600 8601 /* post card */ 8602 atom_asic_init(rdev->mode_info.atom_context); 8603 8604 /* init golden registers */ 8605 cik_init_golden_registers(rdev); 8606 8607 if (rdev->pm.pm_method == PM_METHOD_DPM) 8608 radeon_pm_resume(rdev); 8609 8610 rdev->accel_working = true; 8611 r = cik_startup(rdev); 8612 if (r) { 8613 DRM_ERROR("cik startup failed on resume\n"); 8614 rdev->accel_working = false; 8615 return r; 8616 } 8617 8618 return r; 8619 8620 } 8621 8622 /** 8623 * cik_suspend - suspend the asic 8624 * 8625 * @rdev: radeon_device pointer 8626 * 8627 * Bring the chip into a state suitable for suspend (CIK). 8628 * Called at suspend. 8629 * Returns 0 for success. 8630 */ 8631 int cik_suspend(struct radeon_device *rdev) 8632 { 8633 radeon_pm_suspend(rdev); 8634 radeon_audio_fini(rdev); 8635 radeon_vm_manager_fini(rdev); 8636 cik_cp_enable(rdev, false); 8637 cik_sdma_enable(rdev, false); 8638 if (rdev->has_uvd) { 8639 uvd_v1_0_fini(rdev); 8640 radeon_uvd_suspend(rdev); 8641 } 8642 if (rdev->has_vce) 8643 radeon_vce_suspend(rdev); 8644 cik_fini_pg(rdev); 8645 cik_fini_cg(rdev); 8646 cik_irq_suspend(rdev); 8647 radeon_wb_disable(rdev); 8648 cik_pcie_gart_disable(rdev); 8649 return 0; 8650 } 8651 8652 /* Plan is to move initialization in that function and use 8653 * helper function so that radeon_device_init pretty much 8654 * do nothing more than calling asic specific function. This 8655 * should also allow to remove a bunch of callback function 8656 * like vram_info. 8657 */ 8658 /** 8659 * cik_init - asic specific driver and hw init 8660 * 8661 * @rdev: radeon_device pointer 8662 * 8663 * Setup asic specific driver variables and program the hw 8664 * to a functional state (CIK). 8665 * Called at driver startup. 8666 * Returns 0 for success, errors for failure. 8667 */ 8668 int cik_init(struct radeon_device *rdev) 8669 { 8670 struct radeon_ring *ring; 8671 int r; 8672 8673 /* Read BIOS */ 8674 if (!radeon_get_bios(rdev)) { 8675 if (ASIC_IS_AVIVO(rdev)) 8676 return -EINVAL; 8677 } 8678 /* Must be an ATOMBIOS */ 8679 if (!rdev->is_atom_bios) { 8680 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); 8681 return -EINVAL; 8682 } 8683 r = radeon_atombios_init(rdev); 8684 if (r) 8685 return r; 8686 8687 /* Post card if necessary */ 8688 if (!radeon_card_posted(rdev)) { 8689 if (!rdev->bios) { 8690 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 8691 return -EINVAL; 8692 } 8693 DRM_INFO("GPU not posted. posting now...\n"); 8694 atom_asic_init(rdev->mode_info.atom_context); 8695 } 8696 /* init golden registers */ 8697 cik_init_golden_registers(rdev); 8698 /* Initialize scratch registers */ 8699 cik_scratch_init(rdev); 8700 /* Initialize surface registers */ 8701 radeon_surface_init(rdev); 8702 /* Initialize clocks */ 8703 radeon_get_clock_info(rdev->ddev); 8704 8705 /* Fence driver */ 8706 r = radeon_fence_driver_init(rdev); 8707 if (r) 8708 return r; 8709 8710 /* initialize memory controller */ 8711 r = cik_mc_init(rdev); 8712 if (r) 8713 return r; 8714 /* Memory manager */ 8715 r = radeon_bo_init(rdev); 8716 if (r) 8717 return r; 8718 8719 if (rdev->flags & RADEON_IS_IGP) { 8720 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 8721 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) { 8722 r = cik_init_microcode(rdev); 8723 if (r) { 8724 DRM_ERROR("Failed to load firmware!\n"); 8725 return r; 8726 } 8727 } 8728 } else { 8729 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 8730 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw || 8731 !rdev->mc_fw) { 8732 r = cik_init_microcode(rdev); 8733 if (r) { 8734 DRM_ERROR("Failed to load firmware!\n"); 8735 return r; 8736 } 8737 } 8738 } 8739 8740 /* Initialize power management */ 8741 radeon_pm_init(rdev); 8742 8743 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 8744 ring->ring_obj = NULL; 8745 r600_ring_init(rdev, ring, 1024 * 1024); 8746 8747 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 8748 ring->ring_obj = NULL; 8749 r600_ring_init(rdev, ring, 1024 * 1024); 8750 r = radeon_doorbell_get(rdev, &ring->doorbell_index); 8751 if (r) 8752 return r; 8753 8754 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 8755 ring->ring_obj = NULL; 8756 r600_ring_init(rdev, ring, 1024 * 1024); 8757 r = radeon_doorbell_get(rdev, &ring->doorbell_index); 8758 if (r) 8759 return r; 8760 8761 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 8762 ring->ring_obj = NULL; 8763 r600_ring_init(rdev, ring, 256 * 1024); 8764 8765 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 8766 ring->ring_obj = NULL; 8767 r600_ring_init(rdev, ring, 256 * 1024); 8768 8769 cik_uvd_init(rdev); 8770 cik_vce_init(rdev); 8771 8772 rdev->ih.ring_obj = NULL; 8773 r600_ih_ring_init(rdev, 64 * 1024); 8774 8775 r = r600_pcie_gart_init(rdev); 8776 if (r) 8777 return r; 8778 8779 rdev->accel_working = true; 8780 r = cik_startup(rdev); 8781 if (r) { 8782 dev_err(rdev->dev, "disabling GPU acceleration\n"); 8783 cik_cp_fini(rdev); 8784 cik_sdma_fini(rdev); 8785 cik_irq_fini(rdev); 8786 sumo_rlc_fini(rdev); 8787 cik_mec_fini(rdev); 8788 radeon_wb_fini(rdev); 8789 radeon_ib_pool_fini(rdev); 8790 radeon_vm_manager_fini(rdev); 8791 radeon_irq_kms_fini(rdev); 8792 cik_pcie_gart_fini(rdev); 8793 rdev->accel_working = false; 8794 } 8795 8796 /* Don't start up if the MC ucode is missing. 8797 * The default clocks and voltages before the MC ucode 8798 * is loaded are not suffient for advanced operations. 8799 */ 8800 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 8801 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 8802 return -EINVAL; 8803 } 8804 8805 return 0; 8806 } 8807 8808 /** 8809 * cik_fini - asic specific driver and hw fini 8810 * 8811 * @rdev: radeon_device pointer 8812 * 8813 * Tear down the asic specific driver variables and program the hw 8814 * to an idle state (CIK). 8815 * Called at driver unload. 8816 */ 8817 void cik_fini(struct radeon_device *rdev) 8818 { 8819 radeon_pm_fini(rdev); 8820 cik_cp_fini(rdev); 8821 cik_sdma_fini(rdev); 8822 cik_fini_pg(rdev); 8823 cik_fini_cg(rdev); 8824 cik_irq_fini(rdev); 8825 sumo_rlc_fini(rdev); 8826 cik_mec_fini(rdev); 8827 radeon_wb_fini(rdev); 8828 radeon_vm_manager_fini(rdev); 8829 radeon_ib_pool_fini(rdev); 8830 radeon_irq_kms_fini(rdev); 8831 uvd_v1_0_fini(rdev); 8832 radeon_uvd_fini(rdev); 8833 radeon_vce_fini(rdev); 8834 cik_pcie_gart_fini(rdev); 8835 r600_vram_scratch_fini(rdev); 8836 radeon_gem_fini(rdev); 8837 radeon_fence_driver_fini(rdev); 8838 radeon_bo_fini(rdev); 8839 radeon_atombios_fini(rdev); 8840 cik_fini_microcode(rdev); 8841 kfree(rdev->bios); 8842 rdev->bios = NULL; 8843 } 8844 8845 void dce8_program_fmt(struct drm_encoder *encoder) 8846 { 8847 struct drm_device *dev = encoder->dev; 8848 struct radeon_device *rdev = dev->dev_private; 8849 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); 8850 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); 8851 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); 8852 int bpc = 0; 8853 u32 tmp = 0; 8854 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; 8855 8856 if (connector) { 8857 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 8858 bpc = radeon_get_monitor_bpc(connector); 8859 dither = radeon_connector->dither; 8860 } 8861 8862 /* LVDS/eDP FMT is set up by atom */ 8863 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) 8864 return; 8865 8866 /* not needed for analog */ 8867 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || 8868 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) 8869 return; 8870 8871 if (bpc == 0) 8872 return; 8873 8874 switch (bpc) { 8875 case 6: 8876 if (dither == RADEON_FMT_DITHER_ENABLE) 8877 /* XXX sort out optimal dither settings */ 8878 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8879 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0)); 8880 else 8881 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0)); 8882 break; 8883 case 8: 8884 if (dither == RADEON_FMT_DITHER_ENABLE) 8885 /* XXX sort out optimal dither settings */ 8886 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8887 FMT_RGB_RANDOM_ENABLE | 8888 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1)); 8889 else 8890 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1)); 8891 break; 8892 case 10: 8893 if (dither == RADEON_FMT_DITHER_ENABLE) 8894 /* XXX sort out optimal dither settings */ 8895 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8896 FMT_RGB_RANDOM_ENABLE | 8897 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2)); 8898 else 8899 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2)); 8900 break; 8901 default: 8902 /* not needed */ 8903 break; 8904 } 8905 8906 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); 8907 } 8908 8909 /* display watermark setup */ 8910 /** 8911 * dce8_line_buffer_adjust - Set up the line buffer 8912 * 8913 * @rdev: radeon_device pointer 8914 * @radeon_crtc: the selected display controller 8915 * @mode: the current display mode on the selected display 8916 * controller 8917 * 8918 * Setup up the line buffer allocation for 8919 * the selected display controller (CIK). 8920 * Returns the line buffer size in pixels. 8921 */ 8922 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev, 8923 struct radeon_crtc *radeon_crtc, 8924 struct drm_display_mode *mode) 8925 { 8926 u32 tmp, buffer_alloc, i; 8927 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 8928 /* 8929 * Line Buffer Setup 8930 * There are 6 line buffers, one for each display controllers. 8931 * There are 3 partitions per LB. Select the number of partitions 8932 * to enable based on the display width. For display widths larger 8933 * than 4096, you need use to use 2 display controllers and combine 8934 * them using the stereo blender. 8935 */ 8936 if (radeon_crtc->base.enabled && mode) { 8937 if (mode->crtc_hdisplay < 1920) { 8938 tmp = 1; 8939 buffer_alloc = 2; 8940 } else if (mode->crtc_hdisplay < 2560) { 8941 tmp = 2; 8942 buffer_alloc = 2; 8943 } else if (mode->crtc_hdisplay < 4096) { 8944 tmp = 0; 8945 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; 8946 } else { 8947 DRM_DEBUG_KMS("Mode too big for LB!\n"); 8948 tmp = 0; 8949 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; 8950 } 8951 } else { 8952 tmp = 1; 8953 buffer_alloc = 0; 8954 } 8955 8956 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset, 8957 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0)); 8958 8959 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 8960 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 8961 for (i = 0; i < rdev->usec_timeout; i++) { 8962 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 8963 DMIF_BUFFERS_ALLOCATED_COMPLETED) 8964 break; 8965 udelay(1); 8966 } 8967 8968 if (radeon_crtc->base.enabled && mode) { 8969 switch (tmp) { 8970 case 0: 8971 default: 8972 return 4096 * 2; 8973 case 1: 8974 return 1920 * 2; 8975 case 2: 8976 return 2560 * 2; 8977 } 8978 } 8979 8980 /* controller not enabled, so no lb used */ 8981 return 0; 8982 } 8983 8984 /** 8985 * cik_get_number_of_dram_channels - get the number of dram channels 8986 * 8987 * @rdev: radeon_device pointer 8988 * 8989 * Look up the number of video ram channels (CIK). 8990 * Used for display watermark bandwidth calculations 8991 * Returns the number of dram channels 8992 */ 8993 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev) 8994 { 8995 u32 tmp = RREG32(MC_SHARED_CHMAP); 8996 8997 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 8998 case 0: 8999 default: 9000 return 1; 9001 case 1: 9002 return 2; 9003 case 2: 9004 return 4; 9005 case 3: 9006 return 8; 9007 case 4: 9008 return 3; 9009 case 5: 9010 return 6; 9011 case 6: 9012 return 10; 9013 case 7: 9014 return 12; 9015 case 8: 9016 return 16; 9017 } 9018 } 9019 9020 struct dce8_wm_params { 9021 u32 dram_channels; /* number of dram channels */ 9022 u32 yclk; /* bandwidth per dram data pin in kHz */ 9023 u32 sclk; /* engine clock in kHz */ 9024 u32 disp_clk; /* display clock in kHz */ 9025 u32 src_width; /* viewport width */ 9026 u32 active_time; /* active display time in ns */ 9027 u32 blank_time; /* blank time in ns */ 9028 bool interlaced; /* mode is interlaced */ 9029 fixed20_12 vsc; /* vertical scale ratio */ 9030 u32 num_heads; /* number of active crtcs */ 9031 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 9032 u32 lb_size; /* line buffer allocated to pipe */ 9033 u32 vtaps; /* vertical scaler taps */ 9034 }; 9035 9036 /** 9037 * dce8_dram_bandwidth - get the dram bandwidth 9038 * 9039 * @wm: watermark calculation data 9040 * 9041 * Calculate the raw dram bandwidth (CIK). 9042 * Used for display watermark bandwidth calculations 9043 * Returns the dram bandwidth in MBytes/s 9044 */ 9045 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm) 9046 { 9047 /* Calculate raw DRAM Bandwidth */ 9048 fixed20_12 dram_efficiency; /* 0.7 */ 9049 fixed20_12 yclk, dram_channels, bandwidth; 9050 fixed20_12 a; 9051 9052 a.full = dfixed_const(1000); 9053 yclk.full = dfixed_const(wm->yclk); 9054 yclk.full = dfixed_div(yclk, a); 9055 dram_channels.full = dfixed_const(wm->dram_channels * 4); 9056 a.full = dfixed_const(10); 9057 dram_efficiency.full = dfixed_const(7); 9058 dram_efficiency.full = dfixed_div(dram_efficiency, a); 9059 bandwidth.full = dfixed_mul(dram_channels, yclk); 9060 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 9061 9062 return dfixed_trunc(bandwidth); 9063 } 9064 9065 /** 9066 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display 9067 * 9068 * @wm: watermark calculation data 9069 * 9070 * Calculate the dram bandwidth used for display (CIK). 9071 * Used for display watermark bandwidth calculations 9072 * Returns the dram bandwidth for display in MBytes/s 9073 */ 9074 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm) 9075 { 9076 /* Calculate DRAM Bandwidth and the part allocated to display. */ 9077 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 9078 fixed20_12 yclk, dram_channels, bandwidth; 9079 fixed20_12 a; 9080 9081 a.full = dfixed_const(1000); 9082 yclk.full = dfixed_const(wm->yclk); 9083 yclk.full = dfixed_div(yclk, a); 9084 dram_channels.full = dfixed_const(wm->dram_channels * 4); 9085 a.full = dfixed_const(10); 9086 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 9087 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 9088 bandwidth.full = dfixed_mul(dram_channels, yclk); 9089 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 9090 9091 return dfixed_trunc(bandwidth); 9092 } 9093 9094 /** 9095 * dce8_data_return_bandwidth - get the data return bandwidth 9096 * 9097 * @wm: watermark calculation data 9098 * 9099 * Calculate the data return bandwidth used for display (CIK). 9100 * Used for display watermark bandwidth calculations 9101 * Returns the data return bandwidth in MBytes/s 9102 */ 9103 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm) 9104 { 9105 /* Calculate the display Data return Bandwidth */ 9106 fixed20_12 return_efficiency; /* 0.8 */ 9107 fixed20_12 sclk, bandwidth; 9108 fixed20_12 a; 9109 9110 a.full = dfixed_const(1000); 9111 sclk.full = dfixed_const(wm->sclk); 9112 sclk.full = dfixed_div(sclk, a); 9113 a.full = dfixed_const(10); 9114 return_efficiency.full = dfixed_const(8); 9115 return_efficiency.full = dfixed_div(return_efficiency, a); 9116 a.full = dfixed_const(32); 9117 bandwidth.full = dfixed_mul(a, sclk); 9118 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 9119 9120 return dfixed_trunc(bandwidth); 9121 } 9122 9123 /** 9124 * dce8_dmif_request_bandwidth - get the dmif bandwidth 9125 * 9126 * @wm: watermark calculation data 9127 * 9128 * Calculate the dmif bandwidth used for display (CIK). 9129 * Used for display watermark bandwidth calculations 9130 * Returns the dmif bandwidth in MBytes/s 9131 */ 9132 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm) 9133 { 9134 /* Calculate the DMIF Request Bandwidth */ 9135 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 9136 fixed20_12 disp_clk, bandwidth; 9137 fixed20_12 a, b; 9138 9139 a.full = dfixed_const(1000); 9140 disp_clk.full = dfixed_const(wm->disp_clk); 9141 disp_clk.full = dfixed_div(disp_clk, a); 9142 a.full = dfixed_const(32); 9143 b.full = dfixed_mul(a, disp_clk); 9144 9145 a.full = dfixed_const(10); 9146 disp_clk_request_efficiency.full = dfixed_const(8); 9147 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 9148 9149 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency); 9150 9151 return dfixed_trunc(bandwidth); 9152 } 9153 9154 /** 9155 * dce8_available_bandwidth - get the min available bandwidth 9156 * 9157 * @wm: watermark calculation data 9158 * 9159 * Calculate the min available bandwidth used for display (CIK). 9160 * Used for display watermark bandwidth calculations 9161 * Returns the min available bandwidth in MBytes/s 9162 */ 9163 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm) 9164 { 9165 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 9166 u32 dram_bandwidth = dce8_dram_bandwidth(wm); 9167 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm); 9168 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm); 9169 9170 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 9171 } 9172 9173 /** 9174 * dce8_average_bandwidth - get the average available bandwidth 9175 * 9176 * @wm: watermark calculation data 9177 * 9178 * Calculate the average available bandwidth used for display (CIK). 9179 * Used for display watermark bandwidth calculations 9180 * Returns the average available bandwidth in MBytes/s 9181 */ 9182 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm) 9183 { 9184 /* Calculate the display mode Average Bandwidth 9185 * DisplayMode should contain the source and destination dimensions, 9186 * timing, etc. 9187 */ 9188 fixed20_12 bpp; 9189 fixed20_12 line_time; 9190 fixed20_12 src_width; 9191 fixed20_12 bandwidth; 9192 fixed20_12 a; 9193 9194 a.full = dfixed_const(1000); 9195 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 9196 line_time.full = dfixed_div(line_time, a); 9197 bpp.full = dfixed_const(wm->bytes_per_pixel); 9198 src_width.full = dfixed_const(wm->src_width); 9199 bandwidth.full = dfixed_mul(src_width, bpp); 9200 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 9201 bandwidth.full = dfixed_div(bandwidth, line_time); 9202 9203 return dfixed_trunc(bandwidth); 9204 } 9205 9206 /** 9207 * dce8_latency_watermark - get the latency watermark 9208 * 9209 * @wm: watermark calculation data 9210 * 9211 * Calculate the latency watermark (CIK). 9212 * Used for display watermark bandwidth calculations 9213 * Returns the latency watermark in ns 9214 */ 9215 static u32 dce8_latency_watermark(struct dce8_wm_params *wm) 9216 { 9217 /* First calculate the latency in ns */ 9218 u32 mc_latency = 2000; /* 2000 ns. */ 9219 u32 available_bandwidth = dce8_available_bandwidth(wm); 9220 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 9221 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 9222 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 9223 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 9224 (wm->num_heads * cursor_line_pair_return_time); 9225 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 9226 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 9227 u32 tmp, dmif_size = 12288; 9228 fixed20_12 a, b, c; 9229 9230 if (wm->num_heads == 0) 9231 return 0; 9232 9233 a.full = dfixed_const(2); 9234 b.full = dfixed_const(1); 9235 if ((wm->vsc.full > a.full) || 9236 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 9237 (wm->vtaps >= 5) || 9238 ((wm->vsc.full >= a.full) && wm->interlaced)) 9239 max_src_lines_per_dst_line = 4; 9240 else 9241 max_src_lines_per_dst_line = 2; 9242 9243 a.full = dfixed_const(available_bandwidth); 9244 b.full = dfixed_const(wm->num_heads); 9245 a.full = dfixed_div(a, b); 9246 9247 b.full = dfixed_const(mc_latency + 512); 9248 c.full = dfixed_const(wm->disp_clk); 9249 b.full = dfixed_div(b, c); 9250 9251 c.full = dfixed_const(dmif_size); 9252 b.full = dfixed_div(c, b); 9253 9254 tmp = min(dfixed_trunc(a), dfixed_trunc(b)); 9255 9256 b.full = dfixed_const(1000); 9257 c.full = dfixed_const(wm->disp_clk); 9258 b.full = dfixed_div(c, b); 9259 c.full = dfixed_const(wm->bytes_per_pixel); 9260 b.full = dfixed_mul(b, c); 9261 9262 lb_fill_bw = min(tmp, dfixed_trunc(b)); 9263 9264 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 9265 b.full = dfixed_const(1000); 9266 c.full = dfixed_const(lb_fill_bw); 9267 b.full = dfixed_div(c, b); 9268 a.full = dfixed_div(a, b); 9269 line_fill_time = dfixed_trunc(a); 9270 9271 if (line_fill_time < wm->active_time) 9272 return latency; 9273 else 9274 return latency + (line_fill_time - wm->active_time); 9275 9276 } 9277 9278 /** 9279 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check 9280 * average and available dram bandwidth 9281 * 9282 * @wm: watermark calculation data 9283 * 9284 * Check if the display average bandwidth fits in the display 9285 * dram bandwidth (CIK). 9286 * Used for display watermark bandwidth calculations 9287 * Returns true if the display fits, false if not. 9288 */ 9289 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm) 9290 { 9291 if (dce8_average_bandwidth(wm) <= 9292 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads)) 9293 return true; 9294 else 9295 return false; 9296 } 9297 9298 /** 9299 * dce8_average_bandwidth_vs_available_bandwidth - check 9300 * average and available bandwidth 9301 * 9302 * @wm: watermark calculation data 9303 * 9304 * Check if the display average bandwidth fits in the display 9305 * available bandwidth (CIK). 9306 * Used for display watermark bandwidth calculations 9307 * Returns true if the display fits, false if not. 9308 */ 9309 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm) 9310 { 9311 if (dce8_average_bandwidth(wm) <= 9312 (dce8_available_bandwidth(wm) / wm->num_heads)) 9313 return true; 9314 else 9315 return false; 9316 } 9317 9318 /** 9319 * dce8_check_latency_hiding - check latency hiding 9320 * 9321 * @wm: watermark calculation data 9322 * 9323 * Check latency hiding (CIK). 9324 * Used for display watermark bandwidth calculations 9325 * Returns true if the display fits, false if not. 9326 */ 9327 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm) 9328 { 9329 u32 lb_partitions = wm->lb_size / wm->src_width; 9330 u32 line_time = wm->active_time + wm->blank_time; 9331 u32 latency_tolerant_lines; 9332 u32 latency_hiding; 9333 fixed20_12 a; 9334 9335 a.full = dfixed_const(1); 9336 if (wm->vsc.full > a.full) 9337 latency_tolerant_lines = 1; 9338 else { 9339 if (lb_partitions <= (wm->vtaps + 1)) 9340 latency_tolerant_lines = 1; 9341 else 9342 latency_tolerant_lines = 2; 9343 } 9344 9345 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 9346 9347 if (dce8_latency_watermark(wm) <= latency_hiding) 9348 return true; 9349 else 9350 return false; 9351 } 9352 9353 /** 9354 * dce8_program_watermarks - program display watermarks 9355 * 9356 * @rdev: radeon_device pointer 9357 * @radeon_crtc: the selected display controller 9358 * @lb_size: line buffer size 9359 * @num_heads: number of display controllers in use 9360 * 9361 * Calculate and program the display watermarks for the 9362 * selected display controller (CIK). 9363 */ 9364 static void dce8_program_watermarks(struct radeon_device *rdev, 9365 struct radeon_crtc *radeon_crtc, 9366 u32 lb_size, u32 num_heads) 9367 { 9368 struct drm_display_mode *mode = &radeon_crtc->base.mode; 9369 struct dce8_wm_params wm_low, wm_high; 9370 u32 pixel_period; 9371 u32 line_time = 0; 9372 u32 latency_watermark_a = 0, latency_watermark_b = 0; 9373 u32 tmp, wm_mask; 9374 9375 if (radeon_crtc->base.enabled && num_heads && mode) { 9376 pixel_period = 1000000 / (u32)mode->clock; 9377 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 9378 9379 /* watermark for high clocks */ 9380 if ((rdev->pm.pm_method == PM_METHOD_DPM) && 9381 rdev->pm.dpm_enabled) { 9382 wm_high.yclk = 9383 radeon_dpm_get_mclk(rdev, false) * 10; 9384 wm_high.sclk = 9385 radeon_dpm_get_sclk(rdev, false) * 10; 9386 } else { 9387 wm_high.yclk = rdev->pm.current_mclk * 10; 9388 wm_high.sclk = rdev->pm.current_sclk * 10; 9389 } 9390 9391 wm_high.disp_clk = mode->clock; 9392 wm_high.src_width = mode->crtc_hdisplay; 9393 wm_high.active_time = mode->crtc_hdisplay * pixel_period; 9394 wm_high.blank_time = line_time - wm_high.active_time; 9395 wm_high.interlaced = false; 9396 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 9397 wm_high.interlaced = true; 9398 wm_high.vsc = radeon_crtc->vsc; 9399 wm_high.vtaps = 1; 9400 if (radeon_crtc->rmx_type != RMX_OFF) 9401 wm_high.vtaps = 2; 9402 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 9403 wm_high.lb_size = lb_size; 9404 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev); 9405 wm_high.num_heads = num_heads; 9406 9407 /* set for high clocks */ 9408 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535); 9409 9410 /* possibly force display priority to high */ 9411 /* should really do this at mode validation time... */ 9412 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 9413 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) || 9414 !dce8_check_latency_hiding(&wm_high) || 9415 (rdev->disp_priority == 2)) { 9416 DRM_DEBUG_KMS("force priority to high\n"); 9417 } 9418 9419 /* watermark for low clocks */ 9420 if ((rdev->pm.pm_method == PM_METHOD_DPM) && 9421 rdev->pm.dpm_enabled) { 9422 wm_low.yclk = 9423 radeon_dpm_get_mclk(rdev, true) * 10; 9424 wm_low.sclk = 9425 radeon_dpm_get_sclk(rdev, true) * 10; 9426 } else { 9427 wm_low.yclk = rdev->pm.current_mclk * 10; 9428 wm_low.sclk = rdev->pm.current_sclk * 10; 9429 } 9430 9431 wm_low.disp_clk = mode->clock; 9432 wm_low.src_width = mode->crtc_hdisplay; 9433 wm_low.active_time = mode->crtc_hdisplay * pixel_period; 9434 wm_low.blank_time = line_time - wm_low.active_time; 9435 wm_low.interlaced = false; 9436 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 9437 wm_low.interlaced = true; 9438 wm_low.vsc = radeon_crtc->vsc; 9439 wm_low.vtaps = 1; 9440 if (radeon_crtc->rmx_type != RMX_OFF) 9441 wm_low.vtaps = 2; 9442 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 9443 wm_low.lb_size = lb_size; 9444 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev); 9445 wm_low.num_heads = num_heads; 9446 9447 /* set for low clocks */ 9448 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535); 9449 9450 /* possibly force display priority to high */ 9451 /* should really do this at mode validation time... */ 9452 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 9453 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) || 9454 !dce8_check_latency_hiding(&wm_low) || 9455 (rdev->disp_priority == 2)) { 9456 DRM_DEBUG_KMS("force priority to high\n"); 9457 } 9458 9459 /* Save number of lines the linebuffer leads before the scanout */ 9460 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay); 9461 } 9462 9463 /* select wm A */ 9464 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); 9465 tmp = wm_mask; 9466 tmp &= ~LATENCY_WATERMARK_MASK(3); 9467 tmp |= LATENCY_WATERMARK_MASK(1); 9468 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); 9469 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 9470 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 9471 LATENCY_HIGH_WATERMARK(line_time))); 9472 /* select wm B */ 9473 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); 9474 tmp &= ~LATENCY_WATERMARK_MASK(3); 9475 tmp |= LATENCY_WATERMARK_MASK(2); 9476 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); 9477 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 9478 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 9479 LATENCY_HIGH_WATERMARK(line_time))); 9480 /* restore original selection */ 9481 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask); 9482 9483 /* save values for DPM */ 9484 radeon_crtc->line_time = line_time; 9485 radeon_crtc->wm_high = latency_watermark_a; 9486 radeon_crtc->wm_low = latency_watermark_b; 9487 } 9488 9489 /** 9490 * dce8_bandwidth_update - program display watermarks 9491 * 9492 * @rdev: radeon_device pointer 9493 * 9494 * Calculate and program the display watermarks and line 9495 * buffer allocation (CIK). 9496 */ 9497 void dce8_bandwidth_update(struct radeon_device *rdev) 9498 { 9499 struct drm_display_mode *mode = NULL; 9500 u32 num_heads = 0, lb_size; 9501 int i; 9502 9503 if (!rdev->mode_info.mode_config_initialized) 9504 return; 9505 9506 radeon_update_display_priority(rdev); 9507 9508 for (i = 0; i < rdev->num_crtc; i++) { 9509 if (rdev->mode_info.crtcs[i]->base.enabled) 9510 num_heads++; 9511 } 9512 for (i = 0; i < rdev->num_crtc; i++) { 9513 mode = &rdev->mode_info.crtcs[i]->base.mode; 9514 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode); 9515 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 9516 } 9517 } 9518 9519 /** 9520 * cik_get_gpu_clock_counter - return GPU clock counter snapshot 9521 * 9522 * @rdev: radeon_device pointer 9523 * 9524 * Fetches a GPU clock counter snapshot (SI). 9525 * Returns the 64 bit clock counter snapshot. 9526 */ 9527 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev) 9528 { 9529 uint64_t clock; 9530 9531 mutex_lock(&rdev->gpu_clock_mutex); 9532 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1); 9533 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) | 9534 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 9535 mutex_unlock(&rdev->gpu_clock_mutex); 9536 return clock; 9537 } 9538 9539 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock, 9540 u32 cntl_reg, u32 status_reg) 9541 { 9542 int r, i; 9543 struct atom_clock_dividers dividers; 9544 uint32_t tmp; 9545 9546 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, 9547 clock, false, ÷rs); 9548 if (r) 9549 return r; 9550 9551 tmp = RREG32_SMC(cntl_reg); 9552 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK); 9553 tmp |= dividers.post_divider; 9554 WREG32_SMC(cntl_reg, tmp); 9555 9556 for (i = 0; i < 100; i++) { 9557 if (RREG32_SMC(status_reg) & DCLK_STATUS) 9558 break; 9559 mdelay(10); 9560 } 9561 if (i == 100) 9562 return -ETIMEDOUT; 9563 9564 return 0; 9565 } 9566 9567 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 9568 { 9569 int r = 0; 9570 9571 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); 9572 if (r) 9573 return r; 9574 9575 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); 9576 return r; 9577 } 9578 9579 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk) 9580 { 9581 int r, i; 9582 struct atom_clock_dividers dividers; 9583 u32 tmp; 9584 9585 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, 9586 ecclk, false, ÷rs); 9587 if (r) 9588 return r; 9589 9590 for (i = 0; i < 100; i++) { 9591 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) 9592 break; 9593 mdelay(10); 9594 } 9595 if (i == 100) 9596 return -ETIMEDOUT; 9597 9598 tmp = RREG32_SMC(CG_ECLK_CNTL); 9599 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK); 9600 tmp |= dividers.post_divider; 9601 WREG32_SMC(CG_ECLK_CNTL, tmp); 9602 9603 for (i = 0; i < 100; i++) { 9604 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) 9605 break; 9606 mdelay(10); 9607 } 9608 if (i == 100) 9609 return -ETIMEDOUT; 9610 9611 return 0; 9612 } 9613 9614 static void cik_pcie_gen3_enable(struct radeon_device *rdev) 9615 { 9616 struct pci_dev *root = rdev->pdev->bus->self; 9617 int bridge_pos, gpu_pos; 9618 u32 speed_cntl, mask, current_data_rate; 9619 int ret, i; 9620 u16 tmp16; 9621 9622 if (radeon_pcie_gen2 == 0) 9623 return; 9624 9625 if (rdev->flags & RADEON_IS_IGP) 9626 return; 9627 9628 if (!(rdev->flags & RADEON_IS_PCIE)) 9629 return; 9630 9631 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); 9632 if (ret != 0) 9633 return; 9634 9635 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80))) 9636 return; 9637 9638 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9639 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >> 9640 LC_CURRENT_DATA_RATE_SHIFT; 9641 if (mask & DRM_PCIE_SPEED_80) { 9642 if (current_data_rate == 2) { 9643 DRM_INFO("PCIE gen 3 link speeds already enabled\n"); 9644 return; 9645 } 9646 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n"); 9647 } else if (mask & DRM_PCIE_SPEED_50) { 9648 if (current_data_rate == 1) { 9649 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 9650 return; 9651 } 9652 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 9653 } 9654 9655 bridge_pos = pci_get_pciecap_ptr(root->dev.bsddev); 9656 if (!bridge_pos) 9657 return; 9658 9659 gpu_pos = pci_get_pciecap_ptr(rdev->pdev->dev.bsddev); 9660 if (!gpu_pos) 9661 return; 9662 9663 if (mask & DRM_PCIE_SPEED_80) { 9664 /* re-try equalization if gen3 is not already enabled */ 9665 if (current_data_rate != 2) { 9666 u16 bridge_cfg, gpu_cfg; 9667 u16 bridge_cfg2, gpu_cfg2; 9668 u32 max_lw, current_lw, tmp; 9669 9670 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 9671 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 9672 9673 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD; 9674 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 9675 9676 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD; 9677 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 9678 9679 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1); 9680 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT; 9681 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT; 9682 9683 if (current_lw < max_lw) { 9684 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 9685 if (tmp & LC_RENEGOTIATION_SUPPORT) { 9686 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS); 9687 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT); 9688 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW; 9689 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp); 9690 } 9691 } 9692 9693 for (i = 0; i < 10; i++) { 9694 /* check status */ 9695 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); 9696 if (tmp16 & PCI_EXP_DEVSTA_TRPND) 9697 break; 9698 9699 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 9700 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 9701 9702 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2); 9703 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); 9704 9705 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9706 tmp |= LC_SET_QUIESCE; 9707 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9708 9709 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9710 tmp |= LC_REDO_EQ; 9711 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9712 9713 mdelay(100); 9714 9715 /* linkctl */ 9716 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16); 9717 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 9718 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD); 9719 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 9720 9721 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); 9722 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 9723 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD); 9724 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 9725 9726 /* linkctl2 */ 9727 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16); 9728 tmp16 &= ~((1 << 4) | (7 << 9)); 9729 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9))); 9730 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16); 9731 9732 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 9733 tmp16 &= ~((1 << 4) | (7 << 9)); 9734 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9))); 9735 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 9736 9737 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9738 tmp &= ~LC_SET_QUIESCE; 9739 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9740 } 9741 } 9742 } 9743 9744 /* set the link speed */ 9745 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE; 9746 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE; 9747 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 9748 9749 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 9750 tmp16 &= ~0xf; 9751 if (mask & DRM_PCIE_SPEED_80) 9752 tmp16 |= 3; /* gen3 */ 9753 else if (mask & DRM_PCIE_SPEED_50) 9754 tmp16 |= 2; /* gen2 */ 9755 else 9756 tmp16 |= 1; /* gen1 */ 9757 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 9758 9759 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9760 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE; 9761 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 9762 9763 for (i = 0; i < rdev->usec_timeout; i++) { 9764 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9765 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0) 9766 break; 9767 udelay(1); 9768 } 9769 } 9770 9771 static void cik_program_aspm(struct radeon_device *rdev) 9772 { 9773 u32 data, orig; 9774 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false; 9775 bool disable_clkreq = false; 9776 9777 if (radeon_aspm == 0) 9778 return; 9779 9780 /* XXX double check IGPs */ 9781 if (rdev->flags & RADEON_IS_IGP) 9782 return; 9783 9784 if (!(rdev->flags & RADEON_IS_PCIE)) 9785 return; 9786 9787 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 9788 data &= ~LC_XMIT_N_FTS_MASK; 9789 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN; 9790 if (orig != data) 9791 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data); 9792 9793 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3); 9794 data |= LC_GO_TO_RECOVERY; 9795 if (orig != data) 9796 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data); 9797 9798 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL); 9799 data |= P_IGNORE_EDB_ERR; 9800 if (orig != data) 9801 WREG32_PCIE_PORT(PCIE_P_CNTL, data); 9802 9803 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 9804 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 9805 data |= LC_PMI_TO_L1_DIS; 9806 if (!disable_l0s) 9807 data |= LC_L0S_INACTIVITY(7); 9808 9809 if (!disable_l1) { 9810 data |= LC_L1_INACTIVITY(7); 9811 data &= ~LC_PMI_TO_L1_DIS; 9812 if (orig != data) 9813 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9814 9815 if (!disable_plloff_in_l1) { 9816 bool clk_req_support; 9817 9818 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0); 9819 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 9820 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 9821 if (orig != data) 9822 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data); 9823 9824 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1); 9825 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 9826 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 9827 if (orig != data) 9828 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data); 9829 9830 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0); 9831 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 9832 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 9833 if (orig != data) 9834 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data); 9835 9836 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1); 9837 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 9838 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 9839 if (orig != data) 9840 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data); 9841 9842 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 9843 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 9844 data |= LC_DYN_LANES_PWR_STATE(3); 9845 if (orig != data) 9846 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 9847 9848 if (!disable_clkreq) { 9849 #ifdef zMN_TODO 9850 struct pci_dev *root = rdev->pdev->bus->self; 9851 u32 lnkcap; 9852 9853 clk_req_support = false; 9854 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap); 9855 if (lnkcap & PCI_EXP_LNKCAP_CLKPM) 9856 clk_req_support = true; 9857 #else 9858 clk_req_support = false; 9859 #endif 9860 } else { 9861 clk_req_support = false; 9862 } 9863 9864 if (clk_req_support) { 9865 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2); 9866 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23; 9867 if (orig != data) 9868 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data); 9869 9870 orig = data = RREG32_SMC(THM_CLK_CNTL); 9871 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK); 9872 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1); 9873 if (orig != data) 9874 WREG32_SMC(THM_CLK_CNTL, data); 9875 9876 orig = data = RREG32_SMC(MISC_CLK_CTRL); 9877 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK); 9878 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1); 9879 if (orig != data) 9880 WREG32_SMC(MISC_CLK_CTRL, data); 9881 9882 orig = data = RREG32_SMC(CG_CLKPIN_CNTL); 9883 data &= ~BCLK_AS_XCLK; 9884 if (orig != data) 9885 WREG32_SMC(CG_CLKPIN_CNTL, data); 9886 9887 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2); 9888 data &= ~FORCE_BIF_REFCLK_EN; 9889 if (orig != data) 9890 WREG32_SMC(CG_CLKPIN_CNTL_2, data); 9891 9892 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL); 9893 data &= ~MPLL_CLKOUT_SEL_MASK; 9894 data |= MPLL_CLKOUT_SEL(4); 9895 if (orig != data) 9896 WREG32_SMC(MPLL_BYPASSCLK_SEL, data); 9897 } 9898 } 9899 } else { 9900 if (orig != data) 9901 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9902 } 9903 9904 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); 9905 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN; 9906 if (orig != data) 9907 WREG32_PCIE_PORT(PCIE_CNTL2, data); 9908 9909 if (!disable_l0s) { 9910 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 9911 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) { 9912 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1); 9913 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) { 9914 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 9915 data &= ~LC_L0S_INACTIVITY_MASK; 9916 if (orig != data) 9917 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9918 } 9919 } 9920 } 9921 } 9922