1 /* 2 * Copyright 2012 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <linux/module.h> 26 #include <drm/drmP.h> 27 #include "radeon.h" 28 #include "radeon_asic.h" 29 #include "cikd.h" 30 #include "atom.h" 31 #include "cik_blit_shaders.h" 32 #include "radeon_ucode.h" 33 #include "clearstate_ci.h" 34 35 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin"); 36 MODULE_FIRMWARE("radeon/BONAIRE_me.bin"); 37 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin"); 38 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin"); 39 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin"); 40 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin"); 41 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin"); 42 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin"); 43 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin"); 44 45 MODULE_FIRMWARE("radeon/bonaire_pfp.bin"); 46 MODULE_FIRMWARE("radeon/bonaire_me.bin"); 47 MODULE_FIRMWARE("radeon/bonaire_ce.bin"); 48 MODULE_FIRMWARE("radeon/bonaire_mec.bin"); 49 MODULE_FIRMWARE("radeon/bonaire_mc.bin"); 50 MODULE_FIRMWARE("radeon/bonaire_rlc.bin"); 51 MODULE_FIRMWARE("radeon/bonaire_sdma.bin"); 52 MODULE_FIRMWARE("radeon/bonaire_smc.bin"); 53 54 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin"); 55 MODULE_FIRMWARE("radeon/HAWAII_me.bin"); 56 MODULE_FIRMWARE("radeon/HAWAII_ce.bin"); 57 MODULE_FIRMWARE("radeon/HAWAII_mec.bin"); 58 MODULE_FIRMWARE("radeon/HAWAII_mc.bin"); 59 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin"); 60 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin"); 61 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin"); 62 MODULE_FIRMWARE("radeon/HAWAII_smc.bin"); 63 64 MODULE_FIRMWARE("radeon/hawaii_pfp.bin"); 65 MODULE_FIRMWARE("radeon/hawaii_me.bin"); 66 MODULE_FIRMWARE("radeon/hawaii_ce.bin"); 67 MODULE_FIRMWARE("radeon/hawaii_mec.bin"); 68 MODULE_FIRMWARE("radeon/hawaii_mc.bin"); 69 MODULE_FIRMWARE("radeon/hawaii_rlc.bin"); 70 MODULE_FIRMWARE("radeon/hawaii_sdma.bin"); 71 MODULE_FIRMWARE("radeon/hawaii_smc.bin"); 72 73 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin"); 74 MODULE_FIRMWARE("radeon/KAVERI_me.bin"); 75 MODULE_FIRMWARE("radeon/KAVERI_ce.bin"); 76 MODULE_FIRMWARE("radeon/KAVERI_mec.bin"); 77 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin"); 78 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin"); 79 80 MODULE_FIRMWARE("radeon/kaveri_pfp.bin"); 81 MODULE_FIRMWARE("radeon/kaveri_me.bin"); 82 MODULE_FIRMWARE("radeon/kaveri_ce.bin"); 83 MODULE_FIRMWARE("radeon/kaveri_mec.bin"); 84 MODULE_FIRMWARE("radeon/kaveri_mec2.bin"); 85 MODULE_FIRMWARE("radeon/kaveri_rlc.bin"); 86 MODULE_FIRMWARE("radeon/kaveri_sdma.bin"); 87 88 MODULE_FIRMWARE("radeon/KABINI_pfp.bin"); 89 MODULE_FIRMWARE("radeon/KABINI_me.bin"); 90 MODULE_FIRMWARE("radeon/KABINI_ce.bin"); 91 MODULE_FIRMWARE("radeon/KABINI_mec.bin"); 92 MODULE_FIRMWARE("radeon/KABINI_rlc.bin"); 93 MODULE_FIRMWARE("radeon/KABINI_sdma.bin"); 94 95 MODULE_FIRMWARE("radeon/kabini_pfp.bin"); 96 MODULE_FIRMWARE("radeon/kabini_me.bin"); 97 MODULE_FIRMWARE("radeon/kabini_ce.bin"); 98 MODULE_FIRMWARE("radeon/kabini_mec.bin"); 99 MODULE_FIRMWARE("radeon/kabini_rlc.bin"); 100 MODULE_FIRMWARE("radeon/kabini_sdma.bin"); 101 102 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin"); 103 MODULE_FIRMWARE("radeon/MULLINS_me.bin"); 104 MODULE_FIRMWARE("radeon/MULLINS_ce.bin"); 105 MODULE_FIRMWARE("radeon/MULLINS_mec.bin"); 106 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin"); 107 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin"); 108 109 MODULE_FIRMWARE("radeon/mullins_pfp.bin"); 110 MODULE_FIRMWARE("radeon/mullins_me.bin"); 111 MODULE_FIRMWARE("radeon/mullins_ce.bin"); 112 MODULE_FIRMWARE("radeon/mullins_mec.bin"); 113 MODULE_FIRMWARE("radeon/mullins_rlc.bin"); 114 MODULE_FIRMWARE("radeon/mullins_sdma.bin"); 115 116 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh); 117 static void cik_rlc_stop(struct radeon_device *rdev); 118 static void cik_pcie_gen3_enable(struct radeon_device *rdev); 119 static void cik_program_aspm(struct radeon_device *rdev); 120 static void cik_init_pg(struct radeon_device *rdev); 121 static void cik_init_cg(struct radeon_device *rdev); 122 static void cik_fini_pg(struct radeon_device *rdev); 123 static void cik_fini_cg(struct radeon_device *rdev); 124 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, 125 bool enable); 126 127 /* get temperature in millidegrees */ 128 int ci_get_temp(struct radeon_device *rdev) 129 { 130 u32 temp; 131 int actual_temp = 0; 132 133 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >> 134 CTF_TEMP_SHIFT; 135 136 if (temp & 0x200) 137 actual_temp = 255; 138 else 139 actual_temp = temp & 0x1ff; 140 141 actual_temp = actual_temp * 1000; 142 143 return actual_temp; 144 } 145 146 /* get temperature in millidegrees */ 147 int kv_get_temp(struct radeon_device *rdev) 148 { 149 u32 temp; 150 int actual_temp = 0; 151 152 temp = RREG32_SMC(0xC0300E0C); 153 154 if (temp) 155 actual_temp = (temp / 8) - 49; 156 else 157 actual_temp = 0; 158 159 actual_temp = actual_temp * 1000; 160 161 return actual_temp; 162 } 163 164 /* 165 * Indirect registers accessor 166 */ 167 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg) 168 { 169 u32 r; 170 171 spin_lock(&rdev->pciep_idx_lock); 172 WREG32(PCIE_INDEX, reg); 173 (void)RREG32(PCIE_INDEX); 174 r = RREG32(PCIE_DATA); 175 spin_unlock(&rdev->pciep_idx_lock); 176 return r; 177 } 178 179 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v) 180 { 181 spin_lock(&rdev->pciep_idx_lock); 182 WREG32(PCIE_INDEX, reg); 183 (void)RREG32(PCIE_INDEX); 184 WREG32(PCIE_DATA, v); 185 (void)RREG32(PCIE_DATA); 186 spin_unlock(&rdev->pciep_idx_lock); 187 } 188 189 static const u32 spectre_rlc_save_restore_register_list[] = 190 { 191 (0x0e00 << 16) | (0xc12c >> 2), 192 0x00000000, 193 (0x0e00 << 16) | (0xc140 >> 2), 194 0x00000000, 195 (0x0e00 << 16) | (0xc150 >> 2), 196 0x00000000, 197 (0x0e00 << 16) | (0xc15c >> 2), 198 0x00000000, 199 (0x0e00 << 16) | (0xc168 >> 2), 200 0x00000000, 201 (0x0e00 << 16) | (0xc170 >> 2), 202 0x00000000, 203 (0x0e00 << 16) | (0xc178 >> 2), 204 0x00000000, 205 (0x0e00 << 16) | (0xc204 >> 2), 206 0x00000000, 207 (0x0e00 << 16) | (0xc2b4 >> 2), 208 0x00000000, 209 (0x0e00 << 16) | (0xc2b8 >> 2), 210 0x00000000, 211 (0x0e00 << 16) | (0xc2bc >> 2), 212 0x00000000, 213 (0x0e00 << 16) | (0xc2c0 >> 2), 214 0x00000000, 215 (0x0e00 << 16) | (0x8228 >> 2), 216 0x00000000, 217 (0x0e00 << 16) | (0x829c >> 2), 218 0x00000000, 219 (0x0e00 << 16) | (0x869c >> 2), 220 0x00000000, 221 (0x0600 << 16) | (0x98f4 >> 2), 222 0x00000000, 223 (0x0e00 << 16) | (0x98f8 >> 2), 224 0x00000000, 225 (0x0e00 << 16) | (0x9900 >> 2), 226 0x00000000, 227 (0x0e00 << 16) | (0xc260 >> 2), 228 0x00000000, 229 (0x0e00 << 16) | (0x90e8 >> 2), 230 0x00000000, 231 (0x0e00 << 16) | (0x3c000 >> 2), 232 0x00000000, 233 (0x0e00 << 16) | (0x3c00c >> 2), 234 0x00000000, 235 (0x0e00 << 16) | (0x8c1c >> 2), 236 0x00000000, 237 (0x0e00 << 16) | (0x9700 >> 2), 238 0x00000000, 239 (0x0e00 << 16) | (0xcd20 >> 2), 240 0x00000000, 241 (0x4e00 << 16) | (0xcd20 >> 2), 242 0x00000000, 243 (0x5e00 << 16) | (0xcd20 >> 2), 244 0x00000000, 245 (0x6e00 << 16) | (0xcd20 >> 2), 246 0x00000000, 247 (0x7e00 << 16) | (0xcd20 >> 2), 248 0x00000000, 249 (0x8e00 << 16) | (0xcd20 >> 2), 250 0x00000000, 251 (0x9e00 << 16) | (0xcd20 >> 2), 252 0x00000000, 253 (0xae00 << 16) | (0xcd20 >> 2), 254 0x00000000, 255 (0xbe00 << 16) | (0xcd20 >> 2), 256 0x00000000, 257 (0x0e00 << 16) | (0x89bc >> 2), 258 0x00000000, 259 (0x0e00 << 16) | (0x8900 >> 2), 260 0x00000000, 261 0x3, 262 (0x0e00 << 16) | (0xc130 >> 2), 263 0x00000000, 264 (0x0e00 << 16) | (0xc134 >> 2), 265 0x00000000, 266 (0x0e00 << 16) | (0xc1fc >> 2), 267 0x00000000, 268 (0x0e00 << 16) | (0xc208 >> 2), 269 0x00000000, 270 (0x0e00 << 16) | (0xc264 >> 2), 271 0x00000000, 272 (0x0e00 << 16) | (0xc268 >> 2), 273 0x00000000, 274 (0x0e00 << 16) | (0xc26c >> 2), 275 0x00000000, 276 (0x0e00 << 16) | (0xc270 >> 2), 277 0x00000000, 278 (0x0e00 << 16) | (0xc274 >> 2), 279 0x00000000, 280 (0x0e00 << 16) | (0xc278 >> 2), 281 0x00000000, 282 (0x0e00 << 16) | (0xc27c >> 2), 283 0x00000000, 284 (0x0e00 << 16) | (0xc280 >> 2), 285 0x00000000, 286 (0x0e00 << 16) | (0xc284 >> 2), 287 0x00000000, 288 (0x0e00 << 16) | (0xc288 >> 2), 289 0x00000000, 290 (0x0e00 << 16) | (0xc28c >> 2), 291 0x00000000, 292 (0x0e00 << 16) | (0xc290 >> 2), 293 0x00000000, 294 (0x0e00 << 16) | (0xc294 >> 2), 295 0x00000000, 296 (0x0e00 << 16) | (0xc298 >> 2), 297 0x00000000, 298 (0x0e00 << 16) | (0xc29c >> 2), 299 0x00000000, 300 (0x0e00 << 16) | (0xc2a0 >> 2), 301 0x00000000, 302 (0x0e00 << 16) | (0xc2a4 >> 2), 303 0x00000000, 304 (0x0e00 << 16) | (0xc2a8 >> 2), 305 0x00000000, 306 (0x0e00 << 16) | (0xc2ac >> 2), 307 0x00000000, 308 (0x0e00 << 16) | (0xc2b0 >> 2), 309 0x00000000, 310 (0x0e00 << 16) | (0x301d0 >> 2), 311 0x00000000, 312 (0x0e00 << 16) | (0x30238 >> 2), 313 0x00000000, 314 (0x0e00 << 16) | (0x30250 >> 2), 315 0x00000000, 316 (0x0e00 << 16) | (0x30254 >> 2), 317 0x00000000, 318 (0x0e00 << 16) | (0x30258 >> 2), 319 0x00000000, 320 (0x0e00 << 16) | (0x3025c >> 2), 321 0x00000000, 322 (0x4e00 << 16) | (0xc900 >> 2), 323 0x00000000, 324 (0x5e00 << 16) | (0xc900 >> 2), 325 0x00000000, 326 (0x6e00 << 16) | (0xc900 >> 2), 327 0x00000000, 328 (0x7e00 << 16) | (0xc900 >> 2), 329 0x00000000, 330 (0x8e00 << 16) | (0xc900 >> 2), 331 0x00000000, 332 (0x9e00 << 16) | (0xc900 >> 2), 333 0x00000000, 334 (0xae00 << 16) | (0xc900 >> 2), 335 0x00000000, 336 (0xbe00 << 16) | (0xc900 >> 2), 337 0x00000000, 338 (0x4e00 << 16) | (0xc904 >> 2), 339 0x00000000, 340 (0x5e00 << 16) | (0xc904 >> 2), 341 0x00000000, 342 (0x6e00 << 16) | (0xc904 >> 2), 343 0x00000000, 344 (0x7e00 << 16) | (0xc904 >> 2), 345 0x00000000, 346 (0x8e00 << 16) | (0xc904 >> 2), 347 0x00000000, 348 (0x9e00 << 16) | (0xc904 >> 2), 349 0x00000000, 350 (0xae00 << 16) | (0xc904 >> 2), 351 0x00000000, 352 (0xbe00 << 16) | (0xc904 >> 2), 353 0x00000000, 354 (0x4e00 << 16) | (0xc908 >> 2), 355 0x00000000, 356 (0x5e00 << 16) | (0xc908 >> 2), 357 0x00000000, 358 (0x6e00 << 16) | (0xc908 >> 2), 359 0x00000000, 360 (0x7e00 << 16) | (0xc908 >> 2), 361 0x00000000, 362 (0x8e00 << 16) | (0xc908 >> 2), 363 0x00000000, 364 (0x9e00 << 16) | (0xc908 >> 2), 365 0x00000000, 366 (0xae00 << 16) | (0xc908 >> 2), 367 0x00000000, 368 (0xbe00 << 16) | (0xc908 >> 2), 369 0x00000000, 370 (0x4e00 << 16) | (0xc90c >> 2), 371 0x00000000, 372 (0x5e00 << 16) | (0xc90c >> 2), 373 0x00000000, 374 (0x6e00 << 16) | (0xc90c >> 2), 375 0x00000000, 376 (0x7e00 << 16) | (0xc90c >> 2), 377 0x00000000, 378 (0x8e00 << 16) | (0xc90c >> 2), 379 0x00000000, 380 (0x9e00 << 16) | (0xc90c >> 2), 381 0x00000000, 382 (0xae00 << 16) | (0xc90c >> 2), 383 0x00000000, 384 (0xbe00 << 16) | (0xc90c >> 2), 385 0x00000000, 386 (0x4e00 << 16) | (0xc910 >> 2), 387 0x00000000, 388 (0x5e00 << 16) | (0xc910 >> 2), 389 0x00000000, 390 (0x6e00 << 16) | (0xc910 >> 2), 391 0x00000000, 392 (0x7e00 << 16) | (0xc910 >> 2), 393 0x00000000, 394 (0x8e00 << 16) | (0xc910 >> 2), 395 0x00000000, 396 (0x9e00 << 16) | (0xc910 >> 2), 397 0x00000000, 398 (0xae00 << 16) | (0xc910 >> 2), 399 0x00000000, 400 (0xbe00 << 16) | (0xc910 >> 2), 401 0x00000000, 402 (0x0e00 << 16) | (0xc99c >> 2), 403 0x00000000, 404 (0x0e00 << 16) | (0x9834 >> 2), 405 0x00000000, 406 (0x0000 << 16) | (0x30f00 >> 2), 407 0x00000000, 408 (0x0001 << 16) | (0x30f00 >> 2), 409 0x00000000, 410 (0x0000 << 16) | (0x30f04 >> 2), 411 0x00000000, 412 (0x0001 << 16) | (0x30f04 >> 2), 413 0x00000000, 414 (0x0000 << 16) | (0x30f08 >> 2), 415 0x00000000, 416 (0x0001 << 16) | (0x30f08 >> 2), 417 0x00000000, 418 (0x0000 << 16) | (0x30f0c >> 2), 419 0x00000000, 420 (0x0001 << 16) | (0x30f0c >> 2), 421 0x00000000, 422 (0x0600 << 16) | (0x9b7c >> 2), 423 0x00000000, 424 (0x0e00 << 16) | (0x8a14 >> 2), 425 0x00000000, 426 (0x0e00 << 16) | (0x8a18 >> 2), 427 0x00000000, 428 (0x0600 << 16) | (0x30a00 >> 2), 429 0x00000000, 430 (0x0e00 << 16) | (0x8bf0 >> 2), 431 0x00000000, 432 (0x0e00 << 16) | (0x8bcc >> 2), 433 0x00000000, 434 (0x0e00 << 16) | (0x8b24 >> 2), 435 0x00000000, 436 (0x0e00 << 16) | (0x30a04 >> 2), 437 0x00000000, 438 (0x0600 << 16) | (0x30a10 >> 2), 439 0x00000000, 440 (0x0600 << 16) | (0x30a14 >> 2), 441 0x00000000, 442 (0x0600 << 16) | (0x30a18 >> 2), 443 0x00000000, 444 (0x0600 << 16) | (0x30a2c >> 2), 445 0x00000000, 446 (0x0e00 << 16) | (0xc700 >> 2), 447 0x00000000, 448 (0x0e00 << 16) | (0xc704 >> 2), 449 0x00000000, 450 (0x0e00 << 16) | (0xc708 >> 2), 451 0x00000000, 452 (0x0e00 << 16) | (0xc768 >> 2), 453 0x00000000, 454 (0x0400 << 16) | (0xc770 >> 2), 455 0x00000000, 456 (0x0400 << 16) | (0xc774 >> 2), 457 0x00000000, 458 (0x0400 << 16) | (0xc778 >> 2), 459 0x00000000, 460 (0x0400 << 16) | (0xc77c >> 2), 461 0x00000000, 462 (0x0400 << 16) | (0xc780 >> 2), 463 0x00000000, 464 (0x0400 << 16) | (0xc784 >> 2), 465 0x00000000, 466 (0x0400 << 16) | (0xc788 >> 2), 467 0x00000000, 468 (0x0400 << 16) | (0xc78c >> 2), 469 0x00000000, 470 (0x0400 << 16) | (0xc798 >> 2), 471 0x00000000, 472 (0x0400 << 16) | (0xc79c >> 2), 473 0x00000000, 474 (0x0400 << 16) | (0xc7a0 >> 2), 475 0x00000000, 476 (0x0400 << 16) | (0xc7a4 >> 2), 477 0x00000000, 478 (0x0400 << 16) | (0xc7a8 >> 2), 479 0x00000000, 480 (0x0400 << 16) | (0xc7ac >> 2), 481 0x00000000, 482 (0x0400 << 16) | (0xc7b0 >> 2), 483 0x00000000, 484 (0x0400 << 16) | (0xc7b4 >> 2), 485 0x00000000, 486 (0x0e00 << 16) | (0x9100 >> 2), 487 0x00000000, 488 (0x0e00 << 16) | (0x3c010 >> 2), 489 0x00000000, 490 (0x0e00 << 16) | (0x92a8 >> 2), 491 0x00000000, 492 (0x0e00 << 16) | (0x92ac >> 2), 493 0x00000000, 494 (0x0e00 << 16) | (0x92b4 >> 2), 495 0x00000000, 496 (0x0e00 << 16) | (0x92b8 >> 2), 497 0x00000000, 498 (0x0e00 << 16) | (0x92bc >> 2), 499 0x00000000, 500 (0x0e00 << 16) | (0x92c0 >> 2), 501 0x00000000, 502 (0x0e00 << 16) | (0x92c4 >> 2), 503 0x00000000, 504 (0x0e00 << 16) | (0x92c8 >> 2), 505 0x00000000, 506 (0x0e00 << 16) | (0x92cc >> 2), 507 0x00000000, 508 (0x0e00 << 16) | (0x92d0 >> 2), 509 0x00000000, 510 (0x0e00 << 16) | (0x8c00 >> 2), 511 0x00000000, 512 (0x0e00 << 16) | (0x8c04 >> 2), 513 0x00000000, 514 (0x0e00 << 16) | (0x8c20 >> 2), 515 0x00000000, 516 (0x0e00 << 16) | (0x8c38 >> 2), 517 0x00000000, 518 (0x0e00 << 16) | (0x8c3c >> 2), 519 0x00000000, 520 (0x0e00 << 16) | (0xae00 >> 2), 521 0x00000000, 522 (0x0e00 << 16) | (0x9604 >> 2), 523 0x00000000, 524 (0x0e00 << 16) | (0xac08 >> 2), 525 0x00000000, 526 (0x0e00 << 16) | (0xac0c >> 2), 527 0x00000000, 528 (0x0e00 << 16) | (0xac10 >> 2), 529 0x00000000, 530 (0x0e00 << 16) | (0xac14 >> 2), 531 0x00000000, 532 (0x0e00 << 16) | (0xac58 >> 2), 533 0x00000000, 534 (0x0e00 << 16) | (0xac68 >> 2), 535 0x00000000, 536 (0x0e00 << 16) | (0xac6c >> 2), 537 0x00000000, 538 (0x0e00 << 16) | (0xac70 >> 2), 539 0x00000000, 540 (0x0e00 << 16) | (0xac74 >> 2), 541 0x00000000, 542 (0x0e00 << 16) | (0xac78 >> 2), 543 0x00000000, 544 (0x0e00 << 16) | (0xac7c >> 2), 545 0x00000000, 546 (0x0e00 << 16) | (0xac80 >> 2), 547 0x00000000, 548 (0x0e00 << 16) | (0xac84 >> 2), 549 0x00000000, 550 (0x0e00 << 16) | (0xac88 >> 2), 551 0x00000000, 552 (0x0e00 << 16) | (0xac8c >> 2), 553 0x00000000, 554 (0x0e00 << 16) | (0x970c >> 2), 555 0x00000000, 556 (0x0e00 << 16) | (0x9714 >> 2), 557 0x00000000, 558 (0x0e00 << 16) | (0x9718 >> 2), 559 0x00000000, 560 (0x0e00 << 16) | (0x971c >> 2), 561 0x00000000, 562 (0x0e00 << 16) | (0x31068 >> 2), 563 0x00000000, 564 (0x4e00 << 16) | (0x31068 >> 2), 565 0x00000000, 566 (0x5e00 << 16) | (0x31068 >> 2), 567 0x00000000, 568 (0x6e00 << 16) | (0x31068 >> 2), 569 0x00000000, 570 (0x7e00 << 16) | (0x31068 >> 2), 571 0x00000000, 572 (0x8e00 << 16) | (0x31068 >> 2), 573 0x00000000, 574 (0x9e00 << 16) | (0x31068 >> 2), 575 0x00000000, 576 (0xae00 << 16) | (0x31068 >> 2), 577 0x00000000, 578 (0xbe00 << 16) | (0x31068 >> 2), 579 0x00000000, 580 (0x0e00 << 16) | (0xcd10 >> 2), 581 0x00000000, 582 (0x0e00 << 16) | (0xcd14 >> 2), 583 0x00000000, 584 (0x0e00 << 16) | (0x88b0 >> 2), 585 0x00000000, 586 (0x0e00 << 16) | (0x88b4 >> 2), 587 0x00000000, 588 (0x0e00 << 16) | (0x88b8 >> 2), 589 0x00000000, 590 (0x0e00 << 16) | (0x88bc >> 2), 591 0x00000000, 592 (0x0400 << 16) | (0x89c0 >> 2), 593 0x00000000, 594 (0x0e00 << 16) | (0x88c4 >> 2), 595 0x00000000, 596 (0x0e00 << 16) | (0x88c8 >> 2), 597 0x00000000, 598 (0x0e00 << 16) | (0x88d0 >> 2), 599 0x00000000, 600 (0x0e00 << 16) | (0x88d4 >> 2), 601 0x00000000, 602 (0x0e00 << 16) | (0x88d8 >> 2), 603 0x00000000, 604 (0x0e00 << 16) | (0x8980 >> 2), 605 0x00000000, 606 (0x0e00 << 16) | (0x30938 >> 2), 607 0x00000000, 608 (0x0e00 << 16) | (0x3093c >> 2), 609 0x00000000, 610 (0x0e00 << 16) | (0x30940 >> 2), 611 0x00000000, 612 (0x0e00 << 16) | (0x89a0 >> 2), 613 0x00000000, 614 (0x0e00 << 16) | (0x30900 >> 2), 615 0x00000000, 616 (0x0e00 << 16) | (0x30904 >> 2), 617 0x00000000, 618 (0x0e00 << 16) | (0x89b4 >> 2), 619 0x00000000, 620 (0x0e00 << 16) | (0x3c210 >> 2), 621 0x00000000, 622 (0x0e00 << 16) | (0x3c214 >> 2), 623 0x00000000, 624 (0x0e00 << 16) | (0x3c218 >> 2), 625 0x00000000, 626 (0x0e00 << 16) | (0x8904 >> 2), 627 0x00000000, 628 0x5, 629 (0x0e00 << 16) | (0x8c28 >> 2), 630 (0x0e00 << 16) | (0x8c2c >> 2), 631 (0x0e00 << 16) | (0x8c30 >> 2), 632 (0x0e00 << 16) | (0x8c34 >> 2), 633 (0x0e00 << 16) | (0x9600 >> 2), 634 }; 635 636 static const u32 kalindi_rlc_save_restore_register_list[] = 637 { 638 (0x0e00 << 16) | (0xc12c >> 2), 639 0x00000000, 640 (0x0e00 << 16) | (0xc140 >> 2), 641 0x00000000, 642 (0x0e00 << 16) | (0xc150 >> 2), 643 0x00000000, 644 (0x0e00 << 16) | (0xc15c >> 2), 645 0x00000000, 646 (0x0e00 << 16) | (0xc168 >> 2), 647 0x00000000, 648 (0x0e00 << 16) | (0xc170 >> 2), 649 0x00000000, 650 (0x0e00 << 16) | (0xc204 >> 2), 651 0x00000000, 652 (0x0e00 << 16) | (0xc2b4 >> 2), 653 0x00000000, 654 (0x0e00 << 16) | (0xc2b8 >> 2), 655 0x00000000, 656 (0x0e00 << 16) | (0xc2bc >> 2), 657 0x00000000, 658 (0x0e00 << 16) | (0xc2c0 >> 2), 659 0x00000000, 660 (0x0e00 << 16) | (0x8228 >> 2), 661 0x00000000, 662 (0x0e00 << 16) | (0x829c >> 2), 663 0x00000000, 664 (0x0e00 << 16) | (0x869c >> 2), 665 0x00000000, 666 (0x0600 << 16) | (0x98f4 >> 2), 667 0x00000000, 668 (0x0e00 << 16) | (0x98f8 >> 2), 669 0x00000000, 670 (0x0e00 << 16) | (0x9900 >> 2), 671 0x00000000, 672 (0x0e00 << 16) | (0xc260 >> 2), 673 0x00000000, 674 (0x0e00 << 16) | (0x90e8 >> 2), 675 0x00000000, 676 (0x0e00 << 16) | (0x3c000 >> 2), 677 0x00000000, 678 (0x0e00 << 16) | (0x3c00c >> 2), 679 0x00000000, 680 (0x0e00 << 16) | (0x8c1c >> 2), 681 0x00000000, 682 (0x0e00 << 16) | (0x9700 >> 2), 683 0x00000000, 684 (0x0e00 << 16) | (0xcd20 >> 2), 685 0x00000000, 686 (0x4e00 << 16) | (0xcd20 >> 2), 687 0x00000000, 688 (0x5e00 << 16) | (0xcd20 >> 2), 689 0x00000000, 690 (0x6e00 << 16) | (0xcd20 >> 2), 691 0x00000000, 692 (0x7e00 << 16) | (0xcd20 >> 2), 693 0x00000000, 694 (0x0e00 << 16) | (0x89bc >> 2), 695 0x00000000, 696 (0x0e00 << 16) | (0x8900 >> 2), 697 0x00000000, 698 0x3, 699 (0x0e00 << 16) | (0xc130 >> 2), 700 0x00000000, 701 (0x0e00 << 16) | (0xc134 >> 2), 702 0x00000000, 703 (0x0e00 << 16) | (0xc1fc >> 2), 704 0x00000000, 705 (0x0e00 << 16) | (0xc208 >> 2), 706 0x00000000, 707 (0x0e00 << 16) | (0xc264 >> 2), 708 0x00000000, 709 (0x0e00 << 16) | (0xc268 >> 2), 710 0x00000000, 711 (0x0e00 << 16) | (0xc26c >> 2), 712 0x00000000, 713 (0x0e00 << 16) | (0xc270 >> 2), 714 0x00000000, 715 (0x0e00 << 16) | (0xc274 >> 2), 716 0x00000000, 717 (0x0e00 << 16) | (0xc28c >> 2), 718 0x00000000, 719 (0x0e00 << 16) | (0xc290 >> 2), 720 0x00000000, 721 (0x0e00 << 16) | (0xc294 >> 2), 722 0x00000000, 723 (0x0e00 << 16) | (0xc298 >> 2), 724 0x00000000, 725 (0x0e00 << 16) | (0xc2a0 >> 2), 726 0x00000000, 727 (0x0e00 << 16) | (0xc2a4 >> 2), 728 0x00000000, 729 (0x0e00 << 16) | (0xc2a8 >> 2), 730 0x00000000, 731 (0x0e00 << 16) | (0xc2ac >> 2), 732 0x00000000, 733 (0x0e00 << 16) | (0x301d0 >> 2), 734 0x00000000, 735 (0x0e00 << 16) | (0x30238 >> 2), 736 0x00000000, 737 (0x0e00 << 16) | (0x30250 >> 2), 738 0x00000000, 739 (0x0e00 << 16) | (0x30254 >> 2), 740 0x00000000, 741 (0x0e00 << 16) | (0x30258 >> 2), 742 0x00000000, 743 (0x0e00 << 16) | (0x3025c >> 2), 744 0x00000000, 745 (0x4e00 << 16) | (0xc900 >> 2), 746 0x00000000, 747 (0x5e00 << 16) | (0xc900 >> 2), 748 0x00000000, 749 (0x6e00 << 16) | (0xc900 >> 2), 750 0x00000000, 751 (0x7e00 << 16) | (0xc900 >> 2), 752 0x00000000, 753 (0x4e00 << 16) | (0xc904 >> 2), 754 0x00000000, 755 (0x5e00 << 16) | (0xc904 >> 2), 756 0x00000000, 757 (0x6e00 << 16) | (0xc904 >> 2), 758 0x00000000, 759 (0x7e00 << 16) | (0xc904 >> 2), 760 0x00000000, 761 (0x4e00 << 16) | (0xc908 >> 2), 762 0x00000000, 763 (0x5e00 << 16) | (0xc908 >> 2), 764 0x00000000, 765 (0x6e00 << 16) | (0xc908 >> 2), 766 0x00000000, 767 (0x7e00 << 16) | (0xc908 >> 2), 768 0x00000000, 769 (0x4e00 << 16) | (0xc90c >> 2), 770 0x00000000, 771 (0x5e00 << 16) | (0xc90c >> 2), 772 0x00000000, 773 (0x6e00 << 16) | (0xc90c >> 2), 774 0x00000000, 775 (0x7e00 << 16) | (0xc90c >> 2), 776 0x00000000, 777 (0x4e00 << 16) | (0xc910 >> 2), 778 0x00000000, 779 (0x5e00 << 16) | (0xc910 >> 2), 780 0x00000000, 781 (0x6e00 << 16) | (0xc910 >> 2), 782 0x00000000, 783 (0x7e00 << 16) | (0xc910 >> 2), 784 0x00000000, 785 (0x0e00 << 16) | (0xc99c >> 2), 786 0x00000000, 787 (0x0e00 << 16) | (0x9834 >> 2), 788 0x00000000, 789 (0x0000 << 16) | (0x30f00 >> 2), 790 0x00000000, 791 (0x0000 << 16) | (0x30f04 >> 2), 792 0x00000000, 793 (0x0000 << 16) | (0x30f08 >> 2), 794 0x00000000, 795 (0x0000 << 16) | (0x30f0c >> 2), 796 0x00000000, 797 (0x0600 << 16) | (0x9b7c >> 2), 798 0x00000000, 799 (0x0e00 << 16) | (0x8a14 >> 2), 800 0x00000000, 801 (0x0e00 << 16) | (0x8a18 >> 2), 802 0x00000000, 803 (0x0600 << 16) | (0x30a00 >> 2), 804 0x00000000, 805 (0x0e00 << 16) | (0x8bf0 >> 2), 806 0x00000000, 807 (0x0e00 << 16) | (0x8bcc >> 2), 808 0x00000000, 809 (0x0e00 << 16) | (0x8b24 >> 2), 810 0x00000000, 811 (0x0e00 << 16) | (0x30a04 >> 2), 812 0x00000000, 813 (0x0600 << 16) | (0x30a10 >> 2), 814 0x00000000, 815 (0x0600 << 16) | (0x30a14 >> 2), 816 0x00000000, 817 (0x0600 << 16) | (0x30a18 >> 2), 818 0x00000000, 819 (0x0600 << 16) | (0x30a2c >> 2), 820 0x00000000, 821 (0x0e00 << 16) | (0xc700 >> 2), 822 0x00000000, 823 (0x0e00 << 16) | (0xc704 >> 2), 824 0x00000000, 825 (0x0e00 << 16) | (0xc708 >> 2), 826 0x00000000, 827 (0x0e00 << 16) | (0xc768 >> 2), 828 0x00000000, 829 (0x0400 << 16) | (0xc770 >> 2), 830 0x00000000, 831 (0x0400 << 16) | (0xc774 >> 2), 832 0x00000000, 833 (0x0400 << 16) | (0xc798 >> 2), 834 0x00000000, 835 (0x0400 << 16) | (0xc79c >> 2), 836 0x00000000, 837 (0x0e00 << 16) | (0x9100 >> 2), 838 0x00000000, 839 (0x0e00 << 16) | (0x3c010 >> 2), 840 0x00000000, 841 (0x0e00 << 16) | (0x8c00 >> 2), 842 0x00000000, 843 (0x0e00 << 16) | (0x8c04 >> 2), 844 0x00000000, 845 (0x0e00 << 16) | (0x8c20 >> 2), 846 0x00000000, 847 (0x0e00 << 16) | (0x8c38 >> 2), 848 0x00000000, 849 (0x0e00 << 16) | (0x8c3c >> 2), 850 0x00000000, 851 (0x0e00 << 16) | (0xae00 >> 2), 852 0x00000000, 853 (0x0e00 << 16) | (0x9604 >> 2), 854 0x00000000, 855 (0x0e00 << 16) | (0xac08 >> 2), 856 0x00000000, 857 (0x0e00 << 16) | (0xac0c >> 2), 858 0x00000000, 859 (0x0e00 << 16) | (0xac10 >> 2), 860 0x00000000, 861 (0x0e00 << 16) | (0xac14 >> 2), 862 0x00000000, 863 (0x0e00 << 16) | (0xac58 >> 2), 864 0x00000000, 865 (0x0e00 << 16) | (0xac68 >> 2), 866 0x00000000, 867 (0x0e00 << 16) | (0xac6c >> 2), 868 0x00000000, 869 (0x0e00 << 16) | (0xac70 >> 2), 870 0x00000000, 871 (0x0e00 << 16) | (0xac74 >> 2), 872 0x00000000, 873 (0x0e00 << 16) | (0xac78 >> 2), 874 0x00000000, 875 (0x0e00 << 16) | (0xac7c >> 2), 876 0x00000000, 877 (0x0e00 << 16) | (0xac80 >> 2), 878 0x00000000, 879 (0x0e00 << 16) | (0xac84 >> 2), 880 0x00000000, 881 (0x0e00 << 16) | (0xac88 >> 2), 882 0x00000000, 883 (0x0e00 << 16) | (0xac8c >> 2), 884 0x00000000, 885 (0x0e00 << 16) | (0x970c >> 2), 886 0x00000000, 887 (0x0e00 << 16) | (0x9714 >> 2), 888 0x00000000, 889 (0x0e00 << 16) | (0x9718 >> 2), 890 0x00000000, 891 (0x0e00 << 16) | (0x971c >> 2), 892 0x00000000, 893 (0x0e00 << 16) | (0x31068 >> 2), 894 0x00000000, 895 (0x4e00 << 16) | (0x31068 >> 2), 896 0x00000000, 897 (0x5e00 << 16) | (0x31068 >> 2), 898 0x00000000, 899 (0x6e00 << 16) | (0x31068 >> 2), 900 0x00000000, 901 (0x7e00 << 16) | (0x31068 >> 2), 902 0x00000000, 903 (0x0e00 << 16) | (0xcd10 >> 2), 904 0x00000000, 905 (0x0e00 << 16) | (0xcd14 >> 2), 906 0x00000000, 907 (0x0e00 << 16) | (0x88b0 >> 2), 908 0x00000000, 909 (0x0e00 << 16) | (0x88b4 >> 2), 910 0x00000000, 911 (0x0e00 << 16) | (0x88b8 >> 2), 912 0x00000000, 913 (0x0e00 << 16) | (0x88bc >> 2), 914 0x00000000, 915 (0x0400 << 16) | (0x89c0 >> 2), 916 0x00000000, 917 (0x0e00 << 16) | (0x88c4 >> 2), 918 0x00000000, 919 (0x0e00 << 16) | (0x88c8 >> 2), 920 0x00000000, 921 (0x0e00 << 16) | (0x88d0 >> 2), 922 0x00000000, 923 (0x0e00 << 16) | (0x88d4 >> 2), 924 0x00000000, 925 (0x0e00 << 16) | (0x88d8 >> 2), 926 0x00000000, 927 (0x0e00 << 16) | (0x8980 >> 2), 928 0x00000000, 929 (0x0e00 << 16) | (0x30938 >> 2), 930 0x00000000, 931 (0x0e00 << 16) | (0x3093c >> 2), 932 0x00000000, 933 (0x0e00 << 16) | (0x30940 >> 2), 934 0x00000000, 935 (0x0e00 << 16) | (0x89a0 >> 2), 936 0x00000000, 937 (0x0e00 << 16) | (0x30900 >> 2), 938 0x00000000, 939 (0x0e00 << 16) | (0x30904 >> 2), 940 0x00000000, 941 (0x0e00 << 16) | (0x89b4 >> 2), 942 0x00000000, 943 (0x0e00 << 16) | (0x3e1fc >> 2), 944 0x00000000, 945 (0x0e00 << 16) | (0x3c210 >> 2), 946 0x00000000, 947 (0x0e00 << 16) | (0x3c214 >> 2), 948 0x00000000, 949 (0x0e00 << 16) | (0x3c218 >> 2), 950 0x00000000, 951 (0x0e00 << 16) | (0x8904 >> 2), 952 0x00000000, 953 0x5, 954 (0x0e00 << 16) | (0x8c28 >> 2), 955 (0x0e00 << 16) | (0x8c2c >> 2), 956 (0x0e00 << 16) | (0x8c30 >> 2), 957 (0x0e00 << 16) | (0x8c34 >> 2), 958 (0x0e00 << 16) | (0x9600 >> 2), 959 }; 960 961 static const u32 bonaire_golden_spm_registers[] = 962 { 963 0x30800, 0xe0ffffff, 0xe0000000 964 }; 965 966 static const u32 bonaire_golden_common_registers[] = 967 { 968 0xc770, 0xffffffff, 0x00000800, 969 0xc774, 0xffffffff, 0x00000800, 970 0xc798, 0xffffffff, 0x00007fbf, 971 0xc79c, 0xffffffff, 0x00007faf 972 }; 973 974 static const u32 bonaire_golden_registers[] = 975 { 976 0x3354, 0x00000333, 0x00000333, 977 0x3350, 0x000c0fc0, 0x00040200, 978 0x9a10, 0x00010000, 0x00058208, 979 0x3c000, 0xffff1fff, 0x00140000, 980 0x3c200, 0xfdfc0fff, 0x00000100, 981 0x3c234, 0x40000000, 0x40000200, 982 0x9830, 0xffffffff, 0x00000000, 983 0x9834, 0xf00fffff, 0x00000400, 984 0x9838, 0x0002021c, 0x00020200, 985 0xc78, 0x00000080, 0x00000000, 986 0x5bb0, 0x000000f0, 0x00000070, 987 0x5bc0, 0xf0311fff, 0x80300000, 988 0x98f8, 0x73773777, 0x12010001, 989 0x350c, 0x00810000, 0x408af000, 990 0x7030, 0x31000111, 0x00000011, 991 0x2f48, 0x73773777, 0x12010001, 992 0x220c, 0x00007fb6, 0x0021a1b1, 993 0x2210, 0x00007fb6, 0x002021b1, 994 0x2180, 0x00007fb6, 0x00002191, 995 0x2218, 0x00007fb6, 0x002121b1, 996 0x221c, 0x00007fb6, 0x002021b1, 997 0x21dc, 0x00007fb6, 0x00002191, 998 0x21e0, 0x00007fb6, 0x00002191, 999 0x3628, 0x0000003f, 0x0000000a, 1000 0x362c, 0x0000003f, 0x0000000a, 1001 0x2ae4, 0x00073ffe, 0x000022a2, 1002 0x240c, 0x000007ff, 0x00000000, 1003 0x8a14, 0xf000003f, 0x00000007, 1004 0x8bf0, 0x00002001, 0x00000001, 1005 0x8b24, 0xffffffff, 0x00ffffff, 1006 0x30a04, 0x0000ff0f, 0x00000000, 1007 0x28a4c, 0x07ffffff, 0x06000000, 1008 0x4d8, 0x00000fff, 0x00000100, 1009 0x3e78, 0x00000001, 0x00000002, 1010 0x9100, 0x03000000, 0x0362c688, 1011 0x8c00, 0x000000ff, 0x00000001, 1012 0xe40, 0x00001fff, 0x00001fff, 1013 0x9060, 0x0000007f, 0x00000020, 1014 0x9508, 0x00010000, 0x00010000, 1015 0xac14, 0x000003ff, 0x000000f3, 1016 0xac0c, 0xffffffff, 0x00001032 1017 }; 1018 1019 static const u32 bonaire_mgcg_cgcg_init[] = 1020 { 1021 0xc420, 0xffffffff, 0xfffffffc, 1022 0x30800, 0xffffffff, 0xe0000000, 1023 0x3c2a0, 0xffffffff, 0x00000100, 1024 0x3c208, 0xffffffff, 0x00000100, 1025 0x3c2c0, 0xffffffff, 0xc0000100, 1026 0x3c2c8, 0xffffffff, 0xc0000100, 1027 0x3c2c4, 0xffffffff, 0xc0000100, 1028 0x55e4, 0xffffffff, 0x00600100, 1029 0x3c280, 0xffffffff, 0x00000100, 1030 0x3c214, 0xffffffff, 0x06000100, 1031 0x3c220, 0xffffffff, 0x00000100, 1032 0x3c218, 0xffffffff, 0x06000100, 1033 0x3c204, 0xffffffff, 0x00000100, 1034 0x3c2e0, 0xffffffff, 0x00000100, 1035 0x3c224, 0xffffffff, 0x00000100, 1036 0x3c200, 0xffffffff, 0x00000100, 1037 0x3c230, 0xffffffff, 0x00000100, 1038 0x3c234, 0xffffffff, 0x00000100, 1039 0x3c250, 0xffffffff, 0x00000100, 1040 0x3c254, 0xffffffff, 0x00000100, 1041 0x3c258, 0xffffffff, 0x00000100, 1042 0x3c25c, 0xffffffff, 0x00000100, 1043 0x3c260, 0xffffffff, 0x00000100, 1044 0x3c27c, 0xffffffff, 0x00000100, 1045 0x3c278, 0xffffffff, 0x00000100, 1046 0x3c210, 0xffffffff, 0x06000100, 1047 0x3c290, 0xffffffff, 0x00000100, 1048 0x3c274, 0xffffffff, 0x00000100, 1049 0x3c2b4, 0xffffffff, 0x00000100, 1050 0x3c2b0, 0xffffffff, 0x00000100, 1051 0x3c270, 0xffffffff, 0x00000100, 1052 0x30800, 0xffffffff, 0xe0000000, 1053 0x3c020, 0xffffffff, 0x00010000, 1054 0x3c024, 0xffffffff, 0x00030002, 1055 0x3c028, 0xffffffff, 0x00040007, 1056 0x3c02c, 0xffffffff, 0x00060005, 1057 0x3c030, 0xffffffff, 0x00090008, 1058 0x3c034, 0xffffffff, 0x00010000, 1059 0x3c038, 0xffffffff, 0x00030002, 1060 0x3c03c, 0xffffffff, 0x00040007, 1061 0x3c040, 0xffffffff, 0x00060005, 1062 0x3c044, 0xffffffff, 0x00090008, 1063 0x3c048, 0xffffffff, 0x00010000, 1064 0x3c04c, 0xffffffff, 0x00030002, 1065 0x3c050, 0xffffffff, 0x00040007, 1066 0x3c054, 0xffffffff, 0x00060005, 1067 0x3c058, 0xffffffff, 0x00090008, 1068 0x3c05c, 0xffffffff, 0x00010000, 1069 0x3c060, 0xffffffff, 0x00030002, 1070 0x3c064, 0xffffffff, 0x00040007, 1071 0x3c068, 0xffffffff, 0x00060005, 1072 0x3c06c, 0xffffffff, 0x00090008, 1073 0x3c070, 0xffffffff, 0x00010000, 1074 0x3c074, 0xffffffff, 0x00030002, 1075 0x3c078, 0xffffffff, 0x00040007, 1076 0x3c07c, 0xffffffff, 0x00060005, 1077 0x3c080, 0xffffffff, 0x00090008, 1078 0x3c084, 0xffffffff, 0x00010000, 1079 0x3c088, 0xffffffff, 0x00030002, 1080 0x3c08c, 0xffffffff, 0x00040007, 1081 0x3c090, 0xffffffff, 0x00060005, 1082 0x3c094, 0xffffffff, 0x00090008, 1083 0x3c098, 0xffffffff, 0x00010000, 1084 0x3c09c, 0xffffffff, 0x00030002, 1085 0x3c0a0, 0xffffffff, 0x00040007, 1086 0x3c0a4, 0xffffffff, 0x00060005, 1087 0x3c0a8, 0xffffffff, 0x00090008, 1088 0x3c000, 0xffffffff, 0x96e00200, 1089 0x8708, 0xffffffff, 0x00900100, 1090 0xc424, 0xffffffff, 0x0020003f, 1091 0x38, 0xffffffff, 0x0140001c, 1092 0x3c, 0x000f0000, 0x000f0000, 1093 0x220, 0xffffffff, 0xC060000C, 1094 0x224, 0xc0000fff, 0x00000100, 1095 0xf90, 0xffffffff, 0x00000100, 1096 0xf98, 0x00000101, 0x00000000, 1097 0x20a8, 0xffffffff, 0x00000104, 1098 0x55e4, 0xff000fff, 0x00000100, 1099 0x30cc, 0xc0000fff, 0x00000104, 1100 0xc1e4, 0x00000001, 0x00000001, 1101 0xd00c, 0xff000ff0, 0x00000100, 1102 0xd80c, 0xff000ff0, 0x00000100 1103 }; 1104 1105 static const u32 spectre_golden_spm_registers[] = 1106 { 1107 0x30800, 0xe0ffffff, 0xe0000000 1108 }; 1109 1110 static const u32 spectre_golden_common_registers[] = 1111 { 1112 0xc770, 0xffffffff, 0x00000800, 1113 0xc774, 0xffffffff, 0x00000800, 1114 0xc798, 0xffffffff, 0x00007fbf, 1115 0xc79c, 0xffffffff, 0x00007faf 1116 }; 1117 1118 static const u32 spectre_golden_registers[] = 1119 { 1120 0x3c000, 0xffff1fff, 0x96940200, 1121 0x3c00c, 0xffff0001, 0xff000000, 1122 0x3c200, 0xfffc0fff, 0x00000100, 1123 0x6ed8, 0x00010101, 0x00010000, 1124 0x9834, 0xf00fffff, 0x00000400, 1125 0x9838, 0xfffffffc, 0x00020200, 1126 0x5bb0, 0x000000f0, 0x00000070, 1127 0x5bc0, 0xf0311fff, 0x80300000, 1128 0x98f8, 0x73773777, 0x12010001, 1129 0x9b7c, 0x00ff0000, 0x00fc0000, 1130 0x2f48, 0x73773777, 0x12010001, 1131 0x8a14, 0xf000003f, 0x00000007, 1132 0x8b24, 0xffffffff, 0x00ffffff, 1133 0x28350, 0x3f3f3fff, 0x00000082, 1134 0x28354, 0x0000003f, 0x00000000, 1135 0x3e78, 0x00000001, 0x00000002, 1136 0x913c, 0xffff03df, 0x00000004, 1137 0xc768, 0x00000008, 0x00000008, 1138 0x8c00, 0x000008ff, 0x00000800, 1139 0x9508, 0x00010000, 0x00010000, 1140 0xac0c, 0xffffffff, 0x54763210, 1141 0x214f8, 0x01ff01ff, 0x00000002, 1142 0x21498, 0x007ff800, 0x00200000, 1143 0x2015c, 0xffffffff, 0x00000f40, 1144 0x30934, 0xffffffff, 0x00000001 1145 }; 1146 1147 static const u32 spectre_mgcg_cgcg_init[] = 1148 { 1149 0xc420, 0xffffffff, 0xfffffffc, 1150 0x30800, 0xffffffff, 0xe0000000, 1151 0x3c2a0, 0xffffffff, 0x00000100, 1152 0x3c208, 0xffffffff, 0x00000100, 1153 0x3c2c0, 0xffffffff, 0x00000100, 1154 0x3c2c8, 0xffffffff, 0x00000100, 1155 0x3c2c4, 0xffffffff, 0x00000100, 1156 0x55e4, 0xffffffff, 0x00600100, 1157 0x3c280, 0xffffffff, 0x00000100, 1158 0x3c214, 0xffffffff, 0x06000100, 1159 0x3c220, 0xffffffff, 0x00000100, 1160 0x3c218, 0xffffffff, 0x06000100, 1161 0x3c204, 0xffffffff, 0x00000100, 1162 0x3c2e0, 0xffffffff, 0x00000100, 1163 0x3c224, 0xffffffff, 0x00000100, 1164 0x3c200, 0xffffffff, 0x00000100, 1165 0x3c230, 0xffffffff, 0x00000100, 1166 0x3c234, 0xffffffff, 0x00000100, 1167 0x3c250, 0xffffffff, 0x00000100, 1168 0x3c254, 0xffffffff, 0x00000100, 1169 0x3c258, 0xffffffff, 0x00000100, 1170 0x3c25c, 0xffffffff, 0x00000100, 1171 0x3c260, 0xffffffff, 0x00000100, 1172 0x3c27c, 0xffffffff, 0x00000100, 1173 0x3c278, 0xffffffff, 0x00000100, 1174 0x3c210, 0xffffffff, 0x06000100, 1175 0x3c290, 0xffffffff, 0x00000100, 1176 0x3c274, 0xffffffff, 0x00000100, 1177 0x3c2b4, 0xffffffff, 0x00000100, 1178 0x3c2b0, 0xffffffff, 0x00000100, 1179 0x3c270, 0xffffffff, 0x00000100, 1180 0x30800, 0xffffffff, 0xe0000000, 1181 0x3c020, 0xffffffff, 0x00010000, 1182 0x3c024, 0xffffffff, 0x00030002, 1183 0x3c028, 0xffffffff, 0x00040007, 1184 0x3c02c, 0xffffffff, 0x00060005, 1185 0x3c030, 0xffffffff, 0x00090008, 1186 0x3c034, 0xffffffff, 0x00010000, 1187 0x3c038, 0xffffffff, 0x00030002, 1188 0x3c03c, 0xffffffff, 0x00040007, 1189 0x3c040, 0xffffffff, 0x00060005, 1190 0x3c044, 0xffffffff, 0x00090008, 1191 0x3c048, 0xffffffff, 0x00010000, 1192 0x3c04c, 0xffffffff, 0x00030002, 1193 0x3c050, 0xffffffff, 0x00040007, 1194 0x3c054, 0xffffffff, 0x00060005, 1195 0x3c058, 0xffffffff, 0x00090008, 1196 0x3c05c, 0xffffffff, 0x00010000, 1197 0x3c060, 0xffffffff, 0x00030002, 1198 0x3c064, 0xffffffff, 0x00040007, 1199 0x3c068, 0xffffffff, 0x00060005, 1200 0x3c06c, 0xffffffff, 0x00090008, 1201 0x3c070, 0xffffffff, 0x00010000, 1202 0x3c074, 0xffffffff, 0x00030002, 1203 0x3c078, 0xffffffff, 0x00040007, 1204 0x3c07c, 0xffffffff, 0x00060005, 1205 0x3c080, 0xffffffff, 0x00090008, 1206 0x3c084, 0xffffffff, 0x00010000, 1207 0x3c088, 0xffffffff, 0x00030002, 1208 0x3c08c, 0xffffffff, 0x00040007, 1209 0x3c090, 0xffffffff, 0x00060005, 1210 0x3c094, 0xffffffff, 0x00090008, 1211 0x3c098, 0xffffffff, 0x00010000, 1212 0x3c09c, 0xffffffff, 0x00030002, 1213 0x3c0a0, 0xffffffff, 0x00040007, 1214 0x3c0a4, 0xffffffff, 0x00060005, 1215 0x3c0a8, 0xffffffff, 0x00090008, 1216 0x3c0ac, 0xffffffff, 0x00010000, 1217 0x3c0b0, 0xffffffff, 0x00030002, 1218 0x3c0b4, 0xffffffff, 0x00040007, 1219 0x3c0b8, 0xffffffff, 0x00060005, 1220 0x3c0bc, 0xffffffff, 0x00090008, 1221 0x3c000, 0xffffffff, 0x96e00200, 1222 0x8708, 0xffffffff, 0x00900100, 1223 0xc424, 0xffffffff, 0x0020003f, 1224 0x38, 0xffffffff, 0x0140001c, 1225 0x3c, 0x000f0000, 0x000f0000, 1226 0x220, 0xffffffff, 0xC060000C, 1227 0x224, 0xc0000fff, 0x00000100, 1228 0xf90, 0xffffffff, 0x00000100, 1229 0xf98, 0x00000101, 0x00000000, 1230 0x20a8, 0xffffffff, 0x00000104, 1231 0x55e4, 0xff000fff, 0x00000100, 1232 0x30cc, 0xc0000fff, 0x00000104, 1233 0xc1e4, 0x00000001, 0x00000001, 1234 0xd00c, 0xff000ff0, 0x00000100, 1235 0xd80c, 0xff000ff0, 0x00000100 1236 }; 1237 1238 static const u32 kalindi_golden_spm_registers[] = 1239 { 1240 0x30800, 0xe0ffffff, 0xe0000000 1241 }; 1242 1243 static const u32 kalindi_golden_common_registers[] = 1244 { 1245 0xc770, 0xffffffff, 0x00000800, 1246 0xc774, 0xffffffff, 0x00000800, 1247 0xc798, 0xffffffff, 0x00007fbf, 1248 0xc79c, 0xffffffff, 0x00007faf 1249 }; 1250 1251 static const u32 kalindi_golden_registers[] = 1252 { 1253 0x3c000, 0xffffdfff, 0x6e944040, 1254 0x55e4, 0xff607fff, 0xfc000100, 1255 0x3c220, 0xff000fff, 0x00000100, 1256 0x3c224, 0xff000fff, 0x00000100, 1257 0x3c200, 0xfffc0fff, 0x00000100, 1258 0x6ed8, 0x00010101, 0x00010000, 1259 0x9830, 0xffffffff, 0x00000000, 1260 0x9834, 0xf00fffff, 0x00000400, 1261 0x5bb0, 0x000000f0, 0x00000070, 1262 0x5bc0, 0xf0311fff, 0x80300000, 1263 0x98f8, 0x73773777, 0x12010001, 1264 0x98fc, 0xffffffff, 0x00000010, 1265 0x9b7c, 0x00ff0000, 0x00fc0000, 1266 0x8030, 0x00001f0f, 0x0000100a, 1267 0x2f48, 0x73773777, 0x12010001, 1268 0x2408, 0x000fffff, 0x000c007f, 1269 0x8a14, 0xf000003f, 0x00000007, 1270 0x8b24, 0x3fff3fff, 0x00ffcfff, 1271 0x30a04, 0x0000ff0f, 0x00000000, 1272 0x28a4c, 0x07ffffff, 0x06000000, 1273 0x4d8, 0x00000fff, 0x00000100, 1274 0x3e78, 0x00000001, 0x00000002, 1275 0xc768, 0x00000008, 0x00000008, 1276 0x8c00, 0x000000ff, 0x00000003, 1277 0x214f8, 0x01ff01ff, 0x00000002, 1278 0x21498, 0x007ff800, 0x00200000, 1279 0x2015c, 0xffffffff, 0x00000f40, 1280 0x88c4, 0x001f3ae3, 0x00000082, 1281 0x88d4, 0x0000001f, 0x00000010, 1282 0x30934, 0xffffffff, 0x00000000 1283 }; 1284 1285 static const u32 kalindi_mgcg_cgcg_init[] = 1286 { 1287 0xc420, 0xffffffff, 0xfffffffc, 1288 0x30800, 0xffffffff, 0xe0000000, 1289 0x3c2a0, 0xffffffff, 0x00000100, 1290 0x3c208, 0xffffffff, 0x00000100, 1291 0x3c2c0, 0xffffffff, 0x00000100, 1292 0x3c2c8, 0xffffffff, 0x00000100, 1293 0x3c2c4, 0xffffffff, 0x00000100, 1294 0x55e4, 0xffffffff, 0x00600100, 1295 0x3c280, 0xffffffff, 0x00000100, 1296 0x3c214, 0xffffffff, 0x06000100, 1297 0x3c220, 0xffffffff, 0x00000100, 1298 0x3c218, 0xffffffff, 0x06000100, 1299 0x3c204, 0xffffffff, 0x00000100, 1300 0x3c2e0, 0xffffffff, 0x00000100, 1301 0x3c224, 0xffffffff, 0x00000100, 1302 0x3c200, 0xffffffff, 0x00000100, 1303 0x3c230, 0xffffffff, 0x00000100, 1304 0x3c234, 0xffffffff, 0x00000100, 1305 0x3c250, 0xffffffff, 0x00000100, 1306 0x3c254, 0xffffffff, 0x00000100, 1307 0x3c258, 0xffffffff, 0x00000100, 1308 0x3c25c, 0xffffffff, 0x00000100, 1309 0x3c260, 0xffffffff, 0x00000100, 1310 0x3c27c, 0xffffffff, 0x00000100, 1311 0x3c278, 0xffffffff, 0x00000100, 1312 0x3c210, 0xffffffff, 0x06000100, 1313 0x3c290, 0xffffffff, 0x00000100, 1314 0x3c274, 0xffffffff, 0x00000100, 1315 0x3c2b4, 0xffffffff, 0x00000100, 1316 0x3c2b0, 0xffffffff, 0x00000100, 1317 0x3c270, 0xffffffff, 0x00000100, 1318 0x30800, 0xffffffff, 0xe0000000, 1319 0x3c020, 0xffffffff, 0x00010000, 1320 0x3c024, 0xffffffff, 0x00030002, 1321 0x3c028, 0xffffffff, 0x00040007, 1322 0x3c02c, 0xffffffff, 0x00060005, 1323 0x3c030, 0xffffffff, 0x00090008, 1324 0x3c034, 0xffffffff, 0x00010000, 1325 0x3c038, 0xffffffff, 0x00030002, 1326 0x3c03c, 0xffffffff, 0x00040007, 1327 0x3c040, 0xffffffff, 0x00060005, 1328 0x3c044, 0xffffffff, 0x00090008, 1329 0x3c000, 0xffffffff, 0x96e00200, 1330 0x8708, 0xffffffff, 0x00900100, 1331 0xc424, 0xffffffff, 0x0020003f, 1332 0x38, 0xffffffff, 0x0140001c, 1333 0x3c, 0x000f0000, 0x000f0000, 1334 0x220, 0xffffffff, 0xC060000C, 1335 0x224, 0xc0000fff, 0x00000100, 1336 0x20a8, 0xffffffff, 0x00000104, 1337 0x55e4, 0xff000fff, 0x00000100, 1338 0x30cc, 0xc0000fff, 0x00000104, 1339 0xc1e4, 0x00000001, 0x00000001, 1340 0xd00c, 0xff000ff0, 0x00000100, 1341 0xd80c, 0xff000ff0, 0x00000100 1342 }; 1343 1344 static const u32 hawaii_golden_spm_registers[] = 1345 { 1346 0x30800, 0xe0ffffff, 0xe0000000 1347 }; 1348 1349 static const u32 hawaii_golden_common_registers[] = 1350 { 1351 0x30800, 0xffffffff, 0xe0000000, 1352 0x28350, 0xffffffff, 0x3a00161a, 1353 0x28354, 0xffffffff, 0x0000002e, 1354 0x9a10, 0xffffffff, 0x00018208, 1355 0x98f8, 0xffffffff, 0x12011003 1356 }; 1357 1358 static const u32 hawaii_golden_registers[] = 1359 { 1360 0x3354, 0x00000333, 0x00000333, 1361 0x9a10, 0x00010000, 0x00058208, 1362 0x9830, 0xffffffff, 0x00000000, 1363 0x9834, 0xf00fffff, 0x00000400, 1364 0x9838, 0x0002021c, 0x00020200, 1365 0xc78, 0x00000080, 0x00000000, 1366 0x5bb0, 0x000000f0, 0x00000070, 1367 0x5bc0, 0xf0311fff, 0x80300000, 1368 0x350c, 0x00810000, 0x408af000, 1369 0x7030, 0x31000111, 0x00000011, 1370 0x2f48, 0x73773777, 0x12010001, 1371 0x2120, 0x0000007f, 0x0000001b, 1372 0x21dc, 0x00007fb6, 0x00002191, 1373 0x3628, 0x0000003f, 0x0000000a, 1374 0x362c, 0x0000003f, 0x0000000a, 1375 0x2ae4, 0x00073ffe, 0x000022a2, 1376 0x240c, 0x000007ff, 0x00000000, 1377 0x8bf0, 0x00002001, 0x00000001, 1378 0x8b24, 0xffffffff, 0x00ffffff, 1379 0x30a04, 0x0000ff0f, 0x00000000, 1380 0x28a4c, 0x07ffffff, 0x06000000, 1381 0x3e78, 0x00000001, 0x00000002, 1382 0xc768, 0x00000008, 0x00000008, 1383 0xc770, 0x00000f00, 0x00000800, 1384 0xc774, 0x00000f00, 0x00000800, 1385 0xc798, 0x00ffffff, 0x00ff7fbf, 1386 0xc79c, 0x00ffffff, 0x00ff7faf, 1387 0x8c00, 0x000000ff, 0x00000800, 1388 0xe40, 0x00001fff, 0x00001fff, 1389 0x9060, 0x0000007f, 0x00000020, 1390 0x9508, 0x00010000, 0x00010000, 1391 0xae00, 0x00100000, 0x000ff07c, 1392 0xac14, 0x000003ff, 0x0000000f, 1393 0xac10, 0xffffffff, 0x7564fdec, 1394 0xac0c, 0xffffffff, 0x3120b9a8, 1395 0xac08, 0x20000000, 0x0f9c0000 1396 }; 1397 1398 static const u32 hawaii_mgcg_cgcg_init[] = 1399 { 1400 0xc420, 0xffffffff, 0xfffffffd, 1401 0x30800, 0xffffffff, 0xe0000000, 1402 0x3c2a0, 0xffffffff, 0x00000100, 1403 0x3c208, 0xffffffff, 0x00000100, 1404 0x3c2c0, 0xffffffff, 0x00000100, 1405 0x3c2c8, 0xffffffff, 0x00000100, 1406 0x3c2c4, 0xffffffff, 0x00000100, 1407 0x55e4, 0xffffffff, 0x00200100, 1408 0x3c280, 0xffffffff, 0x00000100, 1409 0x3c214, 0xffffffff, 0x06000100, 1410 0x3c220, 0xffffffff, 0x00000100, 1411 0x3c218, 0xffffffff, 0x06000100, 1412 0x3c204, 0xffffffff, 0x00000100, 1413 0x3c2e0, 0xffffffff, 0x00000100, 1414 0x3c224, 0xffffffff, 0x00000100, 1415 0x3c200, 0xffffffff, 0x00000100, 1416 0x3c230, 0xffffffff, 0x00000100, 1417 0x3c234, 0xffffffff, 0x00000100, 1418 0x3c250, 0xffffffff, 0x00000100, 1419 0x3c254, 0xffffffff, 0x00000100, 1420 0x3c258, 0xffffffff, 0x00000100, 1421 0x3c25c, 0xffffffff, 0x00000100, 1422 0x3c260, 0xffffffff, 0x00000100, 1423 0x3c27c, 0xffffffff, 0x00000100, 1424 0x3c278, 0xffffffff, 0x00000100, 1425 0x3c210, 0xffffffff, 0x06000100, 1426 0x3c290, 0xffffffff, 0x00000100, 1427 0x3c274, 0xffffffff, 0x00000100, 1428 0x3c2b4, 0xffffffff, 0x00000100, 1429 0x3c2b0, 0xffffffff, 0x00000100, 1430 0x3c270, 0xffffffff, 0x00000100, 1431 0x30800, 0xffffffff, 0xe0000000, 1432 0x3c020, 0xffffffff, 0x00010000, 1433 0x3c024, 0xffffffff, 0x00030002, 1434 0x3c028, 0xffffffff, 0x00040007, 1435 0x3c02c, 0xffffffff, 0x00060005, 1436 0x3c030, 0xffffffff, 0x00090008, 1437 0x3c034, 0xffffffff, 0x00010000, 1438 0x3c038, 0xffffffff, 0x00030002, 1439 0x3c03c, 0xffffffff, 0x00040007, 1440 0x3c040, 0xffffffff, 0x00060005, 1441 0x3c044, 0xffffffff, 0x00090008, 1442 0x3c048, 0xffffffff, 0x00010000, 1443 0x3c04c, 0xffffffff, 0x00030002, 1444 0x3c050, 0xffffffff, 0x00040007, 1445 0x3c054, 0xffffffff, 0x00060005, 1446 0x3c058, 0xffffffff, 0x00090008, 1447 0x3c05c, 0xffffffff, 0x00010000, 1448 0x3c060, 0xffffffff, 0x00030002, 1449 0x3c064, 0xffffffff, 0x00040007, 1450 0x3c068, 0xffffffff, 0x00060005, 1451 0x3c06c, 0xffffffff, 0x00090008, 1452 0x3c070, 0xffffffff, 0x00010000, 1453 0x3c074, 0xffffffff, 0x00030002, 1454 0x3c078, 0xffffffff, 0x00040007, 1455 0x3c07c, 0xffffffff, 0x00060005, 1456 0x3c080, 0xffffffff, 0x00090008, 1457 0x3c084, 0xffffffff, 0x00010000, 1458 0x3c088, 0xffffffff, 0x00030002, 1459 0x3c08c, 0xffffffff, 0x00040007, 1460 0x3c090, 0xffffffff, 0x00060005, 1461 0x3c094, 0xffffffff, 0x00090008, 1462 0x3c098, 0xffffffff, 0x00010000, 1463 0x3c09c, 0xffffffff, 0x00030002, 1464 0x3c0a0, 0xffffffff, 0x00040007, 1465 0x3c0a4, 0xffffffff, 0x00060005, 1466 0x3c0a8, 0xffffffff, 0x00090008, 1467 0x3c0ac, 0xffffffff, 0x00010000, 1468 0x3c0b0, 0xffffffff, 0x00030002, 1469 0x3c0b4, 0xffffffff, 0x00040007, 1470 0x3c0b8, 0xffffffff, 0x00060005, 1471 0x3c0bc, 0xffffffff, 0x00090008, 1472 0x3c0c0, 0xffffffff, 0x00010000, 1473 0x3c0c4, 0xffffffff, 0x00030002, 1474 0x3c0c8, 0xffffffff, 0x00040007, 1475 0x3c0cc, 0xffffffff, 0x00060005, 1476 0x3c0d0, 0xffffffff, 0x00090008, 1477 0x3c0d4, 0xffffffff, 0x00010000, 1478 0x3c0d8, 0xffffffff, 0x00030002, 1479 0x3c0dc, 0xffffffff, 0x00040007, 1480 0x3c0e0, 0xffffffff, 0x00060005, 1481 0x3c0e4, 0xffffffff, 0x00090008, 1482 0x3c0e8, 0xffffffff, 0x00010000, 1483 0x3c0ec, 0xffffffff, 0x00030002, 1484 0x3c0f0, 0xffffffff, 0x00040007, 1485 0x3c0f4, 0xffffffff, 0x00060005, 1486 0x3c0f8, 0xffffffff, 0x00090008, 1487 0xc318, 0xffffffff, 0x00020200, 1488 0x3350, 0xffffffff, 0x00000200, 1489 0x15c0, 0xffffffff, 0x00000400, 1490 0x55e8, 0xffffffff, 0x00000000, 1491 0x2f50, 0xffffffff, 0x00000902, 1492 0x3c000, 0xffffffff, 0x96940200, 1493 0x8708, 0xffffffff, 0x00900100, 1494 0xc424, 0xffffffff, 0x0020003f, 1495 0x38, 0xffffffff, 0x0140001c, 1496 0x3c, 0x000f0000, 0x000f0000, 1497 0x220, 0xffffffff, 0xc060000c, 1498 0x224, 0xc0000fff, 0x00000100, 1499 0xf90, 0xffffffff, 0x00000100, 1500 0xf98, 0x00000101, 0x00000000, 1501 0x20a8, 0xffffffff, 0x00000104, 1502 0x55e4, 0xff000fff, 0x00000100, 1503 0x30cc, 0xc0000fff, 0x00000104, 1504 0xc1e4, 0x00000001, 0x00000001, 1505 0xd00c, 0xff000ff0, 0x00000100, 1506 0xd80c, 0xff000ff0, 0x00000100 1507 }; 1508 1509 static const u32 godavari_golden_registers[] = 1510 { 1511 0x55e4, 0xff607fff, 0xfc000100, 1512 0x6ed8, 0x00010101, 0x00010000, 1513 0x9830, 0xffffffff, 0x00000000, 1514 0x98302, 0xf00fffff, 0x00000400, 1515 0x6130, 0xffffffff, 0x00010000, 1516 0x5bb0, 0x000000f0, 0x00000070, 1517 0x5bc0, 0xf0311fff, 0x80300000, 1518 0x98f8, 0x73773777, 0x12010001, 1519 0x98fc, 0xffffffff, 0x00000010, 1520 0x8030, 0x00001f0f, 0x0000100a, 1521 0x2f48, 0x73773777, 0x12010001, 1522 0x2408, 0x000fffff, 0x000c007f, 1523 0x8a14, 0xf000003f, 0x00000007, 1524 0x8b24, 0xffffffff, 0x00ff0fff, 1525 0x30a04, 0x0000ff0f, 0x00000000, 1526 0x28a4c, 0x07ffffff, 0x06000000, 1527 0x4d8, 0x00000fff, 0x00000100, 1528 0xd014, 0x00010000, 0x00810001, 1529 0xd814, 0x00010000, 0x00810001, 1530 0x3e78, 0x00000001, 0x00000002, 1531 0xc768, 0x00000008, 0x00000008, 1532 0xc770, 0x00000f00, 0x00000800, 1533 0xc774, 0x00000f00, 0x00000800, 1534 0xc798, 0x00ffffff, 0x00ff7fbf, 1535 0xc79c, 0x00ffffff, 0x00ff7faf, 1536 0x8c00, 0x000000ff, 0x00000001, 1537 0x214f8, 0x01ff01ff, 0x00000002, 1538 0x21498, 0x007ff800, 0x00200000, 1539 0x2015c, 0xffffffff, 0x00000f40, 1540 0x88c4, 0x001f3ae3, 0x00000082, 1541 0x88d4, 0x0000001f, 0x00000010, 1542 0x30934, 0xffffffff, 0x00000000 1543 }; 1544 1545 1546 static void cik_init_golden_registers(struct radeon_device *rdev) 1547 { 1548 switch (rdev->family) { 1549 case CHIP_BONAIRE: 1550 radeon_program_register_sequence(rdev, 1551 bonaire_mgcg_cgcg_init, 1552 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init)); 1553 radeon_program_register_sequence(rdev, 1554 bonaire_golden_registers, 1555 (const u32)ARRAY_SIZE(bonaire_golden_registers)); 1556 radeon_program_register_sequence(rdev, 1557 bonaire_golden_common_registers, 1558 (const u32)ARRAY_SIZE(bonaire_golden_common_registers)); 1559 radeon_program_register_sequence(rdev, 1560 bonaire_golden_spm_registers, 1561 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers)); 1562 break; 1563 case CHIP_KABINI: 1564 radeon_program_register_sequence(rdev, 1565 kalindi_mgcg_cgcg_init, 1566 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 1567 radeon_program_register_sequence(rdev, 1568 kalindi_golden_registers, 1569 (const u32)ARRAY_SIZE(kalindi_golden_registers)); 1570 radeon_program_register_sequence(rdev, 1571 kalindi_golden_common_registers, 1572 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 1573 radeon_program_register_sequence(rdev, 1574 kalindi_golden_spm_registers, 1575 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 1576 break; 1577 case CHIP_MULLINS: 1578 radeon_program_register_sequence(rdev, 1579 kalindi_mgcg_cgcg_init, 1580 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 1581 radeon_program_register_sequence(rdev, 1582 godavari_golden_registers, 1583 (const u32)ARRAY_SIZE(godavari_golden_registers)); 1584 radeon_program_register_sequence(rdev, 1585 kalindi_golden_common_registers, 1586 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 1587 radeon_program_register_sequence(rdev, 1588 kalindi_golden_spm_registers, 1589 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 1590 break; 1591 case CHIP_KAVERI: 1592 radeon_program_register_sequence(rdev, 1593 spectre_mgcg_cgcg_init, 1594 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init)); 1595 radeon_program_register_sequence(rdev, 1596 spectre_golden_registers, 1597 (const u32)ARRAY_SIZE(spectre_golden_registers)); 1598 radeon_program_register_sequence(rdev, 1599 spectre_golden_common_registers, 1600 (const u32)ARRAY_SIZE(spectre_golden_common_registers)); 1601 radeon_program_register_sequence(rdev, 1602 spectre_golden_spm_registers, 1603 (const u32)ARRAY_SIZE(spectre_golden_spm_registers)); 1604 break; 1605 case CHIP_HAWAII: 1606 radeon_program_register_sequence(rdev, 1607 hawaii_mgcg_cgcg_init, 1608 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init)); 1609 radeon_program_register_sequence(rdev, 1610 hawaii_golden_registers, 1611 (const u32)ARRAY_SIZE(hawaii_golden_registers)); 1612 radeon_program_register_sequence(rdev, 1613 hawaii_golden_common_registers, 1614 (const u32)ARRAY_SIZE(hawaii_golden_common_registers)); 1615 radeon_program_register_sequence(rdev, 1616 hawaii_golden_spm_registers, 1617 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers)); 1618 break; 1619 default: 1620 break; 1621 } 1622 } 1623 1624 /** 1625 * cik_get_xclk - get the xclk 1626 * 1627 * @rdev: radeon_device pointer 1628 * 1629 * Returns the reference clock used by the gfx engine 1630 * (CIK). 1631 */ 1632 u32 cik_get_xclk(struct radeon_device *rdev) 1633 { 1634 u32 reference_clock = rdev->clock.spll.reference_freq; 1635 1636 if (rdev->flags & RADEON_IS_IGP) { 1637 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK) 1638 return reference_clock / 2; 1639 } else { 1640 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE) 1641 return reference_clock / 4; 1642 } 1643 return reference_clock; 1644 } 1645 1646 /** 1647 * cik_mm_rdoorbell - read a doorbell dword 1648 * 1649 * @rdev: radeon_device pointer 1650 * @index: doorbell index 1651 * 1652 * Returns the value in the doorbell aperture at the 1653 * requested doorbell index (CIK). 1654 */ 1655 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index) 1656 { 1657 if (index < rdev->doorbell.num_doorbells) { 1658 return readl(rdev->doorbell.ptr + index); 1659 } else { 1660 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 1661 return 0; 1662 } 1663 } 1664 1665 /** 1666 * cik_mm_wdoorbell - write a doorbell dword 1667 * 1668 * @rdev: radeon_device pointer 1669 * @index: doorbell index 1670 * @v: value to write 1671 * 1672 * Writes @v to the doorbell aperture at the 1673 * requested doorbell index (CIK). 1674 */ 1675 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v) 1676 { 1677 if (index < rdev->doorbell.num_doorbells) { 1678 writel(v, rdev->doorbell.ptr + index); 1679 } else { 1680 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 1681 } 1682 } 1683 1684 #define BONAIRE_IO_MC_REGS_SIZE 36 1685 1686 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] = 1687 { 1688 {0x00000070, 0x04400000}, 1689 {0x00000071, 0x80c01803}, 1690 {0x00000072, 0x00004004}, 1691 {0x00000073, 0x00000100}, 1692 {0x00000074, 0x00ff0000}, 1693 {0x00000075, 0x34000000}, 1694 {0x00000076, 0x08000014}, 1695 {0x00000077, 0x00cc08ec}, 1696 {0x00000078, 0x00000400}, 1697 {0x00000079, 0x00000000}, 1698 {0x0000007a, 0x04090000}, 1699 {0x0000007c, 0x00000000}, 1700 {0x0000007e, 0x4408a8e8}, 1701 {0x0000007f, 0x00000304}, 1702 {0x00000080, 0x00000000}, 1703 {0x00000082, 0x00000001}, 1704 {0x00000083, 0x00000002}, 1705 {0x00000084, 0xf3e4f400}, 1706 {0x00000085, 0x052024e3}, 1707 {0x00000087, 0x00000000}, 1708 {0x00000088, 0x01000000}, 1709 {0x0000008a, 0x1c0a0000}, 1710 {0x0000008b, 0xff010000}, 1711 {0x0000008d, 0xffffefff}, 1712 {0x0000008e, 0xfff3efff}, 1713 {0x0000008f, 0xfff3efbf}, 1714 {0x00000092, 0xf7ffffff}, 1715 {0x00000093, 0xffffff7f}, 1716 {0x00000095, 0x00101101}, 1717 {0x00000096, 0x00000fff}, 1718 {0x00000097, 0x00116fff}, 1719 {0x00000098, 0x60010000}, 1720 {0x00000099, 0x10010000}, 1721 {0x0000009a, 0x00006000}, 1722 {0x0000009b, 0x00001000}, 1723 {0x0000009f, 0x00b48000} 1724 }; 1725 1726 #define HAWAII_IO_MC_REGS_SIZE 22 1727 1728 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] = 1729 { 1730 {0x0000007d, 0x40000000}, 1731 {0x0000007e, 0x40180304}, 1732 {0x0000007f, 0x0000ff00}, 1733 {0x00000081, 0x00000000}, 1734 {0x00000083, 0x00000800}, 1735 {0x00000086, 0x00000000}, 1736 {0x00000087, 0x00000100}, 1737 {0x00000088, 0x00020100}, 1738 {0x00000089, 0x00000000}, 1739 {0x0000008b, 0x00040000}, 1740 {0x0000008c, 0x00000100}, 1741 {0x0000008e, 0xff010000}, 1742 {0x00000090, 0xffffefff}, 1743 {0x00000091, 0xfff3efff}, 1744 {0x00000092, 0xfff3efbf}, 1745 {0x00000093, 0xf7ffffff}, 1746 {0x00000094, 0xffffff7f}, 1747 {0x00000095, 0x00000fff}, 1748 {0x00000096, 0x00116fff}, 1749 {0x00000097, 0x60010000}, 1750 {0x00000098, 0x10010000}, 1751 {0x0000009f, 0x00c79000} 1752 }; 1753 1754 1755 /** 1756 * cik_srbm_select - select specific register instances 1757 * 1758 * @rdev: radeon_device pointer 1759 * @me: selected ME (micro engine) 1760 * @pipe: pipe 1761 * @queue: queue 1762 * @vmid: VMID 1763 * 1764 * Switches the currently active registers instances. Some 1765 * registers are instanced per VMID, others are instanced per 1766 * me/pipe/queue combination. 1767 */ 1768 static void cik_srbm_select(struct radeon_device *rdev, 1769 u32 me, u32 pipe, u32 queue, u32 vmid) 1770 { 1771 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) | 1772 MEID(me & 0x3) | 1773 VMID(vmid & 0xf) | 1774 QUEUEID(queue & 0x7)); 1775 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl); 1776 } 1777 1778 /* ucode loading */ 1779 /** 1780 * ci_mc_load_microcode - load MC ucode into the hw 1781 * 1782 * @rdev: radeon_device pointer 1783 * 1784 * Load the GDDR MC ucode into the hw (CIK). 1785 * Returns 0 on success, error on failure. 1786 */ 1787 int ci_mc_load_microcode(struct radeon_device *rdev) 1788 { 1789 const __be32 *fw_data = NULL; 1790 const __le32 *new_fw_data = NULL; 1791 u32 running, blackout = 0; 1792 u32 *io_mc_regs = NULL; 1793 const __le32 *new_io_mc_regs = NULL; 1794 int i, regs_size, ucode_size; 1795 1796 if (!rdev->mc_fw) 1797 return -EINVAL; 1798 1799 if (rdev->new_fw) { 1800 const struct mc_firmware_header_v1_0 *hdr = 1801 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data; 1802 1803 radeon_ucode_print_mc_hdr(&hdr->header); 1804 1805 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2); 1806 new_io_mc_regs = (const __le32 *) 1807 ((const char *)rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); 1808 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 1809 new_fw_data = (const __le32 *) 1810 ((const char *)rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1811 } else { 1812 ucode_size = rdev->mc_fw->datasize / 4; 1813 1814 switch (rdev->family) { 1815 case CHIP_BONAIRE: 1816 io_mc_regs = (u32 *)&bonaire_io_mc_regs; 1817 regs_size = BONAIRE_IO_MC_REGS_SIZE; 1818 break; 1819 case CHIP_HAWAII: 1820 io_mc_regs = (u32 *)&hawaii_io_mc_regs; 1821 regs_size = HAWAII_IO_MC_REGS_SIZE; 1822 break; 1823 default: 1824 return -EINVAL; 1825 } 1826 fw_data = (const __be32 *)rdev->mc_fw->data; 1827 } 1828 1829 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; 1830 1831 if (running == 0) { 1832 if (running) { 1833 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 1834 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 1835 } 1836 1837 /* reset the engine and set to writable */ 1838 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1839 WREG32(MC_SEQ_SUP_CNTL, 0x00000010); 1840 1841 /* load mc io regs */ 1842 for (i = 0; i < regs_size; i++) { 1843 if (rdev->new_fw) { 1844 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++)); 1845 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++)); 1846 } else { 1847 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); 1848 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); 1849 } 1850 } 1851 /* load the MC ucode */ 1852 for (i = 0; i < ucode_size; i++) { 1853 if (rdev->new_fw) 1854 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++)); 1855 else 1856 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); 1857 } 1858 1859 /* put the engine back into the active state */ 1860 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1861 WREG32(MC_SEQ_SUP_CNTL, 0x00000004); 1862 WREG32(MC_SEQ_SUP_CNTL, 0x00000001); 1863 1864 /* wait for training to complete */ 1865 for (i = 0; i < rdev->usec_timeout; i++) { 1866 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0) 1867 break; 1868 udelay(1); 1869 } 1870 for (i = 0; i < rdev->usec_timeout; i++) { 1871 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1) 1872 break; 1873 udelay(1); 1874 } 1875 1876 if (running) 1877 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout); 1878 } 1879 1880 return 0; 1881 } 1882 1883 /** 1884 * cik_init_microcode - load ucode images from disk 1885 * 1886 * @rdev: radeon_device pointer 1887 * 1888 * Use the firmware interface to load the ucode images into 1889 * the driver (not loaded into hw). 1890 * Returns 0 on success, error on failure. 1891 */ 1892 static int cik_init_microcode(struct radeon_device *rdev) 1893 { 1894 const char *chip_name; 1895 const char *new_chip_name; 1896 size_t pfp_req_size, me_req_size, ce_req_size, 1897 mec_req_size, rlc_req_size, mc_req_size = 0, 1898 sdma_req_size, smc_req_size = 0, mc2_req_size = 0; 1899 char fw_name[30]; 1900 int new_fw = 0; 1901 int err; 1902 int num_fw; 1903 1904 DRM_DEBUG("\n"); 1905 1906 switch (rdev->family) { 1907 case CHIP_BONAIRE: 1908 chip_name = "BONAIRE"; 1909 new_chip_name = "bonaire"; 1910 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1911 me_req_size = CIK_ME_UCODE_SIZE * 4; 1912 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1913 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1914 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; 1915 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4; 1916 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4; 1917 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1918 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4); 1919 num_fw = 8; 1920 break; 1921 case CHIP_HAWAII: 1922 chip_name = "HAWAII"; 1923 new_chip_name = "hawaii"; 1924 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1925 me_req_size = CIK_ME_UCODE_SIZE * 4; 1926 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1927 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1928 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; 1929 mc_req_size = HAWAII_MC_UCODE_SIZE * 4; 1930 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4; 1931 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1932 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4); 1933 num_fw = 8; 1934 break; 1935 case CHIP_KAVERI: 1936 chip_name = "KAVERI"; 1937 new_chip_name = "kaveri"; 1938 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1939 me_req_size = CIK_ME_UCODE_SIZE * 4; 1940 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1941 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1942 rlc_req_size = KV_RLC_UCODE_SIZE * 4; 1943 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1944 num_fw = 7; 1945 break; 1946 case CHIP_KABINI: 1947 chip_name = "KABINI"; 1948 new_chip_name = "kabini"; 1949 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1950 me_req_size = CIK_ME_UCODE_SIZE * 4; 1951 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1952 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1953 rlc_req_size = KB_RLC_UCODE_SIZE * 4; 1954 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1955 num_fw = 6; 1956 break; 1957 case CHIP_MULLINS: 1958 chip_name = "MULLINS"; 1959 new_chip_name = "mullins"; 1960 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1961 me_req_size = CIK_ME_UCODE_SIZE * 4; 1962 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1963 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1964 rlc_req_size = ML_RLC_UCODE_SIZE * 4; 1965 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1966 num_fw = 6; 1967 break; 1968 default: BUG(); 1969 } 1970 1971 DRM_INFO("Loading %s Microcode\n", new_chip_name); 1972 1973 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", new_chip_name); 1974 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 1975 if (err) { 1976 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name); 1977 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 1978 if (err) 1979 goto out; 1980 if (rdev->pfp_fw->datasize != pfp_req_size) { 1981 printk(KERN_ERR 1982 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 1983 rdev->pfp_fw->datasize, fw_name); 1984 err = -EINVAL; 1985 goto out; 1986 } 1987 } else { 1988 err = radeon_ucode_validate(rdev->pfp_fw); 1989 if (err) { 1990 printk(KERN_ERR 1991 "cik_fw: validation failed for firmware \"%s\"\n", 1992 fw_name); 1993 goto out; 1994 } else { 1995 new_fw++; 1996 } 1997 } 1998 1999 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", new_chip_name); 2000 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 2001 if (err) { 2002 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name); 2003 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 2004 if (err) 2005 goto out; 2006 if (rdev->me_fw->datasize != me_req_size) { 2007 printk(KERN_ERR 2008 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2009 rdev->me_fw->datasize, fw_name); 2010 err = -EINVAL; 2011 } 2012 } else { 2013 err = radeon_ucode_validate(rdev->me_fw); 2014 if (err) { 2015 printk(KERN_ERR 2016 "cik_fw: validation failed for firmware \"%s\"\n", 2017 fw_name); 2018 goto out; 2019 } else { 2020 new_fw++; 2021 } 2022 } 2023 2024 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", new_chip_name); 2025 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 2026 if (err) { 2027 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", chip_name); 2028 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 2029 if (err) 2030 goto out; 2031 if (rdev->ce_fw->datasize != ce_req_size) { 2032 printk(KERN_ERR 2033 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2034 rdev->ce_fw->datasize, fw_name); 2035 err = -EINVAL; 2036 } 2037 } else { 2038 err = radeon_ucode_validate(rdev->ce_fw); 2039 if (err) { 2040 printk(KERN_ERR 2041 "cik_fw: validation failed for firmware \"%s\"\n", 2042 fw_name); 2043 goto out; 2044 } else { 2045 new_fw++; 2046 } 2047 } 2048 2049 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mec", new_chip_name); 2050 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); 2051 if (err) { 2052 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mec", chip_name); 2053 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); 2054 if (err) 2055 goto out; 2056 if (rdev->mec_fw->datasize != mec_req_size) { 2057 printk(KERN_ERR 2058 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2059 rdev->mec_fw->datasize, fw_name); 2060 err = -EINVAL; 2061 } 2062 } else { 2063 err = radeon_ucode_validate(rdev->mec_fw); 2064 if (err) { 2065 printk(KERN_ERR 2066 "cik_fw: validation failed for firmware \"%s\"\n", 2067 fw_name); 2068 goto out; 2069 } else { 2070 new_fw++; 2071 } 2072 } 2073 2074 if (rdev->family == CHIP_KAVERI) { 2075 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mec2", new_chip_name); 2076 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev); 2077 if (err) { 2078 goto out; 2079 } else { 2080 err = radeon_ucode_validate(rdev->mec2_fw); 2081 if (err) { 2082 goto out; 2083 } else { 2084 new_fw++; 2085 } 2086 } 2087 } 2088 2089 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", new_chip_name); 2090 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 2091 if (err) { 2092 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", chip_name); 2093 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 2094 if (err) 2095 goto out; 2096 if (rdev->rlc_fw->datasize != rlc_req_size) { 2097 printk(KERN_ERR 2098 "cik_rlc: Bogus length %zu in firmware \"%s\"\n", 2099 rdev->rlc_fw->datasize, fw_name); 2100 err = -EINVAL; 2101 } 2102 } else { 2103 err = radeon_ucode_validate(rdev->rlc_fw); 2104 if (err) { 2105 printk(KERN_ERR 2106 "cik_fw: validation failed for firmware \"%s\"\n", 2107 fw_name); 2108 goto out; 2109 } else { 2110 new_fw++; 2111 } 2112 } 2113 2114 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_sdma", new_chip_name); 2115 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); 2116 if (err) { 2117 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_sdma", chip_name); 2118 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); 2119 if (err) 2120 goto out; 2121 if (rdev->sdma_fw->datasize != sdma_req_size) { 2122 printk(KERN_ERR 2123 "cik_sdma: Bogus length %zu in firmware \"%s\"\n", 2124 rdev->sdma_fw->datasize, fw_name); 2125 err = -EINVAL; 2126 } 2127 } else { 2128 err = radeon_ucode_validate(rdev->sdma_fw); 2129 if (err) { 2130 printk(KERN_ERR 2131 "cik_fw: validation failed for firmware \"%s\"\n", 2132 fw_name); 2133 goto out; 2134 } else { 2135 new_fw++; 2136 } 2137 } 2138 2139 /* No SMC, MC ucode on APUs */ 2140 if (!(rdev->flags & RADEON_IS_IGP)) { 2141 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", new_chip_name); 2142 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2143 if (err) { 2144 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc2", chip_name); 2145 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2146 if (err) { 2147 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", chip_name); 2148 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2149 if (err) 2150 goto out; 2151 } 2152 if ((rdev->mc_fw->datasize != mc_req_size) && 2153 (rdev->mc_fw->datasize != mc2_req_size)){ 2154 printk(KERN_ERR 2155 "cik_mc: Bogus length %zu in firmware \"%s\"\n", 2156 rdev->mc_fw->datasize, fw_name); 2157 err = -EINVAL; 2158 } 2159 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->datasize); 2160 } else { 2161 err = radeon_ucode_validate(rdev->mc_fw); 2162 if (err) { 2163 printk(KERN_ERR 2164 "cik_fw: validation failed for firmware \"%s\"\n", 2165 fw_name); 2166 goto out; 2167 } else { 2168 new_fw++; 2169 } 2170 } 2171 2172 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", new_chip_name); 2173 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 2174 if (err) { 2175 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", chip_name); 2176 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 2177 if (err) { 2178 printk(KERN_ERR 2179 "smc: error loading firmware \"%s\"\n", 2180 fw_name); 2181 release_firmware(rdev->smc_fw); 2182 rdev->smc_fw = NULL; 2183 err = 0; 2184 } else if (rdev->smc_fw->datasize != smc_req_size) { 2185 printk(KERN_ERR 2186 "cik_smc: Bogus length %zu in firmware \"%s\"\n", 2187 rdev->smc_fw->datasize, fw_name); 2188 err = -EINVAL; 2189 } 2190 } else { 2191 err = radeon_ucode_validate(rdev->smc_fw); 2192 if (err) { 2193 printk(KERN_ERR 2194 "cik_fw: validation failed for firmware \"%s\"\n", 2195 fw_name); 2196 goto out; 2197 } else { 2198 new_fw++; 2199 } 2200 } 2201 } 2202 2203 if (new_fw == 0) { 2204 rdev->new_fw = false; 2205 } else if (new_fw < num_fw) { 2206 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n"); 2207 err = -EINVAL; 2208 } else { 2209 rdev->new_fw = true; 2210 } 2211 2212 out: 2213 if (err) { 2214 if (err != -EINVAL) 2215 printk(KERN_ERR 2216 "cik_cp: Failed to load firmware \"%s\"\n", 2217 fw_name); 2218 release_firmware(rdev->pfp_fw); 2219 rdev->pfp_fw = NULL; 2220 release_firmware(rdev->me_fw); 2221 rdev->me_fw = NULL; 2222 release_firmware(rdev->ce_fw); 2223 rdev->ce_fw = NULL; 2224 release_firmware(rdev->mec_fw); 2225 rdev->mec_fw = NULL; 2226 release_firmware(rdev->mec2_fw); 2227 rdev->mec2_fw = NULL; 2228 release_firmware(rdev->rlc_fw); 2229 rdev->rlc_fw = NULL; 2230 release_firmware(rdev->sdma_fw); 2231 rdev->sdma_fw = NULL; 2232 release_firmware(rdev->mc_fw); 2233 rdev->mc_fw = NULL; 2234 release_firmware(rdev->smc_fw); 2235 rdev->smc_fw = NULL; 2236 } 2237 return err; 2238 } 2239 2240 /* 2241 * Core functions 2242 */ 2243 /** 2244 * cik_tiling_mode_table_init - init the hw tiling table 2245 * 2246 * @rdev: radeon_device pointer 2247 * 2248 * Starting with SI, the tiling setup is done globally in a 2249 * set of 32 tiling modes. Rather than selecting each set of 2250 * parameters per surface as on older asics, we just select 2251 * which index in the tiling table we want to use, and the 2252 * surface uses those parameters (CIK). 2253 */ 2254 static void cik_tiling_mode_table_init(struct radeon_device *rdev) 2255 { 2256 const u32 num_tile_mode_states = 32; 2257 const u32 num_secondary_tile_mode_states = 16; 2258 u32 reg_offset, gb_tile_moden, split_equal_to_row_size; 2259 u32 num_pipe_configs; 2260 u32 num_rbs = rdev->config.cik.max_backends_per_se * 2261 rdev->config.cik.max_shader_engines; 2262 2263 switch (rdev->config.cik.mem_row_size_in_kb) { 2264 case 1: 2265 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB; 2266 break; 2267 case 2: 2268 default: 2269 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB; 2270 break; 2271 case 4: 2272 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB; 2273 break; 2274 } 2275 2276 num_pipe_configs = rdev->config.cik.max_tile_pipes; 2277 if (num_pipe_configs > 8) 2278 num_pipe_configs = 16; 2279 2280 if (num_pipe_configs == 16) { 2281 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2282 switch (reg_offset) { 2283 case 0: 2284 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2285 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2286 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2287 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2288 break; 2289 case 1: 2290 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2291 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2292 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2293 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2294 break; 2295 case 2: 2296 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2297 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2298 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2299 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2300 break; 2301 case 3: 2302 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2303 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2304 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2305 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2306 break; 2307 case 4: 2308 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2309 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2310 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2311 TILE_SPLIT(split_equal_to_row_size)); 2312 break; 2313 case 5: 2314 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2315 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2316 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2317 break; 2318 case 6: 2319 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2320 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2321 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2322 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2323 break; 2324 case 7: 2325 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2326 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2327 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2328 TILE_SPLIT(split_equal_to_row_size)); 2329 break; 2330 case 8: 2331 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2332 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16)); 2333 break; 2334 case 9: 2335 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2336 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2337 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2338 break; 2339 case 10: 2340 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2341 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2342 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2343 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2344 break; 2345 case 11: 2346 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2347 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2348 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2349 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2350 break; 2351 case 12: 2352 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2353 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2354 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2355 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2356 break; 2357 case 13: 2358 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2359 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2360 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2361 break; 2362 case 14: 2363 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2364 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2365 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2366 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2367 break; 2368 case 16: 2369 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2370 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2371 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2372 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2373 break; 2374 case 17: 2375 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2376 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2377 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2378 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2379 break; 2380 case 27: 2381 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2382 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2383 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2384 break; 2385 case 28: 2386 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2387 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2388 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2389 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2390 break; 2391 case 29: 2392 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2393 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2394 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2395 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2396 break; 2397 case 30: 2398 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2399 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2400 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2401 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2402 break; 2403 default: 2404 gb_tile_moden = 0; 2405 break; 2406 } 2407 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2408 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2409 } 2410 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 2411 switch (reg_offset) { 2412 case 0: 2413 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2414 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2415 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2416 NUM_BANKS(ADDR_SURF_16_BANK)); 2417 break; 2418 case 1: 2419 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2420 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2421 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2422 NUM_BANKS(ADDR_SURF_16_BANK)); 2423 break; 2424 case 2: 2425 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2426 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2427 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2428 NUM_BANKS(ADDR_SURF_16_BANK)); 2429 break; 2430 case 3: 2431 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2432 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2433 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2434 NUM_BANKS(ADDR_SURF_16_BANK)); 2435 break; 2436 case 4: 2437 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2438 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2439 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2440 NUM_BANKS(ADDR_SURF_8_BANK)); 2441 break; 2442 case 5: 2443 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2444 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2445 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2446 NUM_BANKS(ADDR_SURF_4_BANK)); 2447 break; 2448 case 6: 2449 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2450 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2451 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2452 NUM_BANKS(ADDR_SURF_2_BANK)); 2453 break; 2454 case 8: 2455 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2456 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2457 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2458 NUM_BANKS(ADDR_SURF_16_BANK)); 2459 break; 2460 case 9: 2461 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2462 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2463 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2464 NUM_BANKS(ADDR_SURF_16_BANK)); 2465 break; 2466 case 10: 2467 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2468 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2469 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2470 NUM_BANKS(ADDR_SURF_16_BANK)); 2471 break; 2472 case 11: 2473 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2474 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2475 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2476 NUM_BANKS(ADDR_SURF_8_BANK)); 2477 break; 2478 case 12: 2479 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2480 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2481 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2482 NUM_BANKS(ADDR_SURF_4_BANK)); 2483 break; 2484 case 13: 2485 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2488 NUM_BANKS(ADDR_SURF_2_BANK)); 2489 break; 2490 case 14: 2491 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2492 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2493 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2494 NUM_BANKS(ADDR_SURF_2_BANK)); 2495 break; 2496 default: 2497 gb_tile_moden = 0; 2498 break; 2499 } 2500 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 2501 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2502 } 2503 } else if (num_pipe_configs == 8) { 2504 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2505 switch (reg_offset) { 2506 case 0: 2507 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2508 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2509 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2510 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2511 break; 2512 case 1: 2513 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2514 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2515 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2516 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2517 break; 2518 case 2: 2519 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2520 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2521 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2522 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2523 break; 2524 case 3: 2525 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2526 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2527 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2528 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2529 break; 2530 case 4: 2531 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2532 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2533 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2534 TILE_SPLIT(split_equal_to_row_size)); 2535 break; 2536 case 5: 2537 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2538 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2539 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2540 break; 2541 case 6: 2542 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2543 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2544 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2545 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2546 break; 2547 case 7: 2548 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2549 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2550 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2551 TILE_SPLIT(split_equal_to_row_size)); 2552 break; 2553 case 8: 2554 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2555 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2556 break; 2557 case 9: 2558 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2559 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2560 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2561 break; 2562 case 10: 2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2564 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2566 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2567 break; 2568 case 11: 2569 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2570 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2571 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2572 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2573 break; 2574 case 12: 2575 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2576 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2577 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2578 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2579 break; 2580 case 13: 2581 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2582 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2583 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2584 break; 2585 case 14: 2586 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2587 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2588 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2589 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2590 break; 2591 case 16: 2592 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2593 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2594 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2595 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2596 break; 2597 case 17: 2598 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2599 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2600 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2601 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2602 break; 2603 case 27: 2604 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2605 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2606 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2607 break; 2608 case 28: 2609 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2610 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2611 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2612 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2613 break; 2614 case 29: 2615 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2616 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2617 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2618 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2619 break; 2620 case 30: 2621 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2622 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2623 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2624 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2625 break; 2626 default: 2627 gb_tile_moden = 0; 2628 break; 2629 } 2630 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2631 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2632 } 2633 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 2634 switch (reg_offset) { 2635 case 0: 2636 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2637 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2638 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2639 NUM_BANKS(ADDR_SURF_16_BANK)); 2640 break; 2641 case 1: 2642 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2643 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2644 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2645 NUM_BANKS(ADDR_SURF_16_BANK)); 2646 break; 2647 case 2: 2648 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2649 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2650 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2651 NUM_BANKS(ADDR_SURF_16_BANK)); 2652 break; 2653 case 3: 2654 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2655 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2656 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2657 NUM_BANKS(ADDR_SURF_16_BANK)); 2658 break; 2659 case 4: 2660 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2661 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2662 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2663 NUM_BANKS(ADDR_SURF_8_BANK)); 2664 break; 2665 case 5: 2666 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2667 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2668 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2669 NUM_BANKS(ADDR_SURF_4_BANK)); 2670 break; 2671 case 6: 2672 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2673 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2674 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2675 NUM_BANKS(ADDR_SURF_2_BANK)); 2676 break; 2677 case 8: 2678 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2679 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2680 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2681 NUM_BANKS(ADDR_SURF_16_BANK)); 2682 break; 2683 case 9: 2684 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2685 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2686 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2687 NUM_BANKS(ADDR_SURF_16_BANK)); 2688 break; 2689 case 10: 2690 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2693 NUM_BANKS(ADDR_SURF_16_BANK)); 2694 break; 2695 case 11: 2696 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2699 NUM_BANKS(ADDR_SURF_16_BANK)); 2700 break; 2701 case 12: 2702 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2703 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2704 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2705 NUM_BANKS(ADDR_SURF_8_BANK)); 2706 break; 2707 case 13: 2708 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2711 NUM_BANKS(ADDR_SURF_4_BANK)); 2712 break; 2713 case 14: 2714 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2715 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2716 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2717 NUM_BANKS(ADDR_SURF_2_BANK)); 2718 break; 2719 default: 2720 gb_tile_moden = 0; 2721 break; 2722 } 2723 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 2724 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2725 } 2726 } else if (num_pipe_configs == 4) { 2727 if (num_rbs == 4) { 2728 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2729 switch (reg_offset) { 2730 case 0: 2731 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2732 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2733 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2734 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2735 break; 2736 case 1: 2737 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2738 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2739 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2740 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2741 break; 2742 case 2: 2743 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2744 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2745 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2746 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2747 break; 2748 case 3: 2749 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2750 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2751 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2752 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2753 break; 2754 case 4: 2755 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2756 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2757 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2758 TILE_SPLIT(split_equal_to_row_size)); 2759 break; 2760 case 5: 2761 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2762 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2763 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2764 break; 2765 case 6: 2766 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2767 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2768 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2769 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2770 break; 2771 case 7: 2772 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2773 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2774 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2775 TILE_SPLIT(split_equal_to_row_size)); 2776 break; 2777 case 8: 2778 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2779 PIPE_CONFIG(ADDR_SURF_P4_16x16)); 2780 break; 2781 case 9: 2782 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2783 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2784 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2785 break; 2786 case 10: 2787 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2788 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2789 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2790 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2791 break; 2792 case 11: 2793 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2794 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2795 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2796 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2797 break; 2798 case 12: 2799 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2800 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2801 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2802 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2803 break; 2804 case 13: 2805 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2806 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2807 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2808 break; 2809 case 14: 2810 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2811 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2812 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2813 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2814 break; 2815 case 16: 2816 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2817 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2818 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2819 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2820 break; 2821 case 17: 2822 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2823 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2824 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2825 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2826 break; 2827 case 27: 2828 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2829 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2830 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2831 break; 2832 case 28: 2833 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2834 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2835 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2836 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2837 break; 2838 case 29: 2839 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2840 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2841 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2842 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2843 break; 2844 case 30: 2845 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2846 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2847 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2848 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2849 break; 2850 default: 2851 gb_tile_moden = 0; 2852 break; 2853 } 2854 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2855 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2856 } 2857 } else if (num_rbs < 4) { 2858 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2859 switch (reg_offset) { 2860 case 0: 2861 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2862 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2863 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2864 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2865 break; 2866 case 1: 2867 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2868 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2869 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2870 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2871 break; 2872 case 2: 2873 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2874 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2875 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2876 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2877 break; 2878 case 3: 2879 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2880 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2881 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2882 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2883 break; 2884 case 4: 2885 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2886 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2887 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2888 TILE_SPLIT(split_equal_to_row_size)); 2889 break; 2890 case 5: 2891 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2892 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2893 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2894 break; 2895 case 6: 2896 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2897 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2898 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2899 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2900 break; 2901 case 7: 2902 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2903 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2904 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2905 TILE_SPLIT(split_equal_to_row_size)); 2906 break; 2907 case 8: 2908 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2909 PIPE_CONFIG(ADDR_SURF_P4_8x16)); 2910 break; 2911 case 9: 2912 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2913 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2914 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2915 break; 2916 case 10: 2917 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2918 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2919 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2920 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2921 break; 2922 case 11: 2923 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2924 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2925 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2926 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2927 break; 2928 case 12: 2929 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2930 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2931 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2932 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2933 break; 2934 case 13: 2935 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2936 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2937 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2938 break; 2939 case 14: 2940 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2941 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2942 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2943 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2944 break; 2945 case 16: 2946 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2947 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2948 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2949 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2950 break; 2951 case 17: 2952 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2953 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2954 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2955 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2956 break; 2957 case 27: 2958 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2959 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2960 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2961 break; 2962 case 28: 2963 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2964 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2965 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2966 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2967 break; 2968 case 29: 2969 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2970 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2971 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2972 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2973 break; 2974 case 30: 2975 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2976 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2977 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2978 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2979 break; 2980 default: 2981 gb_tile_moden = 0; 2982 break; 2983 } 2984 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2985 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2986 } 2987 } 2988 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 2989 switch (reg_offset) { 2990 case 0: 2991 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2992 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2993 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2994 NUM_BANKS(ADDR_SURF_16_BANK)); 2995 break; 2996 case 1: 2997 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2998 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2999 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3000 NUM_BANKS(ADDR_SURF_16_BANK)); 3001 break; 3002 case 2: 3003 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3004 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3005 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3006 NUM_BANKS(ADDR_SURF_16_BANK)); 3007 break; 3008 case 3: 3009 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3010 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3011 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3012 NUM_BANKS(ADDR_SURF_16_BANK)); 3013 break; 3014 case 4: 3015 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3016 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3017 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3018 NUM_BANKS(ADDR_SURF_16_BANK)); 3019 break; 3020 case 5: 3021 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3022 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3023 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3024 NUM_BANKS(ADDR_SURF_8_BANK)); 3025 break; 3026 case 6: 3027 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3028 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3029 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3030 NUM_BANKS(ADDR_SURF_4_BANK)); 3031 break; 3032 case 8: 3033 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3034 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3035 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3036 NUM_BANKS(ADDR_SURF_16_BANK)); 3037 break; 3038 case 9: 3039 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3040 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3041 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3042 NUM_BANKS(ADDR_SURF_16_BANK)); 3043 break; 3044 case 10: 3045 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3046 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3047 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3048 NUM_BANKS(ADDR_SURF_16_BANK)); 3049 break; 3050 case 11: 3051 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3052 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3053 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3054 NUM_BANKS(ADDR_SURF_16_BANK)); 3055 break; 3056 case 12: 3057 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3058 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3059 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3060 NUM_BANKS(ADDR_SURF_16_BANK)); 3061 break; 3062 case 13: 3063 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3064 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3065 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3066 NUM_BANKS(ADDR_SURF_8_BANK)); 3067 break; 3068 case 14: 3069 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3070 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3071 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3072 NUM_BANKS(ADDR_SURF_4_BANK)); 3073 break; 3074 default: 3075 gb_tile_moden = 0; 3076 break; 3077 } 3078 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 3079 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3080 } 3081 } else if (num_pipe_configs == 2) { 3082 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 3083 switch (reg_offset) { 3084 case 0: 3085 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3086 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3087 PIPE_CONFIG(ADDR_SURF_P2) | 3088 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 3089 break; 3090 case 1: 3091 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3092 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3093 PIPE_CONFIG(ADDR_SURF_P2) | 3094 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 3095 break; 3096 case 2: 3097 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3098 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3099 PIPE_CONFIG(ADDR_SURF_P2) | 3100 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 3101 break; 3102 case 3: 3103 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3104 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3105 PIPE_CONFIG(ADDR_SURF_P2) | 3106 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 3107 break; 3108 case 4: 3109 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3110 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3111 PIPE_CONFIG(ADDR_SURF_P2) | 3112 TILE_SPLIT(split_equal_to_row_size)); 3113 break; 3114 case 5: 3115 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3116 PIPE_CONFIG(ADDR_SURF_P2) | 3117 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3118 break; 3119 case 6: 3120 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3121 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3122 PIPE_CONFIG(ADDR_SURF_P2) | 3123 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 3124 break; 3125 case 7: 3126 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3127 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3128 PIPE_CONFIG(ADDR_SURF_P2) | 3129 TILE_SPLIT(split_equal_to_row_size)); 3130 break; 3131 case 8: 3132 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 3133 PIPE_CONFIG(ADDR_SURF_P2); 3134 break; 3135 case 9: 3136 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3137 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3138 PIPE_CONFIG(ADDR_SURF_P2)); 3139 break; 3140 case 10: 3141 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3142 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3143 PIPE_CONFIG(ADDR_SURF_P2) | 3144 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3145 break; 3146 case 11: 3147 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3148 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3149 PIPE_CONFIG(ADDR_SURF_P2) | 3150 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3151 break; 3152 case 12: 3153 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3154 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3155 PIPE_CONFIG(ADDR_SURF_P2) | 3156 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3157 break; 3158 case 13: 3159 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3160 PIPE_CONFIG(ADDR_SURF_P2) | 3161 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 3162 break; 3163 case 14: 3164 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3165 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3166 PIPE_CONFIG(ADDR_SURF_P2) | 3167 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3168 break; 3169 case 16: 3170 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3171 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3172 PIPE_CONFIG(ADDR_SURF_P2) | 3173 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3174 break; 3175 case 17: 3176 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3177 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3178 PIPE_CONFIG(ADDR_SURF_P2) | 3179 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3180 break; 3181 case 27: 3182 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3183 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3184 PIPE_CONFIG(ADDR_SURF_P2)); 3185 break; 3186 case 28: 3187 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3188 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3189 PIPE_CONFIG(ADDR_SURF_P2) | 3190 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3191 break; 3192 case 29: 3193 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3194 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3195 PIPE_CONFIG(ADDR_SURF_P2) | 3196 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3197 break; 3198 case 30: 3199 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3200 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3201 PIPE_CONFIG(ADDR_SURF_P2) | 3202 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3203 break; 3204 default: 3205 gb_tile_moden = 0; 3206 break; 3207 } 3208 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 3209 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3210 } 3211 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 3212 switch (reg_offset) { 3213 case 0: 3214 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3215 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3216 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3217 NUM_BANKS(ADDR_SURF_16_BANK)); 3218 break; 3219 case 1: 3220 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3221 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3222 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3223 NUM_BANKS(ADDR_SURF_16_BANK)); 3224 break; 3225 case 2: 3226 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3227 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3228 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3229 NUM_BANKS(ADDR_SURF_16_BANK)); 3230 break; 3231 case 3: 3232 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3233 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3234 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3235 NUM_BANKS(ADDR_SURF_16_BANK)); 3236 break; 3237 case 4: 3238 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3239 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3240 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3241 NUM_BANKS(ADDR_SURF_16_BANK)); 3242 break; 3243 case 5: 3244 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3245 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3246 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3247 NUM_BANKS(ADDR_SURF_16_BANK)); 3248 break; 3249 case 6: 3250 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3251 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3252 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3253 NUM_BANKS(ADDR_SURF_8_BANK)); 3254 break; 3255 case 8: 3256 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3257 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3258 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3259 NUM_BANKS(ADDR_SURF_16_BANK)); 3260 break; 3261 case 9: 3262 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3263 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3264 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3265 NUM_BANKS(ADDR_SURF_16_BANK)); 3266 break; 3267 case 10: 3268 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3269 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3270 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3271 NUM_BANKS(ADDR_SURF_16_BANK)); 3272 break; 3273 case 11: 3274 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3275 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3276 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3277 NUM_BANKS(ADDR_SURF_16_BANK)); 3278 break; 3279 case 12: 3280 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3281 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3282 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3283 NUM_BANKS(ADDR_SURF_16_BANK)); 3284 break; 3285 case 13: 3286 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3287 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3288 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3289 NUM_BANKS(ADDR_SURF_16_BANK)); 3290 break; 3291 case 14: 3292 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3293 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3294 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3295 NUM_BANKS(ADDR_SURF_8_BANK)); 3296 break; 3297 default: 3298 gb_tile_moden = 0; 3299 break; 3300 } 3301 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 3302 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3303 } 3304 } else 3305 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs); 3306 } 3307 3308 /** 3309 * cik_select_se_sh - select which SE, SH to address 3310 * 3311 * @rdev: radeon_device pointer 3312 * @se_num: shader engine to address 3313 * @sh_num: sh block to address 3314 * 3315 * Select which SE, SH combinations to address. Certain 3316 * registers are instanced per SE or SH. 0xffffffff means 3317 * broadcast to all SEs or SHs (CIK). 3318 */ 3319 static void cik_select_se_sh(struct radeon_device *rdev, 3320 u32 se_num, u32 sh_num) 3321 { 3322 u32 data = INSTANCE_BROADCAST_WRITES; 3323 3324 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) 3325 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES; 3326 else if (se_num == 0xffffffff) 3327 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num); 3328 else if (sh_num == 0xffffffff) 3329 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num); 3330 else 3331 data |= SH_INDEX(sh_num) | SE_INDEX(se_num); 3332 WREG32(GRBM_GFX_INDEX, data); 3333 } 3334 3335 /** 3336 * cik_create_bitmask - create a bitmask 3337 * 3338 * @bit_width: length of the mask 3339 * 3340 * create a variable length bit mask (CIK). 3341 * Returns the bitmask. 3342 */ 3343 static u32 cik_create_bitmask(u32 bit_width) 3344 { 3345 u32 i, mask = 0; 3346 3347 for (i = 0; i < bit_width; i++) { 3348 mask <<= 1; 3349 mask |= 1; 3350 } 3351 return mask; 3352 } 3353 3354 /** 3355 * cik_get_rb_disabled - computes the mask of disabled RBs 3356 * 3357 * @rdev: radeon_device pointer 3358 * @max_rb_num: max RBs (render backends) for the asic 3359 * @se_num: number of SEs (shader engines) for the asic 3360 * @sh_per_se: number of SH blocks per SE for the asic 3361 * 3362 * Calculates the bitmask of disabled RBs (CIK). 3363 * Returns the disabled RB bitmask. 3364 */ 3365 static u32 cik_get_rb_disabled(struct radeon_device *rdev, 3366 u32 max_rb_num_per_se, 3367 u32 sh_per_se) 3368 { 3369 u32 data, mask; 3370 3371 data = RREG32(CC_RB_BACKEND_DISABLE); 3372 if (data & 1) 3373 data &= BACKEND_DISABLE_MASK; 3374 else 3375 data = 0; 3376 data |= RREG32(GC_USER_RB_BACKEND_DISABLE); 3377 3378 data >>= BACKEND_DISABLE_SHIFT; 3379 3380 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se); 3381 3382 return data & mask; 3383 } 3384 3385 /** 3386 * cik_setup_rb - setup the RBs on the asic 3387 * 3388 * @rdev: radeon_device pointer 3389 * @se_num: number of SEs (shader engines) for the asic 3390 * @sh_per_se: number of SH blocks per SE for the asic 3391 * @max_rb_num: max RBs (render backends) for the asic 3392 * 3393 * Configures per-SE/SH RB registers (CIK). 3394 */ 3395 static void cik_setup_rb(struct radeon_device *rdev, 3396 u32 se_num, u32 sh_per_se, 3397 u32 max_rb_num_per_se) 3398 { 3399 int i, j; 3400 u32 data, mask; 3401 u32 disabled_rbs = 0; 3402 u32 enabled_rbs = 0; 3403 3404 for (i = 0; i < se_num; i++) { 3405 for (j = 0; j < sh_per_se; j++) { 3406 cik_select_se_sh(rdev, i, j); 3407 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se); 3408 if (rdev->family == CHIP_HAWAII) 3409 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH); 3410 else 3411 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH); 3412 } 3413 } 3414 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3415 3416 mask = 1; 3417 for (i = 0; i < max_rb_num_per_se * se_num; i++) { 3418 if (!(disabled_rbs & mask)) 3419 enabled_rbs |= mask; 3420 mask <<= 1; 3421 } 3422 3423 rdev->config.cik.backend_enable_mask = enabled_rbs; 3424 3425 for (i = 0; i < se_num; i++) { 3426 cik_select_se_sh(rdev, i, 0xffffffff); 3427 data = 0; 3428 for (j = 0; j < sh_per_se; j++) { 3429 switch (enabled_rbs & 3) { 3430 case 0: 3431 if (j == 0) 3432 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3); 3433 else 3434 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0); 3435 break; 3436 case 1: 3437 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2); 3438 break; 3439 case 2: 3440 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2); 3441 break; 3442 case 3: 3443 default: 3444 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2); 3445 break; 3446 } 3447 enabled_rbs >>= 2; 3448 } 3449 WREG32(PA_SC_RASTER_CONFIG, data); 3450 } 3451 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3452 } 3453 3454 /** 3455 * cik_gpu_init - setup the 3D engine 3456 * 3457 * @rdev: radeon_device pointer 3458 * 3459 * Configures the 3D engine and tiling configuration 3460 * registers so that the 3D engine is usable. 3461 */ 3462 static void cik_gpu_init(struct radeon_device *rdev) 3463 { 3464 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG); 3465 u32 mc_shared_chmap, mc_arb_ramcfg; 3466 u32 hdp_host_path_cntl; 3467 u32 tmp; 3468 int i, j; 3469 3470 switch (rdev->family) { 3471 case CHIP_BONAIRE: 3472 rdev->config.cik.max_shader_engines = 2; 3473 rdev->config.cik.max_tile_pipes = 4; 3474 rdev->config.cik.max_cu_per_sh = 7; 3475 rdev->config.cik.max_sh_per_se = 1; 3476 rdev->config.cik.max_backends_per_se = 2; 3477 rdev->config.cik.max_texture_channel_caches = 4; 3478 rdev->config.cik.max_gprs = 256; 3479 rdev->config.cik.max_gs_threads = 32; 3480 rdev->config.cik.max_hw_contexts = 8; 3481 3482 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3483 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3484 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3485 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3486 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3487 break; 3488 case CHIP_HAWAII: 3489 rdev->config.cik.max_shader_engines = 4; 3490 rdev->config.cik.max_tile_pipes = 16; 3491 rdev->config.cik.max_cu_per_sh = 11; 3492 rdev->config.cik.max_sh_per_se = 1; 3493 rdev->config.cik.max_backends_per_se = 4; 3494 rdev->config.cik.max_texture_channel_caches = 16; 3495 rdev->config.cik.max_gprs = 256; 3496 rdev->config.cik.max_gs_threads = 32; 3497 rdev->config.cik.max_hw_contexts = 8; 3498 3499 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3500 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3501 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3502 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3503 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN; 3504 break; 3505 case CHIP_KAVERI: 3506 rdev->config.cik.max_shader_engines = 1; 3507 rdev->config.cik.max_tile_pipes = 4; 3508 if ((rdev->pdev->device == 0x1304) || 3509 (rdev->pdev->device == 0x1305) || 3510 (rdev->pdev->device == 0x130C) || 3511 (rdev->pdev->device == 0x130F) || 3512 (rdev->pdev->device == 0x1310) || 3513 (rdev->pdev->device == 0x1311) || 3514 (rdev->pdev->device == 0x131C)) { 3515 rdev->config.cik.max_cu_per_sh = 8; 3516 rdev->config.cik.max_backends_per_se = 2; 3517 } else if ((rdev->pdev->device == 0x1309) || 3518 (rdev->pdev->device == 0x130A) || 3519 (rdev->pdev->device == 0x130D) || 3520 (rdev->pdev->device == 0x1313) || 3521 (rdev->pdev->device == 0x131D)) { 3522 rdev->config.cik.max_cu_per_sh = 6; 3523 rdev->config.cik.max_backends_per_se = 2; 3524 } else if ((rdev->pdev->device == 0x1306) || 3525 (rdev->pdev->device == 0x1307) || 3526 (rdev->pdev->device == 0x130B) || 3527 (rdev->pdev->device == 0x130E) || 3528 (rdev->pdev->device == 0x1315) || 3529 (rdev->pdev->device == 0x1318) || 3530 (rdev->pdev->device == 0x131B)) { 3531 rdev->config.cik.max_cu_per_sh = 4; 3532 rdev->config.cik.max_backends_per_se = 1; 3533 } else { 3534 rdev->config.cik.max_cu_per_sh = 3; 3535 rdev->config.cik.max_backends_per_se = 1; 3536 } 3537 rdev->config.cik.max_sh_per_se = 1; 3538 rdev->config.cik.max_texture_channel_caches = 4; 3539 rdev->config.cik.max_gprs = 256; 3540 rdev->config.cik.max_gs_threads = 16; 3541 rdev->config.cik.max_hw_contexts = 8; 3542 3543 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3544 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3545 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3546 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3547 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3548 break; 3549 case CHIP_KABINI: 3550 case CHIP_MULLINS: 3551 default: 3552 rdev->config.cik.max_shader_engines = 1; 3553 rdev->config.cik.max_tile_pipes = 2; 3554 rdev->config.cik.max_cu_per_sh = 2; 3555 rdev->config.cik.max_sh_per_se = 1; 3556 rdev->config.cik.max_backends_per_se = 1; 3557 rdev->config.cik.max_texture_channel_caches = 2; 3558 rdev->config.cik.max_gprs = 256; 3559 rdev->config.cik.max_gs_threads = 16; 3560 rdev->config.cik.max_hw_contexts = 8; 3561 3562 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3563 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3564 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3565 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3566 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3567 break; 3568 } 3569 3570 /* Initialize HDP */ 3571 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3572 WREG32((0x2c14 + j), 0x00000000); 3573 WREG32((0x2c18 + j), 0x00000000); 3574 WREG32((0x2c1c + j), 0x00000000); 3575 WREG32((0x2c20 + j), 0x00000000); 3576 WREG32((0x2c24 + j), 0x00000000); 3577 } 3578 3579 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 3580 3581 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 3582 3583 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 3584 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3585 3586 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes; 3587 rdev->config.cik.mem_max_burst_length_bytes = 256; 3588 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT; 3589 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 3590 if (rdev->config.cik.mem_row_size_in_kb > 4) 3591 rdev->config.cik.mem_row_size_in_kb = 4; 3592 /* XXX use MC settings? */ 3593 rdev->config.cik.shader_engine_tile_size = 32; 3594 rdev->config.cik.num_gpus = 1; 3595 rdev->config.cik.multi_gpu_tile_size = 64; 3596 3597 /* fix up row size */ 3598 gb_addr_config &= ~ROW_SIZE_MASK; 3599 switch (rdev->config.cik.mem_row_size_in_kb) { 3600 case 1: 3601 default: 3602 gb_addr_config |= ROW_SIZE(0); 3603 break; 3604 case 2: 3605 gb_addr_config |= ROW_SIZE(1); 3606 break; 3607 case 4: 3608 gb_addr_config |= ROW_SIZE(2); 3609 break; 3610 } 3611 3612 /* setup tiling info dword. gb_addr_config is not adequate since it does 3613 * not have bank info, so create a custom tiling dword. 3614 * bits 3:0 num_pipes 3615 * bits 7:4 num_banks 3616 * bits 11:8 group_size 3617 * bits 15:12 row_size 3618 */ 3619 rdev->config.cik.tile_config = 0; 3620 switch (rdev->config.cik.num_tile_pipes) { 3621 case 1: 3622 rdev->config.cik.tile_config |= (0 << 0); 3623 break; 3624 case 2: 3625 rdev->config.cik.tile_config |= (1 << 0); 3626 break; 3627 case 4: 3628 rdev->config.cik.tile_config |= (2 << 0); 3629 break; 3630 case 8: 3631 default: 3632 /* XXX what about 12? */ 3633 rdev->config.cik.tile_config |= (3 << 0); 3634 break; 3635 } 3636 rdev->config.cik.tile_config |= 3637 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4; 3638 rdev->config.cik.tile_config |= 3639 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8; 3640 rdev->config.cik.tile_config |= 3641 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12; 3642 3643 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3644 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3645 WREG32(DMIF_ADDR_CALC, gb_addr_config); 3646 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70); 3647 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70); 3648 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3649 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3650 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3651 3652 cik_tiling_mode_table_init(rdev); 3653 3654 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines, 3655 rdev->config.cik.max_sh_per_se, 3656 rdev->config.cik.max_backends_per_se); 3657 3658 rdev->config.cik.active_cus = 0; 3659 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 3660 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 3661 rdev->config.cik.active_cus += 3662 hweight32(cik_get_cu_active_bitmap(rdev, i, j)); 3663 } 3664 } 3665 3666 /* set HW defaults for 3D engine */ 3667 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); 3668 3669 WREG32(SX_DEBUG_1, 0x20); 3670 3671 WREG32(TA_CNTL_AUX, 0x00010000); 3672 3673 tmp = RREG32(SPI_CONFIG_CNTL); 3674 tmp |= 0x03000000; 3675 WREG32(SPI_CONFIG_CNTL, tmp); 3676 3677 WREG32(SQ_CONFIG, 1); 3678 3679 WREG32(DB_DEBUG, 0); 3680 3681 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff; 3682 tmp |= 0x00000400; 3683 WREG32(DB_DEBUG2, tmp); 3684 3685 tmp = RREG32(DB_DEBUG3) & ~0x0002021c; 3686 tmp |= 0x00020200; 3687 WREG32(DB_DEBUG3, tmp); 3688 3689 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000; 3690 tmp |= 0x00018208; 3691 WREG32(CB_HW_CONTROL, tmp); 3692 3693 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3694 3695 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) | 3696 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) | 3697 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) | 3698 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size))); 3699 3700 WREG32(VGT_NUM_INSTANCES, 1); 3701 3702 WREG32(CP_PERFMON_CNTL, 0); 3703 3704 WREG32(SQ_CONFIG, 0); 3705 3706 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3707 FORCE_EOV_MAX_REZ_CNT(255))); 3708 3709 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) | 3710 AUTO_INVLD_EN(ES_AND_GS_AUTO)); 3711 3712 WREG32(VGT_GS_VERTEX_REUSE, 16); 3713 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3714 3715 tmp = RREG32(HDP_MISC_CNTL); 3716 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3717 WREG32(HDP_MISC_CNTL, tmp); 3718 3719 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3720 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3721 3722 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3723 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER); 3724 3725 udelay(50); 3726 } 3727 3728 /* 3729 * GPU scratch registers helpers function. 3730 */ 3731 /** 3732 * cik_scratch_init - setup driver info for CP scratch regs 3733 * 3734 * @rdev: radeon_device pointer 3735 * 3736 * Set up the number and offset of the CP scratch registers. 3737 * NOTE: use of CP scratch registers is a legacy inferface and 3738 * is not used by default on newer asics (r6xx+). On newer asics, 3739 * memory buffers are used for fences rather than scratch regs. 3740 */ 3741 static void cik_scratch_init(struct radeon_device *rdev) 3742 { 3743 int i; 3744 3745 rdev->scratch.num_reg = 7; 3746 rdev->scratch.reg_base = SCRATCH_REG0; 3747 for (i = 0; i < rdev->scratch.num_reg; i++) { 3748 rdev->scratch.free[i] = true; 3749 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); 3750 } 3751 } 3752 3753 /** 3754 * cik_ring_test - basic gfx ring test 3755 * 3756 * @rdev: radeon_device pointer 3757 * @ring: radeon_ring structure holding ring information 3758 * 3759 * Allocate a scratch register and write to it using the gfx ring (CIK). 3760 * Provides a basic gfx ring test to verify that the ring is working. 3761 * Used by cik_cp_gfx_resume(); 3762 * Returns 0 on success, error on failure. 3763 */ 3764 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) 3765 { 3766 uint32_t scratch; 3767 uint32_t tmp = 0; 3768 unsigned i; 3769 int r; 3770 3771 r = radeon_scratch_get(rdev, &scratch); 3772 if (r) { 3773 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r); 3774 return r; 3775 } 3776 WREG32(scratch, 0xCAFEDEAD); 3777 r = radeon_ring_lock(rdev, ring, 3); 3778 if (r) { 3779 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r); 3780 radeon_scratch_free(rdev, scratch); 3781 return r; 3782 } 3783 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 3784 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2)); 3785 radeon_ring_write(ring, 0xDEADBEEF); 3786 radeon_ring_unlock_commit(rdev, ring, false); 3787 3788 for (i = 0; i < rdev->usec_timeout; i++) { 3789 tmp = RREG32(scratch); 3790 if (tmp == 0xDEADBEEF) 3791 break; 3792 DRM_UDELAY(1); 3793 } 3794 if (i < rdev->usec_timeout) { 3795 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 3796 } else { 3797 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n", 3798 ring->idx, scratch, tmp); 3799 r = -EINVAL; 3800 } 3801 radeon_scratch_free(rdev, scratch); 3802 return r; 3803 } 3804 3805 /** 3806 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp 3807 * 3808 * @rdev: radeon_device pointer 3809 * @ridx: radeon ring index 3810 * 3811 * Emits an hdp flush on the cp. 3812 */ 3813 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev, 3814 int ridx) 3815 { 3816 struct radeon_ring *ring = &rdev->ring[ridx]; 3817 u32 ref_and_mask; 3818 3819 switch (ring->idx) { 3820 case CAYMAN_RING_TYPE_CP1_INDEX: 3821 case CAYMAN_RING_TYPE_CP2_INDEX: 3822 default: 3823 switch (ring->me) { 3824 case 0: 3825 ref_and_mask = CP2 << ring->pipe; 3826 break; 3827 case 1: 3828 ref_and_mask = CP6 << ring->pipe; 3829 break; 3830 default: 3831 return; 3832 } 3833 break; 3834 case RADEON_RING_TYPE_GFX_INDEX: 3835 ref_and_mask = CP0; 3836 break; 3837 } 3838 3839 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 3840 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */ 3841 WAIT_REG_MEM_FUNCTION(3) | /* == */ 3842 WAIT_REG_MEM_ENGINE(1))); /* pfp */ 3843 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2); 3844 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2); 3845 radeon_ring_write(ring, ref_and_mask); 3846 radeon_ring_write(ring, ref_and_mask); 3847 radeon_ring_write(ring, 0x20); /* poll interval */ 3848 } 3849 3850 /** 3851 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring 3852 * 3853 * @rdev: radeon_device pointer 3854 * @fence: radeon fence object 3855 * 3856 * Emits a fence sequnce number on the gfx ring and flushes 3857 * GPU caches. 3858 */ 3859 void cik_fence_gfx_ring_emit(struct radeon_device *rdev, 3860 struct radeon_fence *fence) 3861 { 3862 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3863 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3864 3865 /* EVENT_WRITE_EOP - flush caches, send int */ 3866 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3867 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3868 EOP_TC_ACTION_EN | 3869 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3870 EVENT_INDEX(5))); 3871 radeon_ring_write(ring, addr & 0xfffffffc); 3872 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2)); 3873 radeon_ring_write(ring, fence->seq); 3874 radeon_ring_write(ring, 0); 3875 } 3876 3877 /** 3878 * cik_fence_compute_ring_emit - emit a fence on the compute ring 3879 * 3880 * @rdev: radeon_device pointer 3881 * @fence: radeon fence object 3882 * 3883 * Emits a fence sequnce number on the compute ring and flushes 3884 * GPU caches. 3885 */ 3886 void cik_fence_compute_ring_emit(struct radeon_device *rdev, 3887 struct radeon_fence *fence) 3888 { 3889 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3890 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3891 3892 /* RELEASE_MEM - flush caches, send int */ 3893 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5)); 3894 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3895 EOP_TC_ACTION_EN | 3896 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3897 EVENT_INDEX(5))); 3898 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2)); 3899 radeon_ring_write(ring, addr & 0xfffffffc); 3900 radeon_ring_write(ring, upper_32_bits(addr)); 3901 radeon_ring_write(ring, fence->seq); 3902 radeon_ring_write(ring, 0); 3903 } 3904 3905 /** 3906 * cik_semaphore_ring_emit - emit a semaphore on the CP ring 3907 * 3908 * @rdev: radeon_device pointer 3909 * @ring: radeon ring buffer object 3910 * @semaphore: radeon semaphore object 3911 * @emit_wait: Is this a sempahore wait? 3912 * 3913 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP 3914 * from running ahead of semaphore waits. 3915 */ 3916 bool cik_semaphore_ring_emit(struct radeon_device *rdev, 3917 struct radeon_ring *ring, 3918 struct radeon_semaphore *semaphore, 3919 bool emit_wait) 3920 { 3921 uint64_t addr = semaphore->gpu_addr; 3922 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL; 3923 3924 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1)); 3925 radeon_ring_write(ring, lower_32_bits(addr)); 3926 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel); 3927 3928 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) { 3929 /* Prevent the PFP from running ahead of the semaphore wait */ 3930 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 3931 radeon_ring_write(ring, 0x0); 3932 } 3933 3934 return true; 3935 } 3936 3937 /** 3938 * cik_copy_cpdma - copy pages using the CP DMA engine 3939 * 3940 * @rdev: radeon_device pointer 3941 * @src_offset: src GPU address 3942 * @dst_offset: dst GPU address 3943 * @num_gpu_pages: number of GPU pages to xfer 3944 * @fence: radeon fence object 3945 * 3946 * Copy GPU paging using the CP DMA engine (CIK+). 3947 * Used by the radeon ttm implementation to move pages if 3948 * registered as the asic copy callback. 3949 */ 3950 int cik_copy_cpdma(struct radeon_device *rdev, 3951 uint64_t src_offset, uint64_t dst_offset, 3952 unsigned num_gpu_pages, 3953 struct radeon_fence **fence) 3954 { 3955 struct radeon_semaphore *sem = NULL; 3956 int ring_index = rdev->asic->copy.blit_ring_index; 3957 struct radeon_ring *ring = &rdev->ring[ring_index]; 3958 u32 size_in_bytes, cur_size_in_bytes, control; 3959 int i, num_loops; 3960 int r = 0; 3961 3962 r = radeon_semaphore_create(rdev, &sem); 3963 if (r) { 3964 DRM_ERROR("radeon: moving bo (%d).\n", r); 3965 return r; 3966 } 3967 3968 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); 3969 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); 3970 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18); 3971 if (r) { 3972 DRM_ERROR("radeon: moving bo (%d).\n", r); 3973 radeon_semaphore_free(rdev, &sem, NULL); 3974 return r; 3975 } 3976 3977 radeon_semaphore_sync_to(sem, *fence); 3978 radeon_semaphore_sync_rings(rdev, sem, ring->idx); 3979 3980 for (i = 0; i < num_loops; i++) { 3981 cur_size_in_bytes = size_in_bytes; 3982 if (cur_size_in_bytes > 0x1fffff) 3983 cur_size_in_bytes = 0x1fffff; 3984 size_in_bytes -= cur_size_in_bytes; 3985 control = 0; 3986 if (size_in_bytes == 0) 3987 control |= PACKET3_DMA_DATA_CP_SYNC; 3988 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 3989 radeon_ring_write(ring, control); 3990 radeon_ring_write(ring, lower_32_bits(src_offset)); 3991 radeon_ring_write(ring, upper_32_bits(src_offset)); 3992 radeon_ring_write(ring, lower_32_bits(dst_offset)); 3993 radeon_ring_write(ring, upper_32_bits(dst_offset)); 3994 radeon_ring_write(ring, cur_size_in_bytes); 3995 src_offset += cur_size_in_bytes; 3996 dst_offset += cur_size_in_bytes; 3997 } 3998 3999 r = radeon_fence_emit(rdev, fence, ring->idx); 4000 if (r) { 4001 radeon_ring_unlock_undo(rdev, ring); 4002 radeon_semaphore_free(rdev, &sem, NULL); 4003 return r; 4004 } 4005 4006 radeon_ring_unlock_commit(rdev, ring, false); 4007 radeon_semaphore_free(rdev, &sem, *fence); 4008 4009 return r; 4010 } 4011 4012 /* 4013 * IB stuff 4014 */ 4015 /** 4016 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring 4017 * 4018 * @rdev: radeon_device pointer 4019 * @ib: radeon indirect buffer object 4020 * 4021 * Emits an DE (drawing engine) or CE (constant engine) IB 4022 * on the gfx ring. IBs are usually generated by userspace 4023 * acceleration drivers and submitted to the kernel for 4024 * sheduling on the ring. This function schedules the IB 4025 * on the gfx ring for execution by the GPU. 4026 */ 4027 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 4028 { 4029 struct radeon_ring *ring = &rdev->ring[ib->ring]; 4030 u32 header, control = INDIRECT_BUFFER_VALID; 4031 4032 if (ib->is_const_ib) { 4033 /* set switch buffer packet before const IB */ 4034 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 4035 radeon_ring_write(ring, 0); 4036 4037 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 4038 } else { 4039 u32 next_rptr; 4040 if (ring->rptr_save_reg) { 4041 next_rptr = ring->wptr + 3 + 4; 4042 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 4043 radeon_ring_write(ring, ((ring->rptr_save_reg - 4044 PACKET3_SET_UCONFIG_REG_START) >> 2)); 4045 radeon_ring_write(ring, next_rptr); 4046 } else if (rdev->wb.enabled) { 4047 next_rptr = ring->wptr + 5 + 4; 4048 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 4049 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1)); 4050 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 4051 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr)); 4052 radeon_ring_write(ring, next_rptr); 4053 } 4054 4055 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 4056 } 4057 4058 control |= ib->length_dw | 4059 (ib->vm ? (ib->vm->id << 24) : 0); 4060 4061 radeon_ring_write(ring, header); 4062 radeon_ring_write(ring, 4063 #ifdef __BIG_ENDIAN 4064 (2 << 0) | 4065 #endif 4066 (ib->gpu_addr & 0xFFFFFFFC)); 4067 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 4068 radeon_ring_write(ring, control); 4069 } 4070 4071 /** 4072 * cik_ib_test - basic gfx ring IB test 4073 * 4074 * @rdev: radeon_device pointer 4075 * @ring: radeon_ring structure holding ring information 4076 * 4077 * Allocate an IB and execute it on the gfx ring (CIK). 4078 * Provides a basic gfx ring test to verify that IBs are working. 4079 * Returns 0 on success, error on failure. 4080 */ 4081 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) 4082 { 4083 struct radeon_ib ib; 4084 uint32_t scratch; 4085 uint32_t tmp = 0; 4086 unsigned i; 4087 int r; 4088 4089 r = radeon_scratch_get(rdev, &scratch); 4090 if (r) { 4091 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r); 4092 return r; 4093 } 4094 WREG32(scratch, 0xCAFEDEAD); 4095 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); 4096 if (r) { 4097 DRM_ERROR("radeon: failed to get ib (%d).\n", r); 4098 radeon_scratch_free(rdev, scratch); 4099 return r; 4100 } 4101 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1); 4102 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2); 4103 ib.ptr[2] = 0xDEADBEEF; 4104 ib.length_dw = 3; 4105 r = radeon_ib_schedule(rdev, &ib, NULL, false); 4106 if (r) { 4107 radeon_scratch_free(rdev, scratch); 4108 radeon_ib_free(rdev, &ib); 4109 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); 4110 return r; 4111 } 4112 r = radeon_fence_wait(ib.fence, false); 4113 if (r) { 4114 DRM_ERROR("radeon: fence wait failed (%d).\n", r); 4115 radeon_scratch_free(rdev, scratch); 4116 radeon_ib_free(rdev, &ib); 4117 return r; 4118 } 4119 for (i = 0; i < rdev->usec_timeout; i++) { 4120 tmp = RREG32(scratch); 4121 if (tmp == 0xDEADBEEF) 4122 break; 4123 DRM_UDELAY(1); 4124 } 4125 if (i < rdev->usec_timeout) { 4126 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i); 4127 } else { 4128 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n", 4129 scratch, tmp); 4130 r = -EINVAL; 4131 } 4132 radeon_scratch_free(rdev, scratch); 4133 radeon_ib_free(rdev, &ib); 4134 return r; 4135 } 4136 4137 /* 4138 * CP. 4139 * On CIK, gfx and compute now have independant command processors. 4140 * 4141 * GFX 4142 * Gfx consists of a single ring and can process both gfx jobs and 4143 * compute jobs. The gfx CP consists of three microengines (ME): 4144 * PFP - Pre-Fetch Parser 4145 * ME - Micro Engine 4146 * CE - Constant Engine 4147 * The PFP and ME make up what is considered the Drawing Engine (DE). 4148 * The CE is an asynchronous engine used for updating buffer desciptors 4149 * used by the DE so that they can be loaded into cache in parallel 4150 * while the DE is processing state update packets. 4151 * 4152 * Compute 4153 * The compute CP consists of two microengines (ME): 4154 * MEC1 - Compute MicroEngine 1 4155 * MEC2 - Compute MicroEngine 2 4156 * Each MEC supports 4 compute pipes and each pipe supports 8 queues. 4157 * The queues are exposed to userspace and are programmed directly 4158 * by the compute runtime. 4159 */ 4160 /** 4161 * cik_cp_gfx_enable - enable/disable the gfx CP MEs 4162 * 4163 * @rdev: radeon_device pointer 4164 * @enable: enable or disable the MEs 4165 * 4166 * Halts or unhalts the gfx MEs. 4167 */ 4168 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable) 4169 { 4170 if (enable) 4171 WREG32(CP_ME_CNTL, 0); 4172 else { 4173 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 4174 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 4175 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT)); 4176 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 4177 } 4178 udelay(50); 4179 } 4180 4181 /** 4182 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode 4183 * 4184 * @rdev: radeon_device pointer 4185 * 4186 * Loads the gfx PFP, ME, and CE ucode. 4187 * Returns 0 for success, -EINVAL if the ucode is not available. 4188 */ 4189 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev) 4190 { 4191 int i; 4192 4193 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw) 4194 return -EINVAL; 4195 4196 cik_cp_gfx_enable(rdev, false); 4197 4198 if (rdev->new_fw) { 4199 const struct gfx_firmware_header_v1_0 *pfp_hdr = 4200 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; 4201 const struct gfx_firmware_header_v1_0 *ce_hdr = 4202 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; 4203 const struct gfx_firmware_header_v1_0 *me_hdr = 4204 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; 4205 const __le32 *fw_data; 4206 u32 fw_size; 4207 4208 radeon_ucode_print_gfx_hdr(&pfp_hdr->header); 4209 radeon_ucode_print_gfx_hdr(&ce_hdr->header); 4210 radeon_ucode_print_gfx_hdr(&me_hdr->header); 4211 4212 /* PFP */ 4213 fw_data = (const __le32 *) 4214 ((const char *)rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 4215 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 4216 WREG32(CP_PFP_UCODE_ADDR, 0); 4217 for (i = 0; i < fw_size; i++) 4218 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 4219 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version)); 4220 4221 /* CE */ 4222 fw_data = (const __le32 *) 4223 ((const char *)rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 4224 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 4225 WREG32(CP_CE_UCODE_ADDR, 0); 4226 for (i = 0; i < fw_size; i++) 4227 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 4228 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version)); 4229 4230 /* ME */ 4231 fw_data = (const __be32 *) 4232 ((const char *)rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 4233 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 4234 WREG32(CP_ME_RAM_WADDR, 0); 4235 for (i = 0; i < fw_size; i++) 4236 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 4237 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version)); 4238 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version)); 4239 } else { 4240 const __be32 *fw_data; 4241 4242 /* PFP */ 4243 fw_data = (const __be32 *)rdev->pfp_fw->data; 4244 WREG32(CP_PFP_UCODE_ADDR, 0); 4245 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++) 4246 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 4247 WREG32(CP_PFP_UCODE_ADDR, 0); 4248 4249 /* CE */ 4250 fw_data = (const __be32 *)rdev->ce_fw->data; 4251 WREG32(CP_CE_UCODE_ADDR, 0); 4252 for (i = 0; i < CIK_CE_UCODE_SIZE; i++) 4253 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); 4254 WREG32(CP_CE_UCODE_ADDR, 0); 4255 4256 /* ME */ 4257 fw_data = (const __be32 *)rdev->me_fw->data; 4258 WREG32(CP_ME_RAM_WADDR, 0); 4259 for (i = 0; i < CIK_ME_UCODE_SIZE; i++) 4260 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 4261 WREG32(CP_ME_RAM_WADDR, 0); 4262 } 4263 4264 return 0; 4265 } 4266 4267 /** 4268 * cik_cp_gfx_start - start the gfx ring 4269 * 4270 * @rdev: radeon_device pointer 4271 * 4272 * Enables the ring and loads the clear state context and other 4273 * packets required to init the ring. 4274 * Returns 0 for success, error for failure. 4275 */ 4276 static int cik_cp_gfx_start(struct radeon_device *rdev) 4277 { 4278 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4279 int r, i; 4280 4281 /* init the CP */ 4282 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1); 4283 WREG32(CP_ENDIAN_SWAP, 0); 4284 WREG32(CP_DEVICE_ID, 1); 4285 4286 cik_cp_gfx_enable(rdev, true); 4287 4288 r = radeon_ring_lock(rdev, ring, cik_default_size + 17); 4289 if (r) { 4290 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 4291 return r; 4292 } 4293 4294 /* init the CE partitions. CE only used for gfx on CIK */ 4295 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 4296 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 4297 radeon_ring_write(ring, 0x8000); 4298 radeon_ring_write(ring, 0x8000); 4299 4300 /* setup clear context state */ 4301 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4302 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 4303 4304 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 4305 radeon_ring_write(ring, 0x80000000); 4306 radeon_ring_write(ring, 0x80000000); 4307 4308 for (i = 0; i < cik_default_size; i++) 4309 radeon_ring_write(ring, cik_default_state[i]); 4310 4311 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4312 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 4313 4314 /* set clear context state */ 4315 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 4316 radeon_ring_write(ring, 0); 4317 4318 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 4319 radeon_ring_write(ring, 0x00000316); 4320 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 4321 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */ 4322 4323 radeon_ring_unlock_commit(rdev, ring, false); 4324 4325 return 0; 4326 } 4327 4328 /** 4329 * cik_cp_gfx_fini - stop the gfx ring 4330 * 4331 * @rdev: radeon_device pointer 4332 * 4333 * Stop the gfx ring and tear down the driver ring 4334 * info. 4335 */ 4336 static void cik_cp_gfx_fini(struct radeon_device *rdev) 4337 { 4338 cik_cp_gfx_enable(rdev, false); 4339 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 4340 } 4341 4342 /** 4343 * cik_cp_gfx_resume - setup the gfx ring buffer registers 4344 * 4345 * @rdev: radeon_device pointer 4346 * 4347 * Program the location and size of the gfx ring buffer 4348 * and test it to make sure it's working. 4349 * Returns 0 for success, error for failure. 4350 */ 4351 static int cik_cp_gfx_resume(struct radeon_device *rdev) 4352 { 4353 struct radeon_ring *ring; 4354 u32 tmp; 4355 u32 rb_bufsz; 4356 u64 rb_addr; 4357 int r; 4358 4359 WREG32(CP_SEM_WAIT_TIMER, 0x0); 4360 if (rdev->family != CHIP_HAWAII) 4361 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 4362 4363 /* Set the write pointer delay */ 4364 WREG32(CP_RB_WPTR_DELAY, 0); 4365 4366 /* set the RB to use vmid 0 */ 4367 WREG32(CP_RB_VMID, 0); 4368 4369 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 4370 4371 /* ring 0 - compute and gfx */ 4372 /* Set ring buffer size */ 4373 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4374 rb_bufsz = order_base_2(ring->ring_size / 8); 4375 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 4376 #ifdef __BIG_ENDIAN 4377 tmp |= BUF_SWAP_32BIT; 4378 #endif 4379 WREG32(CP_RB0_CNTL, tmp); 4380 4381 /* Initialize the ring buffer's read and write pointers */ 4382 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA); 4383 ring->wptr = 0; 4384 WREG32(CP_RB0_WPTR, ring->wptr); 4385 4386 /* set the wb address wether it's enabled or not */ 4387 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); 4388 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 4389 4390 /* scratch register shadowing is no longer supported */ 4391 WREG32(SCRATCH_UMSK, 0); 4392 4393 if (!rdev->wb.enabled) 4394 tmp |= RB_NO_UPDATE; 4395 4396 mdelay(1); 4397 WREG32(CP_RB0_CNTL, tmp); 4398 4399 rb_addr = ring->gpu_addr >> 8; 4400 WREG32(CP_RB0_BASE, rb_addr); 4401 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr)); 4402 4403 /* start the ring */ 4404 cik_cp_gfx_start(rdev); 4405 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; 4406 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 4407 if (r) { 4408 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 4409 return r; 4410 } 4411 4412 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 4413 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); 4414 4415 return 0; 4416 } 4417 4418 u32 cik_gfx_get_rptr(struct radeon_device *rdev, 4419 struct radeon_ring *ring) 4420 { 4421 u32 rptr; 4422 4423 if (rdev->wb.enabled) 4424 rptr = rdev->wb.wb[ring->rptr_offs/4]; 4425 else 4426 rptr = RREG32(CP_RB0_RPTR); 4427 4428 return rptr; 4429 } 4430 4431 u32 cik_gfx_get_wptr(struct radeon_device *rdev, 4432 struct radeon_ring *ring) 4433 { 4434 u32 wptr; 4435 4436 wptr = RREG32(CP_RB0_WPTR); 4437 4438 return wptr; 4439 } 4440 4441 void cik_gfx_set_wptr(struct radeon_device *rdev, 4442 struct radeon_ring *ring) 4443 { 4444 WREG32(CP_RB0_WPTR, ring->wptr); 4445 (void)RREG32(CP_RB0_WPTR); 4446 } 4447 4448 u32 cik_compute_get_rptr(struct radeon_device *rdev, 4449 struct radeon_ring *ring) 4450 { 4451 u32 rptr; 4452 4453 if (rdev->wb.enabled) { 4454 rptr = rdev->wb.wb[ring->rptr_offs/4]; 4455 } else { 4456 spin_lock(&rdev->srbm_mutex); 4457 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4458 rptr = RREG32(CP_HQD_PQ_RPTR); 4459 cik_srbm_select(rdev, 0, 0, 0, 0); 4460 spin_unlock(&rdev->srbm_mutex); 4461 } 4462 4463 return rptr; 4464 } 4465 4466 u32 cik_compute_get_wptr(struct radeon_device *rdev, 4467 struct radeon_ring *ring) 4468 { 4469 u32 wptr; 4470 4471 if (rdev->wb.enabled) { 4472 /* XXX check if swapping is necessary on BE */ 4473 wptr = rdev->wb.wb[ring->wptr_offs/4]; 4474 } else { 4475 spin_lock(&rdev->srbm_mutex); 4476 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4477 wptr = RREG32(CP_HQD_PQ_WPTR); 4478 cik_srbm_select(rdev, 0, 0, 0, 0); 4479 spin_unlock(&rdev->srbm_mutex); 4480 } 4481 4482 return wptr; 4483 } 4484 4485 void cik_compute_set_wptr(struct radeon_device *rdev, 4486 struct radeon_ring *ring) 4487 { 4488 /* XXX check if swapping is necessary on BE */ 4489 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr; 4490 WDOORBELL32(ring->doorbell_index, ring->wptr); 4491 } 4492 4493 /** 4494 * cik_cp_compute_enable - enable/disable the compute CP MEs 4495 * 4496 * @rdev: radeon_device pointer 4497 * @enable: enable or disable the MEs 4498 * 4499 * Halts or unhalts the compute MEs. 4500 */ 4501 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable) 4502 { 4503 if (enable) 4504 WREG32(CP_MEC_CNTL, 0); 4505 else { 4506 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT)); 4507 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 4508 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 4509 } 4510 udelay(50); 4511 } 4512 4513 /** 4514 * cik_cp_compute_load_microcode - load the compute CP ME ucode 4515 * 4516 * @rdev: radeon_device pointer 4517 * 4518 * Loads the compute MEC1&2 ucode. 4519 * Returns 0 for success, -EINVAL if the ucode is not available. 4520 */ 4521 static int cik_cp_compute_load_microcode(struct radeon_device *rdev) 4522 { 4523 int i; 4524 4525 if (!rdev->mec_fw) 4526 return -EINVAL; 4527 4528 cik_cp_compute_enable(rdev, false); 4529 4530 if (rdev->new_fw) { 4531 const struct gfx_firmware_header_v1_0 *mec_hdr = 4532 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; 4533 const __le32 *fw_data; 4534 u32 fw_size; 4535 4536 radeon_ucode_print_gfx_hdr(&mec_hdr->header); 4537 4538 /* MEC1 */ 4539 fw_data = (const __le32 *) 4540 ((const char *)rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 4541 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4; 4542 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4543 for (i = 0; i < fw_size; i++) 4544 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++)); 4545 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version)); 4546 4547 /* MEC2 */ 4548 if (rdev->family == CHIP_KAVERI) { 4549 const struct gfx_firmware_header_v1_0 *mec2_hdr = 4550 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; 4551 4552 fw_data = (const __le32 *) 4553 ((const char *)rdev->mec2_fw->data + 4554 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes)); 4555 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4; 4556 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4557 for (i = 0; i < fw_size; i++) 4558 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++)); 4559 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version)); 4560 } 4561 } else { 4562 const __be32 *fw_data; 4563 4564 /* MEC1 */ 4565 fw_data = (const __be32 *)rdev->mec_fw->data; 4566 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4567 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) 4568 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++)); 4569 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4570 4571 if (rdev->family == CHIP_KAVERI) { 4572 /* MEC2 */ 4573 fw_data = (const __be32 *)rdev->mec_fw->data; 4574 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4575 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) 4576 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++)); 4577 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4578 } 4579 } 4580 4581 return 0; 4582 } 4583 4584 /** 4585 * cik_cp_compute_start - start the compute queues 4586 * 4587 * @rdev: radeon_device pointer 4588 * 4589 * Enable the compute queues. 4590 * Returns 0 for success, error for failure. 4591 */ 4592 static int cik_cp_compute_start(struct radeon_device *rdev) 4593 { 4594 cik_cp_compute_enable(rdev, true); 4595 4596 return 0; 4597 } 4598 4599 /** 4600 * cik_cp_compute_fini - stop the compute queues 4601 * 4602 * @rdev: radeon_device pointer 4603 * 4604 * Stop the compute queues and tear down the driver queue 4605 * info. 4606 */ 4607 static void cik_cp_compute_fini(struct radeon_device *rdev) 4608 { 4609 int i, idx, r; 4610 4611 cik_cp_compute_enable(rdev, false); 4612 4613 for (i = 0; i < 2; i++) { 4614 if (i == 0) 4615 idx = CAYMAN_RING_TYPE_CP1_INDEX; 4616 else 4617 idx = CAYMAN_RING_TYPE_CP2_INDEX; 4618 4619 if (rdev->ring[idx].mqd_obj) { 4620 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); 4621 if (unlikely(r != 0)) 4622 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r); 4623 4624 radeon_bo_unpin(rdev->ring[idx].mqd_obj); 4625 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); 4626 4627 radeon_bo_unref(&rdev->ring[idx].mqd_obj); 4628 rdev->ring[idx].mqd_obj = NULL; 4629 } 4630 } 4631 } 4632 4633 static void cik_mec_fini(struct radeon_device *rdev) 4634 { 4635 int r; 4636 4637 if (rdev->mec.hpd_eop_obj) { 4638 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); 4639 if (unlikely(r != 0)) 4640 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r); 4641 radeon_bo_unpin(rdev->mec.hpd_eop_obj); 4642 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); 4643 4644 radeon_bo_unref(&rdev->mec.hpd_eop_obj); 4645 rdev->mec.hpd_eop_obj = NULL; 4646 } 4647 } 4648 4649 #define MEC_HPD_SIZE 2048 4650 4651 static int cik_mec_init(struct radeon_device *rdev) 4652 { 4653 int r; 4654 u32 *hpd; 4655 4656 /* 4657 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total 4658 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total 4659 */ 4660 if (rdev->family == CHIP_KAVERI) 4661 rdev->mec.num_mec = 2; 4662 else 4663 rdev->mec.num_mec = 1; 4664 rdev->mec.num_pipe = 4; 4665 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8; 4666 4667 if (rdev->mec.hpd_eop_obj == NULL) { 4668 r = radeon_bo_create(rdev, 4669 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2, 4670 PAGE_SIZE, true, 4671 RADEON_GEM_DOMAIN_GTT, 0, NULL, 4672 &rdev->mec.hpd_eop_obj); 4673 if (r) { 4674 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r); 4675 return r; 4676 } 4677 } 4678 4679 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); 4680 if (unlikely(r != 0)) { 4681 cik_mec_fini(rdev); 4682 return r; 4683 } 4684 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT, 4685 &rdev->mec.hpd_eop_gpu_addr); 4686 if (r) { 4687 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r); 4688 cik_mec_fini(rdev); 4689 return r; 4690 } 4691 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd); 4692 if (r) { 4693 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r); 4694 cik_mec_fini(rdev); 4695 return r; 4696 } 4697 4698 /* clear memory. Not sure if this is required or not */ 4699 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2); 4700 4701 radeon_bo_kunmap(rdev->mec.hpd_eop_obj); 4702 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); 4703 4704 return 0; 4705 } 4706 4707 struct hqd_registers 4708 { 4709 u32 cp_mqd_base_addr; 4710 u32 cp_mqd_base_addr_hi; 4711 u32 cp_hqd_active; 4712 u32 cp_hqd_vmid; 4713 u32 cp_hqd_persistent_state; 4714 u32 cp_hqd_pipe_priority; 4715 u32 cp_hqd_queue_priority; 4716 u32 cp_hqd_quantum; 4717 u32 cp_hqd_pq_base; 4718 u32 cp_hqd_pq_base_hi; 4719 u32 cp_hqd_pq_rptr; 4720 u32 cp_hqd_pq_rptr_report_addr; 4721 u32 cp_hqd_pq_rptr_report_addr_hi; 4722 u32 cp_hqd_pq_wptr_poll_addr; 4723 u32 cp_hqd_pq_wptr_poll_addr_hi; 4724 u32 cp_hqd_pq_doorbell_control; 4725 u32 cp_hqd_pq_wptr; 4726 u32 cp_hqd_pq_control; 4727 u32 cp_hqd_ib_base_addr; 4728 u32 cp_hqd_ib_base_addr_hi; 4729 u32 cp_hqd_ib_rptr; 4730 u32 cp_hqd_ib_control; 4731 u32 cp_hqd_iq_timer; 4732 u32 cp_hqd_iq_rptr; 4733 u32 cp_hqd_dequeue_request; 4734 u32 cp_hqd_dma_offload; 4735 u32 cp_hqd_sema_cmd; 4736 u32 cp_hqd_msg_type; 4737 u32 cp_hqd_atomic0_preop_lo; 4738 u32 cp_hqd_atomic0_preop_hi; 4739 u32 cp_hqd_atomic1_preop_lo; 4740 u32 cp_hqd_atomic1_preop_hi; 4741 u32 cp_hqd_hq_scheduler0; 4742 u32 cp_hqd_hq_scheduler1; 4743 u32 cp_mqd_control; 4744 }; 4745 4746 struct bonaire_mqd 4747 { 4748 u32 header; 4749 u32 dispatch_initiator; 4750 u32 dimensions[3]; 4751 u32 start_idx[3]; 4752 u32 num_threads[3]; 4753 u32 pipeline_stat_enable; 4754 u32 perf_counter_enable; 4755 u32 pgm[2]; 4756 u32 tba[2]; 4757 u32 tma[2]; 4758 u32 pgm_rsrc[2]; 4759 u32 vmid; 4760 u32 resource_limits; 4761 u32 static_thread_mgmt01[2]; 4762 u32 tmp_ring_size; 4763 u32 static_thread_mgmt23[2]; 4764 u32 restart[3]; 4765 u32 thread_trace_enable; 4766 u32 reserved1; 4767 u32 user_data[16]; 4768 u32 vgtcs_invoke_count[2]; 4769 struct hqd_registers queue_state; 4770 u32 dequeue_cntr; 4771 u32 interrupt_queue[64]; 4772 }; 4773 4774 /** 4775 * cik_cp_compute_resume - setup the compute queue registers 4776 * 4777 * @rdev: radeon_device pointer 4778 * 4779 * Program the compute queues and test them to make sure they 4780 * are working. 4781 * Returns 0 for success, error for failure. 4782 */ 4783 static int cik_cp_compute_resume(struct radeon_device *rdev) 4784 { 4785 int r, i, j, idx; 4786 u32 tmp; 4787 bool use_doorbell = true; 4788 u64 hqd_gpu_addr; 4789 u64 mqd_gpu_addr; 4790 u64 eop_gpu_addr; 4791 u64 wb_gpu_addr; 4792 u32 *buf; 4793 struct bonaire_mqd *mqd; 4794 4795 r = cik_cp_compute_start(rdev); 4796 if (r) 4797 return r; 4798 4799 /* fix up chicken bits */ 4800 tmp = RREG32(CP_CPF_DEBUG); 4801 tmp |= (1 << 23); 4802 WREG32(CP_CPF_DEBUG, tmp); 4803 4804 /* init the pipes */ 4805 spin_lock(&rdev->srbm_mutex); 4806 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) { 4807 int me = (i < 4) ? 1 : 2; 4808 int pipe = (i < 4) ? i : (i - 4); 4809 4810 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2); 4811 4812 cik_srbm_select(rdev, me, pipe, 0, 0); 4813 4814 /* write the EOP addr */ 4815 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8); 4816 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8); 4817 4818 /* set the VMID assigned */ 4819 WREG32(CP_HPD_EOP_VMID, 0); 4820 4821 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4822 tmp = RREG32(CP_HPD_EOP_CONTROL); 4823 tmp &= ~EOP_SIZE_MASK; 4824 tmp |= order_base_2(MEC_HPD_SIZE / 8); 4825 WREG32(CP_HPD_EOP_CONTROL, tmp); 4826 } 4827 cik_srbm_select(rdev, 0, 0, 0, 0); 4828 spin_unlock(&rdev->srbm_mutex); 4829 4830 /* init the queues. Just two for now. */ 4831 for (i = 0; i < 2; i++) { 4832 if (i == 0) 4833 idx = CAYMAN_RING_TYPE_CP1_INDEX; 4834 else 4835 idx = CAYMAN_RING_TYPE_CP2_INDEX; 4836 4837 if (rdev->ring[idx].mqd_obj == NULL) { 4838 r = radeon_bo_create(rdev, 4839 sizeof(struct bonaire_mqd), 4840 PAGE_SIZE, true, 4841 RADEON_GEM_DOMAIN_GTT, 0, NULL, 4842 &rdev->ring[idx].mqd_obj); 4843 if (r) { 4844 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r); 4845 return r; 4846 } 4847 } 4848 4849 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); 4850 if (unlikely(r != 0)) { 4851 cik_cp_compute_fini(rdev); 4852 return r; 4853 } 4854 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT, 4855 &mqd_gpu_addr); 4856 if (r) { 4857 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r); 4858 cik_cp_compute_fini(rdev); 4859 return r; 4860 } 4861 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf); 4862 if (r) { 4863 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r); 4864 cik_cp_compute_fini(rdev); 4865 return r; 4866 } 4867 4868 /* init the mqd struct */ 4869 memset(buf, 0, sizeof(struct bonaire_mqd)); 4870 4871 mqd = (struct bonaire_mqd *)buf; 4872 mqd->header = 0xC0310800; 4873 mqd->static_thread_mgmt01[0] = 0xffffffff; 4874 mqd->static_thread_mgmt01[1] = 0xffffffff; 4875 mqd->static_thread_mgmt23[0] = 0xffffffff; 4876 mqd->static_thread_mgmt23[1] = 0xffffffff; 4877 4878 spin_lock(&rdev->srbm_mutex); 4879 cik_srbm_select(rdev, rdev->ring[idx].me, 4880 rdev->ring[idx].pipe, 4881 rdev->ring[idx].queue, 0); 4882 4883 /* disable wptr polling */ 4884 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL); 4885 tmp &= ~WPTR_POLL_EN; 4886 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp); 4887 4888 /* enable doorbell? */ 4889 mqd->queue_state.cp_hqd_pq_doorbell_control = 4890 RREG32(CP_HQD_PQ_DOORBELL_CONTROL); 4891 if (use_doorbell) 4892 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; 4893 else 4894 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN; 4895 WREG32(CP_HQD_PQ_DOORBELL_CONTROL, 4896 mqd->queue_state.cp_hqd_pq_doorbell_control); 4897 4898 /* disable the queue if it's active */ 4899 mqd->queue_state.cp_hqd_dequeue_request = 0; 4900 mqd->queue_state.cp_hqd_pq_rptr = 0; 4901 mqd->queue_state.cp_hqd_pq_wptr= 0; 4902 if (RREG32(CP_HQD_ACTIVE) & 1) { 4903 WREG32(CP_HQD_DEQUEUE_REQUEST, 1); 4904 for (j = 0; j < rdev->usec_timeout; j++) { 4905 if (!(RREG32(CP_HQD_ACTIVE) & 1)) 4906 break; 4907 udelay(1); 4908 } 4909 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request); 4910 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr); 4911 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); 4912 } 4913 4914 /* set the pointer to the MQD */ 4915 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc; 4916 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr); 4917 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr); 4918 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi); 4919 /* set MQD vmid to 0 */ 4920 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL); 4921 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK; 4922 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control); 4923 4924 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4925 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8; 4926 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr; 4927 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4928 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base); 4929 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi); 4930 4931 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4932 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL); 4933 mqd->queue_state.cp_hqd_pq_control &= 4934 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK); 4935 4936 mqd->queue_state.cp_hqd_pq_control |= 4937 order_base_2(rdev->ring[idx].ring_size / 8); 4938 mqd->queue_state.cp_hqd_pq_control |= 4939 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8); 4940 #ifdef __BIG_ENDIAN 4941 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT; 4942 #endif 4943 mqd->queue_state.cp_hqd_pq_control &= 4944 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE); 4945 mqd->queue_state.cp_hqd_pq_control |= 4946 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */ 4947 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control); 4948 4949 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */ 4950 if (i == 0) 4951 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET; 4952 else 4953 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET; 4954 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc; 4955 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4956 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr); 4957 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI, 4958 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi); 4959 4960 /* set the wb address wether it's enabled or not */ 4961 if (i == 0) 4962 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET; 4963 else 4964 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET; 4965 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc; 4966 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi = 4967 upper_32_bits(wb_gpu_addr) & 0xffff; 4968 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR, 4969 mqd->queue_state.cp_hqd_pq_rptr_report_addr); 4970 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI, 4971 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi); 4972 4973 /* enable the doorbell if requested */ 4974 if (use_doorbell) { 4975 mqd->queue_state.cp_hqd_pq_doorbell_control = 4976 RREG32(CP_HQD_PQ_DOORBELL_CONTROL); 4977 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK; 4978 mqd->queue_state.cp_hqd_pq_doorbell_control |= 4979 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index); 4980 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; 4981 mqd->queue_state.cp_hqd_pq_doorbell_control &= 4982 ~(DOORBELL_SOURCE | DOORBELL_HIT); 4983 4984 } else { 4985 mqd->queue_state.cp_hqd_pq_doorbell_control = 0; 4986 } 4987 WREG32(CP_HQD_PQ_DOORBELL_CONTROL, 4988 mqd->queue_state.cp_hqd_pq_doorbell_control); 4989 4990 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4991 rdev->ring[idx].wptr = 0; 4992 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr; 4993 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); 4994 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR); 4995 4996 /* set the vmid for the queue */ 4997 mqd->queue_state.cp_hqd_vmid = 0; 4998 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid); 4999 5000 /* activate the queue */ 5001 mqd->queue_state.cp_hqd_active = 1; 5002 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active); 5003 5004 cik_srbm_select(rdev, 0, 0, 0, 0); 5005 spin_unlock(&rdev->srbm_mutex); 5006 5007 radeon_bo_kunmap(rdev->ring[idx].mqd_obj); 5008 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); 5009 5010 rdev->ring[idx].ready = true; 5011 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]); 5012 if (r) 5013 rdev->ring[idx].ready = false; 5014 } 5015 5016 return 0; 5017 } 5018 5019 static void cik_cp_enable(struct radeon_device *rdev, bool enable) 5020 { 5021 cik_cp_gfx_enable(rdev, enable); 5022 cik_cp_compute_enable(rdev, enable); 5023 } 5024 5025 static int cik_cp_load_microcode(struct radeon_device *rdev) 5026 { 5027 int r; 5028 5029 r = cik_cp_gfx_load_microcode(rdev); 5030 if (r) 5031 return r; 5032 r = cik_cp_compute_load_microcode(rdev); 5033 if (r) 5034 return r; 5035 5036 return 0; 5037 } 5038 5039 static void cik_cp_fini(struct radeon_device *rdev) 5040 { 5041 cik_cp_gfx_fini(rdev); 5042 cik_cp_compute_fini(rdev); 5043 } 5044 5045 static int cik_cp_resume(struct radeon_device *rdev) 5046 { 5047 int r; 5048 5049 cik_enable_gui_idle_interrupt(rdev, false); 5050 5051 r = cik_cp_load_microcode(rdev); 5052 if (r) 5053 return r; 5054 5055 r = cik_cp_gfx_resume(rdev); 5056 if (r) 5057 return r; 5058 r = cik_cp_compute_resume(rdev); 5059 if (r) 5060 return r; 5061 5062 cik_enable_gui_idle_interrupt(rdev, true); 5063 5064 return 0; 5065 } 5066 5067 static void cik_print_gpu_status_regs(struct radeon_device *rdev) 5068 { 5069 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", 5070 RREG32(GRBM_STATUS)); 5071 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n", 5072 RREG32(GRBM_STATUS2)); 5073 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", 5074 RREG32(GRBM_STATUS_SE0)); 5075 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", 5076 RREG32(GRBM_STATUS_SE1)); 5077 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n", 5078 RREG32(GRBM_STATUS_SE2)); 5079 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n", 5080 RREG32(GRBM_STATUS_SE3)); 5081 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", 5082 RREG32(SRBM_STATUS)); 5083 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n", 5084 RREG32(SRBM_STATUS2)); 5085 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n", 5086 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET)); 5087 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n", 5088 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET)); 5089 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT)); 5090 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n", 5091 RREG32(CP_STALLED_STAT1)); 5092 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n", 5093 RREG32(CP_STALLED_STAT2)); 5094 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n", 5095 RREG32(CP_STALLED_STAT3)); 5096 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n", 5097 RREG32(CP_CPF_BUSY_STAT)); 5098 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n", 5099 RREG32(CP_CPF_STALLED_STAT1)); 5100 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS)); 5101 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT)); 5102 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n", 5103 RREG32(CP_CPC_STALLED_STAT1)); 5104 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS)); 5105 } 5106 5107 /** 5108 * cik_gpu_check_soft_reset - check which blocks are busy 5109 * 5110 * @rdev: radeon_device pointer 5111 * 5112 * Check which blocks are busy and return the relevant reset 5113 * mask to be used by cik_gpu_soft_reset(). 5114 * Returns a mask of the blocks to be reset. 5115 */ 5116 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev) 5117 { 5118 u32 reset_mask = 0; 5119 u32 tmp; 5120 5121 /* GRBM_STATUS */ 5122 tmp = RREG32(GRBM_STATUS); 5123 if (tmp & (PA_BUSY | SC_BUSY | 5124 BCI_BUSY | SX_BUSY | 5125 TA_BUSY | VGT_BUSY | 5126 DB_BUSY | CB_BUSY | 5127 GDS_BUSY | SPI_BUSY | 5128 IA_BUSY | IA_BUSY_NO_DMA)) 5129 reset_mask |= RADEON_RESET_GFX; 5130 5131 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY)) 5132 reset_mask |= RADEON_RESET_CP; 5133 5134 /* GRBM_STATUS2 */ 5135 tmp = RREG32(GRBM_STATUS2); 5136 if (tmp & RLC_BUSY) 5137 reset_mask |= RADEON_RESET_RLC; 5138 5139 /* SDMA0_STATUS_REG */ 5140 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET); 5141 if (!(tmp & SDMA_IDLE)) 5142 reset_mask |= RADEON_RESET_DMA; 5143 5144 /* SDMA1_STATUS_REG */ 5145 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET); 5146 if (!(tmp & SDMA_IDLE)) 5147 reset_mask |= RADEON_RESET_DMA1; 5148 5149 /* SRBM_STATUS2 */ 5150 tmp = RREG32(SRBM_STATUS2); 5151 if (tmp & SDMA_BUSY) 5152 reset_mask |= RADEON_RESET_DMA; 5153 5154 if (tmp & SDMA1_BUSY) 5155 reset_mask |= RADEON_RESET_DMA1; 5156 5157 /* SRBM_STATUS */ 5158 tmp = RREG32(SRBM_STATUS); 5159 5160 if (tmp & IH_BUSY) 5161 reset_mask |= RADEON_RESET_IH; 5162 5163 if (tmp & SEM_BUSY) 5164 reset_mask |= RADEON_RESET_SEM; 5165 5166 if (tmp & GRBM_RQ_PENDING) 5167 reset_mask |= RADEON_RESET_GRBM; 5168 5169 if (tmp & VMC_BUSY) 5170 reset_mask |= RADEON_RESET_VMC; 5171 5172 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 5173 MCC_BUSY | MCD_BUSY)) 5174 reset_mask |= RADEON_RESET_MC; 5175 5176 if (evergreen_is_display_hung(rdev)) 5177 reset_mask |= RADEON_RESET_DISPLAY; 5178 5179 /* Skip MC reset as it's mostly likely not hung, just busy */ 5180 if (reset_mask & RADEON_RESET_MC) { 5181 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 5182 reset_mask &= ~RADEON_RESET_MC; 5183 } 5184 5185 return reset_mask; 5186 } 5187 5188 /** 5189 * cik_gpu_soft_reset - soft reset GPU 5190 * 5191 * @rdev: radeon_device pointer 5192 * @reset_mask: mask of which blocks to reset 5193 * 5194 * Soft reset the blocks specified in @reset_mask. 5195 */ 5196 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 5197 { 5198 struct evergreen_mc_save save; 5199 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 5200 u32 tmp; 5201 5202 if (reset_mask == 0) 5203 return; 5204 5205 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 5206 5207 cik_print_gpu_status_regs(rdev); 5208 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 5209 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); 5210 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 5211 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); 5212 5213 /* disable CG/PG */ 5214 cik_fini_pg(rdev); 5215 cik_fini_cg(rdev); 5216 5217 /* stop the rlc */ 5218 cik_rlc_stop(rdev); 5219 5220 /* Disable GFX parsing/prefetching */ 5221 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 5222 5223 /* Disable MEC parsing/prefetching */ 5224 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); 5225 5226 if (reset_mask & RADEON_RESET_DMA) { 5227 /* sdma0 */ 5228 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); 5229 tmp |= SDMA_HALT; 5230 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); 5231 } 5232 if (reset_mask & RADEON_RESET_DMA1) { 5233 /* sdma1 */ 5234 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); 5235 tmp |= SDMA_HALT; 5236 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); 5237 } 5238 5239 evergreen_mc_stop(rdev, &save); 5240 if (evergreen_mc_wait_for_idle(rdev)) { 5241 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5242 } 5243 5244 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) 5245 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX; 5246 5247 if (reset_mask & RADEON_RESET_CP) { 5248 grbm_soft_reset |= SOFT_RESET_CP; 5249 5250 srbm_soft_reset |= SOFT_RESET_GRBM; 5251 } 5252 5253 if (reset_mask & RADEON_RESET_DMA) 5254 srbm_soft_reset |= SOFT_RESET_SDMA; 5255 5256 if (reset_mask & RADEON_RESET_DMA1) 5257 srbm_soft_reset |= SOFT_RESET_SDMA1; 5258 5259 if (reset_mask & RADEON_RESET_DISPLAY) 5260 srbm_soft_reset |= SOFT_RESET_DC; 5261 5262 if (reset_mask & RADEON_RESET_RLC) 5263 grbm_soft_reset |= SOFT_RESET_RLC; 5264 5265 if (reset_mask & RADEON_RESET_SEM) 5266 srbm_soft_reset |= SOFT_RESET_SEM; 5267 5268 if (reset_mask & RADEON_RESET_IH) 5269 srbm_soft_reset |= SOFT_RESET_IH; 5270 5271 if (reset_mask & RADEON_RESET_GRBM) 5272 srbm_soft_reset |= SOFT_RESET_GRBM; 5273 5274 if (reset_mask & RADEON_RESET_VMC) 5275 srbm_soft_reset |= SOFT_RESET_VMC; 5276 5277 if (!(rdev->flags & RADEON_IS_IGP)) { 5278 if (reset_mask & RADEON_RESET_MC) 5279 srbm_soft_reset |= SOFT_RESET_MC; 5280 } 5281 5282 if (grbm_soft_reset) { 5283 tmp = RREG32(GRBM_SOFT_RESET); 5284 tmp |= grbm_soft_reset; 5285 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 5286 WREG32(GRBM_SOFT_RESET, tmp); 5287 tmp = RREG32(GRBM_SOFT_RESET); 5288 5289 udelay(50); 5290 5291 tmp &= ~grbm_soft_reset; 5292 WREG32(GRBM_SOFT_RESET, tmp); 5293 tmp = RREG32(GRBM_SOFT_RESET); 5294 } 5295 5296 if (srbm_soft_reset) { 5297 tmp = RREG32(SRBM_SOFT_RESET); 5298 tmp |= srbm_soft_reset; 5299 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 5300 WREG32(SRBM_SOFT_RESET, tmp); 5301 tmp = RREG32(SRBM_SOFT_RESET); 5302 5303 udelay(50); 5304 5305 tmp &= ~srbm_soft_reset; 5306 WREG32(SRBM_SOFT_RESET, tmp); 5307 tmp = RREG32(SRBM_SOFT_RESET); 5308 } 5309 5310 /* Wait a little for things to settle down */ 5311 udelay(50); 5312 5313 evergreen_mc_resume(rdev, &save); 5314 udelay(50); 5315 5316 cik_print_gpu_status_regs(rdev); 5317 } 5318 5319 struct kv_reset_save_regs { 5320 u32 gmcon_reng_execute; 5321 u32 gmcon_misc; 5322 u32 gmcon_misc3; 5323 }; 5324 5325 static void kv_save_regs_for_reset(struct radeon_device *rdev, 5326 struct kv_reset_save_regs *save) 5327 { 5328 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE); 5329 save->gmcon_misc = RREG32(GMCON_MISC); 5330 save->gmcon_misc3 = RREG32(GMCON_MISC3); 5331 5332 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP); 5333 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE | 5334 STCTRL_STUTTER_EN)); 5335 } 5336 5337 static void kv_restore_regs_for_reset(struct radeon_device *rdev, 5338 struct kv_reset_save_regs *save) 5339 { 5340 int i; 5341 5342 WREG32(GMCON_PGFSM_WRITE, 0); 5343 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff); 5344 5345 for (i = 0; i < 5; i++) 5346 WREG32(GMCON_PGFSM_WRITE, 0); 5347 5348 WREG32(GMCON_PGFSM_WRITE, 0); 5349 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff); 5350 5351 for (i = 0; i < 5; i++) 5352 WREG32(GMCON_PGFSM_WRITE, 0); 5353 5354 WREG32(GMCON_PGFSM_WRITE, 0x210000); 5355 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff); 5356 5357 for (i = 0; i < 5; i++) 5358 WREG32(GMCON_PGFSM_WRITE, 0); 5359 5360 WREG32(GMCON_PGFSM_WRITE, 0x21003); 5361 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff); 5362 5363 for (i = 0; i < 5; i++) 5364 WREG32(GMCON_PGFSM_WRITE, 0); 5365 5366 WREG32(GMCON_PGFSM_WRITE, 0x2b00); 5367 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff); 5368 5369 for (i = 0; i < 5; i++) 5370 WREG32(GMCON_PGFSM_WRITE, 0); 5371 5372 WREG32(GMCON_PGFSM_WRITE, 0); 5373 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff); 5374 5375 for (i = 0; i < 5; i++) 5376 WREG32(GMCON_PGFSM_WRITE, 0); 5377 5378 WREG32(GMCON_PGFSM_WRITE, 0x420000); 5379 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff); 5380 5381 for (i = 0; i < 5; i++) 5382 WREG32(GMCON_PGFSM_WRITE, 0); 5383 5384 WREG32(GMCON_PGFSM_WRITE, 0x120202); 5385 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff); 5386 5387 for (i = 0; i < 5; i++) 5388 WREG32(GMCON_PGFSM_WRITE, 0); 5389 5390 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36); 5391 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff); 5392 5393 for (i = 0; i < 5; i++) 5394 WREG32(GMCON_PGFSM_WRITE, 0); 5395 5396 WREG32(GMCON_PGFSM_WRITE, 0x373f3e); 5397 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff); 5398 5399 for (i = 0; i < 5; i++) 5400 WREG32(GMCON_PGFSM_WRITE, 0); 5401 5402 WREG32(GMCON_PGFSM_WRITE, 0x3e1332); 5403 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff); 5404 5405 WREG32(GMCON_MISC3, save->gmcon_misc3); 5406 WREG32(GMCON_MISC, save->gmcon_misc); 5407 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute); 5408 } 5409 5410 static void cik_gpu_pci_config_reset(struct radeon_device *rdev) 5411 { 5412 struct evergreen_mc_save save; 5413 struct kv_reset_save_regs kv_save = { 0 }; 5414 u32 tmp, i; 5415 5416 dev_info(rdev->dev, "GPU pci config reset\n"); 5417 5418 /* disable dpm? */ 5419 5420 /* disable cg/pg */ 5421 cik_fini_pg(rdev); 5422 cik_fini_cg(rdev); 5423 5424 /* Disable GFX parsing/prefetching */ 5425 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 5426 5427 /* Disable MEC parsing/prefetching */ 5428 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); 5429 5430 /* sdma0 */ 5431 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); 5432 tmp |= SDMA_HALT; 5433 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); 5434 /* sdma1 */ 5435 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); 5436 tmp |= SDMA_HALT; 5437 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); 5438 /* XXX other engines? */ 5439 5440 /* halt the rlc, disable cp internal ints */ 5441 cik_rlc_stop(rdev); 5442 5443 udelay(50); 5444 5445 /* disable mem access */ 5446 evergreen_mc_stop(rdev, &save); 5447 if (evergreen_mc_wait_for_idle(rdev)) { 5448 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 5449 } 5450 5451 if (rdev->flags & RADEON_IS_IGP) 5452 kv_save_regs_for_reset(rdev, &kv_save); 5453 5454 /* disable BM */ 5455 pci_disable_busmaster(rdev->pdev->dev); 5456 /* reset */ 5457 radeon_pci_config_reset(rdev); 5458 5459 udelay(100); 5460 5461 /* wait for asic to come out of reset */ 5462 for (i = 0; i < rdev->usec_timeout; i++) { 5463 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 5464 break; 5465 udelay(1); 5466 } 5467 5468 /* does asic init need to be run first??? */ 5469 if (rdev->flags & RADEON_IS_IGP) 5470 kv_restore_regs_for_reset(rdev, &kv_save); 5471 } 5472 5473 /** 5474 * cik_asic_reset - soft reset GPU 5475 * 5476 * @rdev: radeon_device pointer 5477 * 5478 * Look up which blocks are hung and attempt 5479 * to reset them. 5480 * Returns 0 for success. 5481 */ 5482 int cik_asic_reset(struct radeon_device *rdev) 5483 { 5484 u32 reset_mask; 5485 5486 reset_mask = cik_gpu_check_soft_reset(rdev); 5487 5488 if (reset_mask) 5489 r600_set_bios_scratch_engine_hung(rdev, true); 5490 5491 /* try soft reset */ 5492 cik_gpu_soft_reset(rdev, reset_mask); 5493 5494 reset_mask = cik_gpu_check_soft_reset(rdev); 5495 5496 /* try pci config reset */ 5497 if (reset_mask && radeon_hard_reset) 5498 cik_gpu_pci_config_reset(rdev); 5499 5500 reset_mask = cik_gpu_check_soft_reset(rdev); 5501 5502 if (!reset_mask) 5503 r600_set_bios_scratch_engine_hung(rdev, false); 5504 5505 return 0; 5506 } 5507 5508 /** 5509 * cik_gfx_is_lockup - check if the 3D engine is locked up 5510 * 5511 * @rdev: radeon_device pointer 5512 * @ring: radeon_ring structure holding ring information 5513 * 5514 * Check if the 3D engine is locked up (CIK). 5515 * Returns true if the engine is locked, false if not. 5516 */ 5517 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 5518 { 5519 u32 reset_mask = cik_gpu_check_soft_reset(rdev); 5520 5521 if (!(reset_mask & (RADEON_RESET_GFX | 5522 RADEON_RESET_COMPUTE | 5523 RADEON_RESET_CP))) { 5524 radeon_ring_lockup_update(rdev, ring); 5525 return false; 5526 } 5527 return radeon_ring_test_lockup(rdev, ring); 5528 } 5529 5530 /* MC */ 5531 /** 5532 * cik_mc_program - program the GPU memory controller 5533 * 5534 * @rdev: radeon_device pointer 5535 * 5536 * Set the location of vram, gart, and AGP in the GPU's 5537 * physical address space (CIK). 5538 */ 5539 static void cik_mc_program(struct radeon_device *rdev) 5540 { 5541 struct evergreen_mc_save save; 5542 u32 tmp; 5543 int i, j; 5544 5545 /* Initialize HDP */ 5546 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 5547 WREG32((0x2c14 + j), 0x00000000); 5548 WREG32((0x2c18 + j), 0x00000000); 5549 WREG32((0x2c1c + j), 0x00000000); 5550 WREG32((0x2c20 + j), 0x00000000); 5551 WREG32((0x2c24 + j), 0x00000000); 5552 } 5553 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 5554 5555 evergreen_mc_stop(rdev, &save); 5556 if (radeon_mc_wait_for_idle(rdev)) { 5557 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5558 } 5559 /* Lockout access through VGA aperture*/ 5560 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 5561 /* Update configuration */ 5562 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 5563 rdev->mc.vram_start >> 12); 5564 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 5565 rdev->mc.vram_end >> 12); 5566 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 5567 rdev->vram_scratch.gpu_addr >> 12); 5568 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 5569 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 5570 WREG32(MC_VM_FB_LOCATION, tmp); 5571 /* XXX double check these! */ 5572 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 5573 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 5574 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 5575 WREG32(MC_VM_AGP_BASE, 0); 5576 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 5577 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 5578 if (radeon_mc_wait_for_idle(rdev)) { 5579 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5580 } 5581 evergreen_mc_resume(rdev, &save); 5582 /* we need to own VRAM, so turn off the VGA renderer here 5583 * to stop it overwriting our objects */ 5584 rv515_vga_render_disable(rdev); 5585 } 5586 5587 /** 5588 * cik_mc_init - initialize the memory controller driver params 5589 * 5590 * @rdev: radeon_device pointer 5591 * 5592 * Look up the amount of vram, vram width, and decide how to place 5593 * vram and gart within the GPU's physical address space (CIK). 5594 * Returns 0 for success. 5595 */ 5596 static int cik_mc_init(struct radeon_device *rdev) 5597 { 5598 u32 tmp; 5599 int chansize, numchan; 5600 5601 /* Get VRAM informations */ 5602 rdev->mc.vram_is_ddr = true; 5603 tmp = RREG32(MC_ARB_RAMCFG); 5604 if (tmp & CHANSIZE_MASK) { 5605 chansize = 64; 5606 } else { 5607 chansize = 32; 5608 } 5609 tmp = RREG32(MC_SHARED_CHMAP); 5610 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 5611 case 0: 5612 default: 5613 numchan = 1; 5614 break; 5615 case 1: 5616 numchan = 2; 5617 break; 5618 case 2: 5619 numchan = 4; 5620 break; 5621 case 3: 5622 numchan = 8; 5623 break; 5624 case 4: 5625 numchan = 3; 5626 break; 5627 case 5: 5628 numchan = 6; 5629 break; 5630 case 6: 5631 numchan = 10; 5632 break; 5633 case 7: 5634 numchan = 12; 5635 break; 5636 case 8: 5637 numchan = 16; 5638 break; 5639 } 5640 rdev->mc.vram_width = numchan * chansize; 5641 /* Could aper size report 0 ? */ 5642 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 5643 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 5644 /* size in MB on si */ 5645 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 5646 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 5647 rdev->mc.visible_vram_size = rdev->mc.aper_size; 5648 si_vram_gtt_location(rdev, &rdev->mc); 5649 radeon_update_bandwidth_info(rdev); 5650 5651 return 0; 5652 } 5653 5654 /* 5655 * GART 5656 * VMID 0 is the physical GPU addresses as used by the kernel. 5657 * VMIDs 1-15 are used for userspace clients and are handled 5658 * by the radeon vm/hsa code. 5659 */ 5660 /** 5661 * cik_pcie_gart_tlb_flush - gart tlb flush callback 5662 * 5663 * @rdev: radeon_device pointer 5664 * 5665 * Flush the TLB for the VMID 0 page table (CIK). 5666 */ 5667 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev) 5668 { 5669 /* flush hdp cache */ 5670 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0); 5671 5672 /* bits 0-15 are the VM contexts0-15 */ 5673 WREG32(VM_INVALIDATE_REQUEST, 0x1); 5674 } 5675 5676 /** 5677 * cik_pcie_gart_enable - gart enable 5678 * 5679 * @rdev: radeon_device pointer 5680 * 5681 * This sets up the TLBs, programs the page tables for VMID0, 5682 * sets up the hw for VMIDs 1-15 which are allocated on 5683 * demand, and sets up the global locations for the LDS, GDS, 5684 * and GPUVM for FSA64 clients (CIK). 5685 * Returns 0 for success, errors for failure. 5686 */ 5687 static int cik_pcie_gart_enable(struct radeon_device *rdev) 5688 { 5689 int r, i; 5690 5691 if (rdev->gart.robj == NULL) { 5692 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 5693 return -EINVAL; 5694 } 5695 r = radeon_gart_table_vram_pin(rdev); 5696 if (r) 5697 return r; 5698 /* Setup TLB control */ 5699 WREG32(MC_VM_MX_L1_TLB_CNTL, 5700 (0xA << 7) | 5701 ENABLE_L1_TLB | 5702 ENABLE_L1_FRAGMENT_PROCESSING | 5703 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 5704 ENABLE_ADVANCED_DRIVER_MODEL | 5705 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 5706 /* Setup L2 cache */ 5707 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | 5708 ENABLE_L2_FRAGMENT_PROCESSING | 5709 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 5710 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 5711 EFFECTIVE_L2_QUEUE_SIZE(7) | 5712 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 5713 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); 5714 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 5715 BANK_SELECT(4) | 5716 L2_CACHE_BIGK_FRAGMENT_SIZE(4)); 5717 /* setup context0 */ 5718 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 5719 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 5720 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 5721 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 5722 (u32)(rdev->dummy_page.addr >> 12)); 5723 WREG32(VM_CONTEXT0_CNTL2, 0); 5724 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 5725 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT)); 5726 5727 WREG32(0x15D4, 0); 5728 WREG32(0x15D8, 0); 5729 WREG32(0x15DC, 0); 5730 5731 /* restore context1-15 */ 5732 /* set vm size, must be a multiple of 4 */ 5733 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); 5734 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn); 5735 for (i = 1; i < 16; i++) { 5736 if (i < 8) 5737 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), 5738 rdev->vm_manager.saved_table_addr[i]); 5739 else 5740 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2), 5741 rdev->vm_manager.saved_table_addr[i]); 5742 } 5743 5744 /* enable context1-15 */ 5745 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, 5746 (u32)(rdev->dummy_page.addr >> 12)); 5747 WREG32(VM_CONTEXT1_CNTL2, 4); 5748 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | 5749 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) | 5750 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5751 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | 5752 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5753 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT | 5754 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT | 5755 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT | 5756 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT | 5757 VALID_PROTECTION_FAULT_ENABLE_DEFAULT | 5758 READ_PROTECTION_FAULT_ENABLE_INTERRUPT | 5759 READ_PROTECTION_FAULT_ENABLE_DEFAULT | 5760 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5761 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT); 5762 5763 if (rdev->family == CHIP_KAVERI) { 5764 u32 tmp = RREG32(CHUB_CONTROL); 5765 tmp &= ~BYPASS_VM; 5766 WREG32(CHUB_CONTROL, tmp); 5767 } 5768 5769 /* XXX SH_MEM regs */ 5770 /* where to put LDS, scratch, GPUVM in FSA64 space */ 5771 spin_lock(&rdev->srbm_mutex); 5772 for (i = 0; i < 16; i++) { 5773 cik_srbm_select(rdev, 0, 0, 0, i); 5774 /* CP and shaders */ 5775 WREG32(SH_MEM_CONFIG, 0); 5776 WREG32(SH_MEM_APE1_BASE, 1); 5777 WREG32(SH_MEM_APE1_LIMIT, 0); 5778 WREG32(SH_MEM_BASES, 0); 5779 /* SDMA GFX */ 5780 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0); 5781 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0); 5782 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0); 5783 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0); 5784 /* XXX SDMA RLC - todo */ 5785 } 5786 cik_srbm_select(rdev, 0, 0, 0, 0); 5787 spin_unlock(&rdev->srbm_mutex); 5788 5789 cik_pcie_gart_tlb_flush(rdev); 5790 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 5791 (unsigned)(rdev->mc.gtt_size >> 20), 5792 (unsigned long long)rdev->gart.table_addr); 5793 rdev->gart.ready = true; 5794 return 0; 5795 } 5796 5797 /** 5798 * cik_pcie_gart_disable - gart disable 5799 * 5800 * @rdev: radeon_device pointer 5801 * 5802 * This disables all VM page table (CIK). 5803 */ 5804 static void cik_pcie_gart_disable(struct radeon_device *rdev) 5805 { 5806 unsigned i; 5807 5808 for (i = 1; i < 16; ++i) { 5809 uint32_t reg; 5810 if (i < 8) 5811 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2); 5812 else 5813 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2); 5814 rdev->vm_manager.saved_table_addr[i] = RREG32(reg); 5815 } 5816 5817 /* Disable all tables */ 5818 WREG32(VM_CONTEXT0_CNTL, 0); 5819 WREG32(VM_CONTEXT1_CNTL, 0); 5820 /* Setup TLB control */ 5821 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS | 5822 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 5823 /* Setup L2 cache */ 5824 WREG32(VM_L2_CNTL, 5825 ENABLE_L2_FRAGMENT_PROCESSING | 5826 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 5827 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 5828 EFFECTIVE_L2_QUEUE_SIZE(7) | 5829 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 5830 WREG32(VM_L2_CNTL2, 0); 5831 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 5832 L2_CACHE_BIGK_FRAGMENT_SIZE(6)); 5833 radeon_gart_table_vram_unpin(rdev); 5834 } 5835 5836 /** 5837 * cik_pcie_gart_fini - vm fini callback 5838 * 5839 * @rdev: radeon_device pointer 5840 * 5841 * Tears down the driver GART/VM setup (CIK). 5842 */ 5843 static void cik_pcie_gart_fini(struct radeon_device *rdev) 5844 { 5845 cik_pcie_gart_disable(rdev); 5846 radeon_gart_table_vram_free(rdev); 5847 radeon_gart_fini(rdev); 5848 } 5849 5850 /* vm parser */ 5851 /** 5852 * cik_ib_parse - vm ib_parse callback 5853 * 5854 * @rdev: radeon_device pointer 5855 * @ib: indirect buffer pointer 5856 * 5857 * CIK uses hw IB checking so this is a nop (CIK). 5858 */ 5859 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) 5860 { 5861 return 0; 5862 } 5863 5864 /* 5865 * vm 5866 * VMID 0 is the physical GPU addresses as used by the kernel. 5867 * VMIDs 1-15 are used for userspace clients and are handled 5868 * by the radeon vm/hsa code. 5869 */ 5870 /** 5871 * cik_vm_init - cik vm init callback 5872 * 5873 * @rdev: radeon_device pointer 5874 * 5875 * Inits cik specific vm parameters (number of VMs, base of vram for 5876 * VMIDs 1-15) (CIK). 5877 * Returns 0 for success. 5878 */ 5879 int cik_vm_init(struct radeon_device *rdev) 5880 { 5881 /* number of VMs */ 5882 rdev->vm_manager.nvm = 16; 5883 /* base offset of vram pages */ 5884 if (rdev->flags & RADEON_IS_IGP) { 5885 u64 tmp = RREG32(MC_VM_FB_OFFSET); 5886 tmp <<= 22; 5887 rdev->vm_manager.vram_base_offset = tmp; 5888 } else 5889 rdev->vm_manager.vram_base_offset = 0; 5890 5891 return 0; 5892 } 5893 5894 /** 5895 * cik_vm_fini - cik vm fini callback 5896 * 5897 * @rdev: radeon_device pointer 5898 * 5899 * Tear down any asic specific VM setup (CIK). 5900 */ 5901 void cik_vm_fini(struct radeon_device *rdev) 5902 { 5903 } 5904 5905 /** 5906 * cik_vm_decode_fault - print human readable fault info 5907 * 5908 * @rdev: radeon_device pointer 5909 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value 5910 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value 5911 * 5912 * Print human readable fault information (CIK). 5913 */ 5914 static void cik_vm_decode_fault(struct radeon_device *rdev, 5915 u32 status, u32 addr, u32 mc_client) 5916 { 5917 u32 mc_id; 5918 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; 5919 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; 5920 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff, 5921 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 }; 5922 5923 if (rdev->family == CHIP_HAWAII) 5924 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 5925 else 5926 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 5927 5928 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n", 5929 protections, vmid, addr, 5930 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", 5931 block, mc_client, mc_id); 5932 } 5933 5934 /** 5935 * cik_vm_flush - cik vm flush using the CP 5936 * 5937 * @rdev: radeon_device pointer 5938 * 5939 * Update the page table base and flush the VM TLB 5940 * using the CP (CIK). 5941 */ 5942 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) 5943 { 5944 struct radeon_ring *ring = &rdev->ring[ridx]; 5945 int usepfp = (ridx == RADEON_RING_TYPE_GFX_INDEX); 5946 5947 if (vm == NULL) 5948 return; 5949 5950 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5951 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5952 WRITE_DATA_DST_SEL(0))); 5953 if (vm->id < 8) { 5954 radeon_ring_write(ring, 5955 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2); 5956 } else { 5957 radeon_ring_write(ring, 5958 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2); 5959 } 5960 radeon_ring_write(ring, 0); 5961 radeon_ring_write(ring, vm->pd_gpu_addr >> 12); 5962 5963 /* update SH_MEM_* regs */ 5964 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5965 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5966 WRITE_DATA_DST_SEL(0))); 5967 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); 5968 radeon_ring_write(ring, 0); 5969 radeon_ring_write(ring, VMID(vm->id)); 5970 5971 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6)); 5972 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5973 WRITE_DATA_DST_SEL(0))); 5974 radeon_ring_write(ring, SH_MEM_BASES >> 2); 5975 radeon_ring_write(ring, 0); 5976 5977 radeon_ring_write(ring, 0); /* SH_MEM_BASES */ 5978 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */ 5979 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */ 5980 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */ 5981 5982 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5983 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5984 WRITE_DATA_DST_SEL(0))); 5985 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); 5986 radeon_ring_write(ring, 0); 5987 radeon_ring_write(ring, VMID(0)); 5988 5989 /* HDP flush */ 5990 cik_hdp_flush_cp_ring_emit(rdev, ridx); 5991 5992 /* bits 0-15 are the VM contexts0-15 */ 5993 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5994 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5995 WRITE_DATA_DST_SEL(0))); 5996 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 5997 radeon_ring_write(ring, 0); 5998 radeon_ring_write(ring, 1 << vm->id); 5999 6000 /* compute doesn't have PFP */ 6001 if (usepfp) { 6002 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 6003 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 6004 radeon_ring_write(ring, 0x0); 6005 } 6006 } 6007 6008 /* 6009 * RLC 6010 * The RLC is a multi-purpose microengine that handles a 6011 * variety of functions, the most important of which is 6012 * the interrupt controller. 6013 */ 6014 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, 6015 bool enable) 6016 { 6017 u32 tmp = RREG32(CP_INT_CNTL_RING0); 6018 6019 if (enable) 6020 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 6021 else 6022 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 6023 WREG32(CP_INT_CNTL_RING0, tmp); 6024 } 6025 6026 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable) 6027 { 6028 u32 tmp; 6029 6030 tmp = RREG32(RLC_LB_CNTL); 6031 if (enable) 6032 tmp |= LOAD_BALANCE_ENABLE; 6033 else 6034 tmp &= ~LOAD_BALANCE_ENABLE; 6035 WREG32(RLC_LB_CNTL, tmp); 6036 } 6037 6038 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) 6039 { 6040 u32 i, j, k; 6041 u32 mask; 6042 6043 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 6044 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 6045 cik_select_se_sh(rdev, i, j); 6046 for (k = 0; k < rdev->usec_timeout; k++) { 6047 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0) 6048 break; 6049 udelay(1); 6050 } 6051 } 6052 } 6053 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6054 6055 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY; 6056 for (k = 0; k < rdev->usec_timeout; k++) { 6057 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 6058 break; 6059 udelay(1); 6060 } 6061 } 6062 6063 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc) 6064 { 6065 u32 tmp; 6066 6067 tmp = RREG32(RLC_CNTL); 6068 if (tmp != rlc) 6069 WREG32(RLC_CNTL, rlc); 6070 } 6071 6072 static u32 cik_halt_rlc(struct radeon_device *rdev) 6073 { 6074 u32 data, orig; 6075 6076 orig = data = RREG32(RLC_CNTL); 6077 6078 if (data & RLC_ENABLE) { 6079 u32 i; 6080 6081 data &= ~RLC_ENABLE; 6082 WREG32(RLC_CNTL, data); 6083 6084 for (i = 0; i < rdev->usec_timeout; i++) { 6085 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0) 6086 break; 6087 udelay(1); 6088 } 6089 6090 cik_wait_for_rlc_serdes(rdev); 6091 } 6092 6093 return orig; 6094 } 6095 6096 void cik_enter_rlc_safe_mode(struct radeon_device *rdev) 6097 { 6098 u32 tmp, i, mask; 6099 6100 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE); 6101 WREG32(RLC_GPR_REG2, tmp); 6102 6103 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS; 6104 for (i = 0; i < rdev->usec_timeout; i++) { 6105 if ((RREG32(RLC_GPM_STAT) & mask) == mask) 6106 break; 6107 udelay(1); 6108 } 6109 6110 for (i = 0; i < rdev->usec_timeout; i++) { 6111 if ((RREG32(RLC_GPR_REG2) & REQ) == 0) 6112 break; 6113 udelay(1); 6114 } 6115 } 6116 6117 void cik_exit_rlc_safe_mode(struct radeon_device *rdev) 6118 { 6119 u32 tmp; 6120 6121 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE); 6122 WREG32(RLC_GPR_REG2, tmp); 6123 } 6124 6125 /** 6126 * cik_rlc_stop - stop the RLC ME 6127 * 6128 * @rdev: radeon_device pointer 6129 * 6130 * Halt the RLC ME (MicroEngine) (CIK). 6131 */ 6132 static void cik_rlc_stop(struct radeon_device *rdev) 6133 { 6134 WREG32(RLC_CNTL, 0); 6135 6136 cik_enable_gui_idle_interrupt(rdev, false); 6137 6138 cik_wait_for_rlc_serdes(rdev); 6139 } 6140 6141 /** 6142 * cik_rlc_start - start the RLC ME 6143 * 6144 * @rdev: radeon_device pointer 6145 * 6146 * Unhalt the RLC ME (MicroEngine) (CIK). 6147 */ 6148 static void cik_rlc_start(struct radeon_device *rdev) 6149 { 6150 WREG32(RLC_CNTL, RLC_ENABLE); 6151 6152 cik_enable_gui_idle_interrupt(rdev, true); 6153 6154 udelay(50); 6155 } 6156 6157 /** 6158 * cik_rlc_resume - setup the RLC hw 6159 * 6160 * @rdev: radeon_device pointer 6161 * 6162 * Initialize the RLC registers, load the ucode, 6163 * and start the RLC (CIK). 6164 * Returns 0 for success, -EINVAL if the ucode is not available. 6165 */ 6166 static int cik_rlc_resume(struct radeon_device *rdev) 6167 { 6168 u32 i, size, tmp; 6169 6170 if (!rdev->rlc_fw) 6171 return -EINVAL; 6172 6173 cik_rlc_stop(rdev); 6174 6175 /* disable CG */ 6176 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc; 6177 WREG32(RLC_CGCG_CGLS_CTRL, tmp); 6178 6179 si_rlc_reset(rdev); 6180 6181 cik_init_pg(rdev); 6182 6183 cik_init_cg(rdev); 6184 6185 WREG32(RLC_LB_CNTR_INIT, 0); 6186 WREG32(RLC_LB_CNTR_MAX, 0x00008000); 6187 6188 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6189 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); 6190 WREG32(RLC_LB_PARAMS, 0x00600408); 6191 WREG32(RLC_LB_CNTL, 0x80000004); 6192 6193 WREG32(RLC_MC_CNTL, 0); 6194 WREG32(RLC_UCODE_CNTL, 0); 6195 6196 if (rdev->new_fw) { 6197 const struct rlc_firmware_header_v1_0 *hdr = 6198 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data; 6199 const __le32 *fw_data = (const __le32 *) 6200 ((const char *)rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6201 6202 radeon_ucode_print_rlc_hdr(&hdr->header); 6203 6204 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 6205 WREG32(RLC_GPM_UCODE_ADDR, 0); 6206 for (i = 0; i < size; i++) 6207 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 6208 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version)); 6209 } else { 6210 const __be32 *fw_data; 6211 6212 switch (rdev->family) { 6213 case CHIP_BONAIRE: 6214 case CHIP_HAWAII: 6215 default: 6216 size = BONAIRE_RLC_UCODE_SIZE; 6217 break; 6218 case CHIP_KAVERI: 6219 size = KV_RLC_UCODE_SIZE; 6220 break; 6221 case CHIP_KABINI: 6222 size = KB_RLC_UCODE_SIZE; 6223 break; 6224 case CHIP_MULLINS: 6225 size = ML_RLC_UCODE_SIZE; 6226 break; 6227 } 6228 6229 fw_data = (const __be32 *)rdev->rlc_fw->data; 6230 WREG32(RLC_GPM_UCODE_ADDR, 0); 6231 for (i = 0; i < size; i++) 6232 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++)); 6233 WREG32(RLC_GPM_UCODE_ADDR, 0); 6234 } 6235 6236 /* XXX - find out what chips support lbpw */ 6237 cik_enable_lbpw(rdev, false); 6238 6239 if (rdev->family == CHIP_BONAIRE) 6240 WREG32(RLC_DRIVER_DMA_STATUS, 0); 6241 6242 cik_rlc_start(rdev); 6243 6244 return 0; 6245 } 6246 6247 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) 6248 { 6249 u32 data, orig, tmp, tmp2; 6250 6251 orig = data = RREG32(RLC_CGCG_CGLS_CTRL); 6252 6253 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { 6254 cik_enable_gui_idle_interrupt(rdev, true); 6255 6256 tmp = cik_halt_rlc(rdev); 6257 6258 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6259 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6260 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6261 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE; 6262 WREG32(RLC_SERDES_WR_CTRL, tmp2); 6263 6264 cik_update_rlc(rdev, tmp); 6265 6266 data |= CGCG_EN | CGLS_EN; 6267 } else { 6268 cik_enable_gui_idle_interrupt(rdev, false); 6269 6270 RREG32(CB_CGTT_SCLK_CTRL); 6271 RREG32(CB_CGTT_SCLK_CTRL); 6272 RREG32(CB_CGTT_SCLK_CTRL); 6273 RREG32(CB_CGTT_SCLK_CTRL); 6274 6275 data &= ~(CGCG_EN | CGLS_EN); 6276 } 6277 6278 if (orig != data) 6279 WREG32(RLC_CGCG_CGLS_CTRL, data); 6280 6281 } 6282 6283 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) 6284 { 6285 u32 data, orig, tmp = 0; 6286 6287 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { 6288 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) { 6289 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { 6290 orig = data = RREG32(CP_MEM_SLP_CNTL); 6291 data |= CP_MEM_LS_EN; 6292 if (orig != data) 6293 WREG32(CP_MEM_SLP_CNTL, data); 6294 } 6295 } 6296 6297 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 6298 data &= 0xfffffffd; 6299 if (orig != data) 6300 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 6301 6302 tmp = cik_halt_rlc(rdev); 6303 6304 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6305 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6306 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6307 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0; 6308 WREG32(RLC_SERDES_WR_CTRL, data); 6309 6310 cik_update_rlc(rdev, tmp); 6311 6312 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) { 6313 orig = data = RREG32(CGTS_SM_CTRL_REG); 6314 data &= ~SM_MODE_MASK; 6315 data |= SM_MODE(0x2); 6316 data |= SM_MODE_ENABLE; 6317 data &= ~CGTS_OVERRIDE; 6318 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) && 6319 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS)) 6320 data &= ~CGTS_LS_OVERRIDE; 6321 data &= ~ON_MONITOR_ADD_MASK; 6322 data |= ON_MONITOR_ADD_EN; 6323 data |= ON_MONITOR_ADD(0x96); 6324 if (orig != data) 6325 WREG32(CGTS_SM_CTRL_REG, data); 6326 } 6327 } else { 6328 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 6329 data |= 0x00000002; 6330 if (orig != data) 6331 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 6332 6333 data = RREG32(RLC_MEM_SLP_CNTL); 6334 if (data & RLC_MEM_LS_EN) { 6335 data &= ~RLC_MEM_LS_EN; 6336 WREG32(RLC_MEM_SLP_CNTL, data); 6337 } 6338 6339 data = RREG32(CP_MEM_SLP_CNTL); 6340 if (data & CP_MEM_LS_EN) { 6341 data &= ~CP_MEM_LS_EN; 6342 WREG32(CP_MEM_SLP_CNTL, data); 6343 } 6344 6345 orig = data = RREG32(CGTS_SM_CTRL_REG); 6346 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE; 6347 if (orig != data) 6348 WREG32(CGTS_SM_CTRL_REG, data); 6349 6350 tmp = cik_halt_rlc(rdev); 6351 6352 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6353 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6354 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6355 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1; 6356 WREG32(RLC_SERDES_WR_CTRL, data); 6357 6358 cik_update_rlc(rdev, tmp); 6359 } 6360 } 6361 6362 static const u32 mc_cg_registers[] = 6363 { 6364 MC_HUB_MISC_HUB_CG, 6365 MC_HUB_MISC_SIP_CG, 6366 MC_HUB_MISC_VM_CG, 6367 MC_XPB_CLK_GAT, 6368 ATC_MISC_CG, 6369 MC_CITF_MISC_WR_CG, 6370 MC_CITF_MISC_RD_CG, 6371 MC_CITF_MISC_VM_CG, 6372 VM_L2_CG, 6373 }; 6374 6375 static void cik_enable_mc_ls(struct radeon_device *rdev, 6376 bool enable) 6377 { 6378 int i; 6379 u32 orig, data; 6380 6381 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 6382 orig = data = RREG32(mc_cg_registers[i]); 6383 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) 6384 data |= MC_LS_ENABLE; 6385 else 6386 data &= ~MC_LS_ENABLE; 6387 if (data != orig) 6388 WREG32(mc_cg_registers[i], data); 6389 } 6390 } 6391 6392 static void cik_enable_mc_mgcg(struct radeon_device *rdev, 6393 bool enable) 6394 { 6395 int i; 6396 u32 orig, data; 6397 6398 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 6399 orig = data = RREG32(mc_cg_registers[i]); 6400 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) 6401 data |= MC_CG_ENABLE; 6402 else 6403 data &= ~MC_CG_ENABLE; 6404 if (data != orig) 6405 WREG32(mc_cg_registers[i], data); 6406 } 6407 } 6408 6409 static void cik_enable_sdma_mgcg(struct radeon_device *rdev, 6410 bool enable) 6411 { 6412 u32 orig, data; 6413 6414 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { 6415 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100); 6416 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100); 6417 } else { 6418 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET); 6419 data |= 0xff000000; 6420 if (data != orig) 6421 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data); 6422 6423 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET); 6424 data |= 0xff000000; 6425 if (data != orig) 6426 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data); 6427 } 6428 } 6429 6430 static void cik_enable_sdma_mgls(struct radeon_device *rdev, 6431 bool enable) 6432 { 6433 u32 orig, data; 6434 6435 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) { 6436 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); 6437 data |= 0x100; 6438 if (orig != data) 6439 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); 6440 6441 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); 6442 data |= 0x100; 6443 if (orig != data) 6444 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); 6445 } else { 6446 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); 6447 data &= ~0x100; 6448 if (orig != data) 6449 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); 6450 6451 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); 6452 data &= ~0x100; 6453 if (orig != data) 6454 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); 6455 } 6456 } 6457 6458 static void cik_enable_uvd_mgcg(struct radeon_device *rdev, 6459 bool enable) 6460 { 6461 u32 orig, data; 6462 6463 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { 6464 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 6465 data = 0xfff; 6466 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); 6467 6468 orig = data = RREG32(UVD_CGC_CTRL); 6469 data |= DCM; 6470 if (orig != data) 6471 WREG32(UVD_CGC_CTRL, data); 6472 } else { 6473 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 6474 data &= ~0xfff; 6475 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); 6476 6477 orig = data = RREG32(UVD_CGC_CTRL); 6478 data &= ~DCM; 6479 if (orig != data) 6480 WREG32(UVD_CGC_CTRL, data); 6481 } 6482 } 6483 6484 static void cik_enable_bif_mgls(struct radeon_device *rdev, 6485 bool enable) 6486 { 6487 u32 orig, data; 6488 6489 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); 6490 6491 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) 6492 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | 6493 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN; 6494 else 6495 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN | 6496 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN); 6497 6498 if (orig != data) 6499 WREG32_PCIE_PORT(PCIE_CNTL2, data); 6500 } 6501 6502 static void cik_enable_hdp_mgcg(struct radeon_device *rdev, 6503 bool enable) 6504 { 6505 u32 orig, data; 6506 6507 orig = data = RREG32(HDP_HOST_PATH_CNTL); 6508 6509 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) 6510 data &= ~CLOCK_GATING_DIS; 6511 else 6512 data |= CLOCK_GATING_DIS; 6513 6514 if (orig != data) 6515 WREG32(HDP_HOST_PATH_CNTL, data); 6516 } 6517 6518 static void cik_enable_hdp_ls(struct radeon_device *rdev, 6519 bool enable) 6520 { 6521 u32 orig, data; 6522 6523 orig = data = RREG32(HDP_MEM_POWER_LS); 6524 6525 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) 6526 data |= HDP_LS_ENABLE; 6527 else 6528 data &= ~HDP_LS_ENABLE; 6529 6530 if (orig != data) 6531 WREG32(HDP_MEM_POWER_LS, data); 6532 } 6533 6534 void cik_update_cg(struct radeon_device *rdev, 6535 u32 block, bool enable) 6536 { 6537 6538 if (block & RADEON_CG_BLOCK_GFX) { 6539 cik_enable_gui_idle_interrupt(rdev, false); 6540 /* order matters! */ 6541 if (enable) { 6542 cik_enable_mgcg(rdev, true); 6543 cik_enable_cgcg(rdev, true); 6544 } else { 6545 cik_enable_cgcg(rdev, false); 6546 cik_enable_mgcg(rdev, false); 6547 } 6548 cik_enable_gui_idle_interrupt(rdev, true); 6549 } 6550 6551 if (block & RADEON_CG_BLOCK_MC) { 6552 if (!(rdev->flags & RADEON_IS_IGP)) { 6553 cik_enable_mc_mgcg(rdev, enable); 6554 cik_enable_mc_ls(rdev, enable); 6555 } 6556 } 6557 6558 if (block & RADEON_CG_BLOCK_SDMA) { 6559 cik_enable_sdma_mgcg(rdev, enable); 6560 cik_enable_sdma_mgls(rdev, enable); 6561 } 6562 6563 if (block & RADEON_CG_BLOCK_BIF) { 6564 cik_enable_bif_mgls(rdev, enable); 6565 } 6566 6567 if (block & RADEON_CG_BLOCK_UVD) { 6568 if (rdev->has_uvd) 6569 cik_enable_uvd_mgcg(rdev, enable); 6570 } 6571 6572 if (block & RADEON_CG_BLOCK_HDP) { 6573 cik_enable_hdp_mgcg(rdev, enable); 6574 cik_enable_hdp_ls(rdev, enable); 6575 } 6576 6577 if (block & RADEON_CG_BLOCK_VCE) { 6578 vce_v2_0_enable_mgcg(rdev, enable); 6579 } 6580 } 6581 6582 static void cik_init_cg(struct radeon_device *rdev) 6583 { 6584 6585 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true); 6586 6587 if (rdev->has_uvd) 6588 si_init_uvd_internal_cg(rdev); 6589 6590 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | 6591 RADEON_CG_BLOCK_SDMA | 6592 RADEON_CG_BLOCK_BIF | 6593 RADEON_CG_BLOCK_UVD | 6594 RADEON_CG_BLOCK_HDP), true); 6595 } 6596 6597 static void cik_fini_cg(struct radeon_device *rdev) 6598 { 6599 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | 6600 RADEON_CG_BLOCK_SDMA | 6601 RADEON_CG_BLOCK_BIF | 6602 RADEON_CG_BLOCK_UVD | 6603 RADEON_CG_BLOCK_HDP), false); 6604 6605 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false); 6606 } 6607 6608 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev, 6609 bool enable) 6610 { 6611 u32 data, orig; 6612 6613 orig = data = RREG32(RLC_PG_CNTL); 6614 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) 6615 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE; 6616 else 6617 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE; 6618 if (orig != data) 6619 WREG32(RLC_PG_CNTL, data); 6620 } 6621 6622 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev, 6623 bool enable) 6624 { 6625 u32 data, orig; 6626 6627 orig = data = RREG32(RLC_PG_CNTL); 6628 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) 6629 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE; 6630 else 6631 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE; 6632 if (orig != data) 6633 WREG32(RLC_PG_CNTL, data); 6634 } 6635 6636 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable) 6637 { 6638 u32 data, orig; 6639 6640 orig = data = RREG32(RLC_PG_CNTL); 6641 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP)) 6642 data &= ~DISABLE_CP_PG; 6643 else 6644 data |= DISABLE_CP_PG; 6645 if (orig != data) 6646 WREG32(RLC_PG_CNTL, data); 6647 } 6648 6649 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable) 6650 { 6651 u32 data, orig; 6652 6653 orig = data = RREG32(RLC_PG_CNTL); 6654 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS)) 6655 data &= ~DISABLE_GDS_PG; 6656 else 6657 data |= DISABLE_GDS_PG; 6658 if (orig != data) 6659 WREG32(RLC_PG_CNTL, data); 6660 } 6661 6662 #define CP_ME_TABLE_SIZE 96 6663 #define CP_ME_TABLE_OFFSET 2048 6664 #define CP_MEC_TABLE_OFFSET 4096 6665 6666 void cik_init_cp_pg_table(struct radeon_device *rdev) 6667 { 6668 volatile u32 *dst_ptr; 6669 int me, i, max_me = 4; 6670 u32 bo_offset = 0; 6671 u32 table_offset, table_size; 6672 6673 if (rdev->family == CHIP_KAVERI) 6674 max_me = 5; 6675 6676 if (rdev->rlc.cp_table_ptr == NULL) 6677 return; 6678 6679 /* write the cp table buffer */ 6680 dst_ptr = rdev->rlc.cp_table_ptr; 6681 for (me = 0; me < max_me; me++) { 6682 if (rdev->new_fw) { 6683 const __le32 *fw_data; 6684 const struct gfx_firmware_header_v1_0 *hdr; 6685 6686 if (me == 0) { 6687 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; 6688 fw_data = (const __le32 *) 6689 ((const char *)rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6690 table_offset = le32_to_cpu(hdr->jt_offset); 6691 table_size = le32_to_cpu(hdr->jt_size); 6692 } else if (me == 1) { 6693 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; 6694 fw_data = (const __le32 *) 6695 ((const char *)rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6696 table_offset = le32_to_cpu(hdr->jt_offset); 6697 table_size = le32_to_cpu(hdr->jt_size); 6698 } else if (me == 2) { 6699 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; 6700 fw_data = (const __le32 *) 6701 ((const char *)rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6702 table_offset = le32_to_cpu(hdr->jt_offset); 6703 table_size = le32_to_cpu(hdr->jt_size); 6704 } else if (me == 3) { 6705 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; 6706 fw_data = (const __le32 *) 6707 ((const char *)rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6708 table_offset = le32_to_cpu(hdr->jt_offset); 6709 table_size = le32_to_cpu(hdr->jt_size); 6710 } else { 6711 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; 6712 fw_data = (const __le32 *) 6713 ((const char *)rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6714 table_offset = le32_to_cpu(hdr->jt_offset); 6715 table_size = le32_to_cpu(hdr->jt_size); 6716 } 6717 6718 for (i = 0; i < table_size; i ++) { 6719 dst_ptr[bo_offset + i] = 6720 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i])); 6721 } 6722 bo_offset += table_size; 6723 } else { 6724 const __be32 *fw_data; 6725 table_size = CP_ME_TABLE_SIZE; 6726 6727 if (me == 0) { 6728 fw_data = (const __be32 *)rdev->ce_fw->data; 6729 table_offset = CP_ME_TABLE_OFFSET; 6730 } else if (me == 1) { 6731 fw_data = (const __be32 *)rdev->pfp_fw->data; 6732 table_offset = CP_ME_TABLE_OFFSET; 6733 } else if (me == 2) { 6734 fw_data = (const __be32 *)rdev->me_fw->data; 6735 table_offset = CP_ME_TABLE_OFFSET; 6736 } else { 6737 fw_data = (const __be32 *)rdev->mec_fw->data; 6738 table_offset = CP_MEC_TABLE_OFFSET; 6739 } 6740 6741 for (i = 0; i < table_size; i ++) { 6742 dst_ptr[bo_offset + i] = 6743 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i])); 6744 } 6745 bo_offset += table_size; 6746 } 6747 } 6748 } 6749 6750 static void cik_enable_gfx_cgpg(struct radeon_device *rdev, 6751 bool enable) 6752 { 6753 u32 data, orig; 6754 6755 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) { 6756 orig = data = RREG32(RLC_PG_CNTL); 6757 data |= GFX_PG_ENABLE; 6758 if (orig != data) 6759 WREG32(RLC_PG_CNTL, data); 6760 6761 orig = data = RREG32(RLC_AUTO_PG_CTRL); 6762 data |= AUTO_PG_EN; 6763 if (orig != data) 6764 WREG32(RLC_AUTO_PG_CTRL, data); 6765 } else { 6766 orig = data = RREG32(RLC_PG_CNTL); 6767 data &= ~GFX_PG_ENABLE; 6768 if (orig != data) 6769 WREG32(RLC_PG_CNTL, data); 6770 6771 orig = data = RREG32(RLC_AUTO_PG_CTRL); 6772 data &= ~AUTO_PG_EN; 6773 if (orig != data) 6774 WREG32(RLC_AUTO_PG_CTRL, data); 6775 6776 data = RREG32(DB_RENDER_CONTROL); 6777 } 6778 } 6779 6780 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) 6781 { 6782 u32 mask = 0, tmp, tmp1; 6783 int i; 6784 6785 cik_select_se_sh(rdev, se, sh); 6786 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 6787 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); 6788 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6789 6790 tmp &= 0xffff0000; 6791 6792 tmp |= tmp1; 6793 tmp >>= 16; 6794 6795 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) { 6796 mask <<= 1; 6797 mask |= 1; 6798 } 6799 6800 return (~tmp) & mask; 6801 } 6802 6803 static void cik_init_ao_cu_mask(struct radeon_device *rdev) 6804 { 6805 u32 i, j, k, active_cu_number = 0; 6806 u32 mask, counter, cu_bitmap; 6807 u32 tmp = 0; 6808 6809 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 6810 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 6811 mask = 1; 6812 cu_bitmap = 0; 6813 counter = 0; 6814 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) { 6815 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) { 6816 if (counter < 2) 6817 cu_bitmap |= mask; 6818 counter ++; 6819 } 6820 mask <<= 1; 6821 } 6822 6823 active_cu_number += counter; 6824 tmp |= (cu_bitmap << (i * 16 + j * 8)); 6825 } 6826 } 6827 6828 WREG32(RLC_PG_AO_CU_MASK, tmp); 6829 6830 tmp = RREG32(RLC_MAX_PG_CU); 6831 tmp &= ~MAX_PU_CU_MASK; 6832 tmp |= MAX_PU_CU(active_cu_number); 6833 WREG32(RLC_MAX_PG_CU, tmp); 6834 } 6835 6836 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev, 6837 bool enable) 6838 { 6839 u32 data, orig; 6840 6841 orig = data = RREG32(RLC_PG_CNTL); 6842 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG)) 6843 data |= STATIC_PER_CU_PG_ENABLE; 6844 else 6845 data &= ~STATIC_PER_CU_PG_ENABLE; 6846 if (orig != data) 6847 WREG32(RLC_PG_CNTL, data); 6848 } 6849 6850 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev, 6851 bool enable) 6852 { 6853 u32 data, orig; 6854 6855 orig = data = RREG32(RLC_PG_CNTL); 6856 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG)) 6857 data |= DYN_PER_CU_PG_ENABLE; 6858 else 6859 data &= ~DYN_PER_CU_PG_ENABLE; 6860 if (orig != data) 6861 WREG32(RLC_PG_CNTL, data); 6862 } 6863 6864 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90 6865 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D 6866 6867 static void cik_init_gfx_cgpg(struct radeon_device *rdev) 6868 { 6869 u32 data, orig; 6870 u32 i; 6871 6872 if (rdev->rlc.cs_data) { 6873 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); 6874 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr)); 6875 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr)); 6876 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size); 6877 } else { 6878 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); 6879 for (i = 0; i < 3; i++) 6880 WREG32(RLC_GPM_SCRATCH_DATA, 0); 6881 } 6882 if (rdev->rlc.reg_list) { 6883 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET); 6884 for (i = 0; i < rdev->rlc.reg_list_size; i++) 6885 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]); 6886 } 6887 6888 orig = data = RREG32(RLC_PG_CNTL); 6889 data |= GFX_PG_SRC; 6890 if (orig != data) 6891 WREG32(RLC_PG_CNTL, data); 6892 6893 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 6894 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8); 6895 6896 data = RREG32(CP_RB_WPTR_POLL_CNTL); 6897 data &= ~IDLE_POLL_COUNT_MASK; 6898 data |= IDLE_POLL_COUNT(0x60); 6899 WREG32(CP_RB_WPTR_POLL_CNTL, data); 6900 6901 data = 0x10101010; 6902 WREG32(RLC_PG_DELAY, data); 6903 6904 data = RREG32(RLC_PG_DELAY_2); 6905 data &= ~0xff; 6906 data |= 0x3; 6907 WREG32(RLC_PG_DELAY_2, data); 6908 6909 data = RREG32(RLC_AUTO_PG_CTRL); 6910 data &= ~GRBM_REG_SGIT_MASK; 6911 data |= GRBM_REG_SGIT(0x700); 6912 WREG32(RLC_AUTO_PG_CTRL, data); 6913 6914 } 6915 6916 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable) 6917 { 6918 cik_enable_gfx_cgpg(rdev, enable); 6919 cik_enable_gfx_static_mgpg(rdev, enable); 6920 cik_enable_gfx_dynamic_mgpg(rdev, enable); 6921 } 6922 6923 u32 cik_get_csb_size(struct radeon_device *rdev) 6924 { 6925 u32 count = 0; 6926 const struct cs_section_def *sect = NULL; 6927 const struct cs_extent_def *ext = NULL; 6928 6929 if (rdev->rlc.cs_data == NULL) 6930 return 0; 6931 6932 /* begin clear state */ 6933 count += 2; 6934 /* context control state */ 6935 count += 3; 6936 6937 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 6938 for (ext = sect->section; ext->extent != NULL; ++ext) { 6939 if (sect->id == SECT_CONTEXT) 6940 count += 2 + ext->reg_count; 6941 else 6942 return 0; 6943 } 6944 } 6945 /* pa_sc_raster_config/pa_sc_raster_config1 */ 6946 count += 4; 6947 /* end clear state */ 6948 count += 2; 6949 /* clear state */ 6950 count += 2; 6951 6952 return count; 6953 } 6954 6955 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) 6956 { 6957 u32 count = 0, i; 6958 const struct cs_section_def *sect = NULL; 6959 const struct cs_extent_def *ext = NULL; 6960 6961 if (rdev->rlc.cs_data == NULL) 6962 return; 6963 if (buffer == NULL) 6964 return; 6965 6966 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 6967 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 6968 6969 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 6970 buffer[count++] = cpu_to_le32(0x80000000); 6971 buffer[count++] = cpu_to_le32(0x80000000); 6972 6973 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 6974 for (ext = sect->section; ext->extent != NULL; ++ext) { 6975 if (sect->id == SECT_CONTEXT) { 6976 buffer[count++] = 6977 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 6978 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000); 6979 for (i = 0; i < ext->reg_count; i++) 6980 buffer[count++] = cpu_to_le32(ext->extent[i]); 6981 } else { 6982 return; 6983 } 6984 } 6985 } 6986 6987 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 6988 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 6989 switch (rdev->family) { 6990 case CHIP_BONAIRE: 6991 buffer[count++] = cpu_to_le32(0x16000012); 6992 buffer[count++] = cpu_to_le32(0x00000000); 6993 break; 6994 case CHIP_KAVERI: 6995 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ 6996 buffer[count++] = cpu_to_le32(0x00000000); 6997 break; 6998 case CHIP_KABINI: 6999 case CHIP_MULLINS: 7000 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ 7001 buffer[count++] = cpu_to_le32(0x00000000); 7002 break; 7003 case CHIP_HAWAII: 7004 buffer[count++] = cpu_to_le32(0x3a00161a); 7005 buffer[count++] = cpu_to_le32(0x0000002e); 7006 break; 7007 default: 7008 buffer[count++] = cpu_to_le32(0x00000000); 7009 buffer[count++] = cpu_to_le32(0x00000000); 7010 break; 7011 } 7012 7013 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 7014 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 7015 7016 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 7017 buffer[count++] = cpu_to_le32(0); 7018 } 7019 7020 static void cik_init_pg(struct radeon_device *rdev) 7021 { 7022 if (rdev->pg_flags) { 7023 cik_enable_sck_slowdown_on_pu(rdev, true); 7024 cik_enable_sck_slowdown_on_pd(rdev, true); 7025 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 7026 cik_init_gfx_cgpg(rdev); 7027 cik_enable_cp_pg(rdev, true); 7028 cik_enable_gds_pg(rdev, true); 7029 } 7030 cik_init_ao_cu_mask(rdev); 7031 cik_update_gfx_pg(rdev, true); 7032 } 7033 } 7034 7035 static void cik_fini_pg(struct radeon_device *rdev) 7036 { 7037 if (rdev->pg_flags) { 7038 cik_update_gfx_pg(rdev, false); 7039 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 7040 cik_enable_cp_pg(rdev, false); 7041 cik_enable_gds_pg(rdev, false); 7042 } 7043 } 7044 } 7045 7046 /* 7047 * Interrupts 7048 * Starting with r6xx, interrupts are handled via a ring buffer. 7049 * Ring buffers are areas of GPU accessible memory that the GPU 7050 * writes interrupt vectors into and the host reads vectors out of. 7051 * There is a rptr (read pointer) that determines where the 7052 * host is currently reading, and a wptr (write pointer) 7053 * which determines where the GPU has written. When the 7054 * pointers are equal, the ring is idle. When the GPU 7055 * writes vectors to the ring buffer, it increments the 7056 * wptr. When there is an interrupt, the host then starts 7057 * fetching commands and processing them until the pointers are 7058 * equal again at which point it updates the rptr. 7059 */ 7060 7061 /** 7062 * cik_enable_interrupts - Enable the interrupt ring buffer 7063 * 7064 * @rdev: radeon_device pointer 7065 * 7066 * Enable the interrupt ring buffer (CIK). 7067 */ 7068 static void cik_enable_interrupts(struct radeon_device *rdev) 7069 { 7070 u32 ih_cntl = RREG32(IH_CNTL); 7071 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 7072 7073 ih_cntl |= ENABLE_INTR; 7074 ih_rb_cntl |= IH_RB_ENABLE; 7075 WREG32(IH_CNTL, ih_cntl); 7076 WREG32(IH_RB_CNTL, ih_rb_cntl); 7077 rdev->ih.enabled = true; 7078 } 7079 7080 /** 7081 * cik_disable_interrupts - Disable the interrupt ring buffer 7082 * 7083 * @rdev: radeon_device pointer 7084 * 7085 * Disable the interrupt ring buffer (CIK). 7086 */ 7087 static void cik_disable_interrupts(struct radeon_device *rdev) 7088 { 7089 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 7090 u32 ih_cntl = RREG32(IH_CNTL); 7091 7092 ih_rb_cntl &= ~IH_RB_ENABLE; 7093 ih_cntl &= ~ENABLE_INTR; 7094 WREG32(IH_RB_CNTL, ih_rb_cntl); 7095 WREG32(IH_CNTL, ih_cntl); 7096 /* set rptr, wptr to 0 */ 7097 WREG32(IH_RB_RPTR, 0); 7098 WREG32(IH_RB_WPTR, 0); 7099 rdev->ih.enabled = false; 7100 rdev->ih.rptr = 0; 7101 } 7102 7103 /** 7104 * cik_disable_interrupt_state - Disable all interrupt sources 7105 * 7106 * @rdev: radeon_device pointer 7107 * 7108 * Clear all interrupt enable bits used by the driver (CIK). 7109 */ 7110 static void cik_disable_interrupt_state(struct radeon_device *rdev) 7111 { 7112 u32 tmp; 7113 7114 /* gfx ring */ 7115 tmp = RREG32(CP_INT_CNTL_RING0) & 7116 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 7117 WREG32(CP_INT_CNTL_RING0, tmp); 7118 /* sdma */ 7119 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 7120 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp); 7121 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 7122 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp); 7123 /* compute queues */ 7124 WREG32(CP_ME1_PIPE0_INT_CNTL, 0); 7125 WREG32(CP_ME1_PIPE1_INT_CNTL, 0); 7126 WREG32(CP_ME1_PIPE2_INT_CNTL, 0); 7127 WREG32(CP_ME1_PIPE3_INT_CNTL, 0); 7128 WREG32(CP_ME2_PIPE0_INT_CNTL, 0); 7129 WREG32(CP_ME2_PIPE1_INT_CNTL, 0); 7130 WREG32(CP_ME2_PIPE2_INT_CNTL, 0); 7131 WREG32(CP_ME2_PIPE3_INT_CNTL, 0); 7132 /* grbm */ 7133 WREG32(GRBM_INT_CNTL, 0); 7134 /* vline/vblank, etc. */ 7135 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 7136 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 7137 if (rdev->num_crtc >= 4) { 7138 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 7139 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 7140 } 7141 if (rdev->num_crtc >= 6) { 7142 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 7143 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 7144 } 7145 /* pflip */ 7146 if (rdev->num_crtc >= 2) { 7147 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 7148 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 7149 } 7150 if (rdev->num_crtc >= 4) { 7151 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 7152 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 7153 } 7154 if (rdev->num_crtc >= 6) { 7155 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 7156 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 7157 } 7158 7159 /* dac hotplug */ 7160 WREG32(DAC_AUTODETECT_INT_CONTROL, 0); 7161 7162 /* digital hotplug */ 7163 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7164 WREG32(DC_HPD1_INT_CONTROL, tmp); 7165 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7166 WREG32(DC_HPD2_INT_CONTROL, tmp); 7167 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7168 WREG32(DC_HPD3_INT_CONTROL, tmp); 7169 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7170 WREG32(DC_HPD4_INT_CONTROL, tmp); 7171 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7172 WREG32(DC_HPD5_INT_CONTROL, tmp); 7173 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7174 WREG32(DC_HPD6_INT_CONTROL, tmp); 7175 7176 } 7177 7178 /** 7179 * cik_irq_init - init and enable the interrupt ring 7180 * 7181 * @rdev: radeon_device pointer 7182 * 7183 * Allocate a ring buffer for the interrupt controller, 7184 * enable the RLC, disable interrupts, enable the IH 7185 * ring buffer and enable it (CIK). 7186 * Called at device load and reume. 7187 * Returns 0 for success, errors for failure. 7188 */ 7189 static int cik_irq_init(struct radeon_device *rdev) 7190 { 7191 int ret = 0; 7192 int rb_bufsz; 7193 u32 interrupt_cntl, ih_cntl, ih_rb_cntl; 7194 7195 /* allocate ring */ 7196 ret = r600_ih_ring_alloc(rdev); 7197 if (ret) 7198 return ret; 7199 7200 /* disable irqs */ 7201 cik_disable_interrupts(rdev); 7202 7203 /* init rlc */ 7204 ret = cik_rlc_resume(rdev); 7205 if (ret) { 7206 r600_ih_ring_fini(rdev); 7207 return ret; 7208 } 7209 7210 /* setup interrupt control */ 7211 /* XXX this should actually be a bus address, not an MC address. same on older asics */ 7212 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8); 7213 interrupt_cntl = RREG32(INTERRUPT_CNTL); 7214 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi 7215 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN 7216 */ 7217 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE; 7218 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */ 7219 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN; 7220 WREG32(INTERRUPT_CNTL, interrupt_cntl); 7221 7222 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); 7223 rb_bufsz = order_base_2(rdev->ih.ring_size / 4); 7224 7225 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | 7226 IH_WPTR_OVERFLOW_CLEAR | 7227 (rb_bufsz << 1)); 7228 7229 if (rdev->wb.enabled) 7230 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE; 7231 7232 /* set the writeback address whether it's enabled or not */ 7233 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); 7234 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); 7235 7236 WREG32(IH_RB_CNTL, ih_rb_cntl); 7237 7238 /* set rptr, wptr to 0 */ 7239 WREG32(IH_RB_RPTR, 0); 7240 WREG32(IH_RB_WPTR, 0); 7241 7242 /* Default settings for IH_CNTL (disabled at first) */ 7243 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0); 7244 /* RPTR_REARM only works if msi's are enabled */ 7245 if (rdev->msi_enabled) 7246 ih_cntl |= RPTR_REARM; 7247 WREG32(IH_CNTL, ih_cntl); 7248 7249 /* force the active interrupt state to all disabled */ 7250 cik_disable_interrupt_state(rdev); 7251 7252 pci_enable_busmaster(rdev->pdev->dev); 7253 7254 /* enable irqs */ 7255 cik_enable_interrupts(rdev); 7256 7257 return ret; 7258 } 7259 7260 /** 7261 * cik_irq_set - enable/disable interrupt sources 7262 * 7263 * @rdev: radeon_device pointer 7264 * 7265 * Enable interrupt sources on the GPU (vblanks, hpd, 7266 * etc.) (CIK). 7267 * Returns 0 for success, errors for failure. 7268 */ 7269 int cik_irq_set(struct radeon_device *rdev) 7270 { 7271 u32 cp_int_cntl; 7272 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3; 7273 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3; 7274 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 7275 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 7276 u32 grbm_int_cntl = 0; 7277 u32 dma_cntl, dma_cntl1; 7278 u32 thermal_int; 7279 7280 if (!rdev->irq.installed) { 7281 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 7282 return -EINVAL; 7283 } 7284 /* don't enable anything if the ih is disabled */ 7285 if (!rdev->ih.enabled) { 7286 cik_disable_interrupts(rdev); 7287 /* force the active interrupt state to all disabled */ 7288 cik_disable_interrupt_state(rdev); 7289 return 0; 7290 } 7291 7292 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) & 7293 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 7294 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE; 7295 7296 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; 7297 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; 7298 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; 7299 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; 7300 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 7301 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 7302 7303 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 7304 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 7305 7306 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7307 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7308 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7309 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7310 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7311 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7312 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7313 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7314 7315 if (rdev->flags & RADEON_IS_IGP) 7316 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) & 7317 ~(THERM_INTH_MASK | THERM_INTL_MASK); 7318 else 7319 thermal_int = RREG32_SMC(CG_THERMAL_INT) & 7320 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 7321 7322 /* enable CP interrupts on all rings */ 7323 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 7324 DRM_DEBUG("cik_irq_set: sw int gfx\n"); 7325 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 7326 } 7327 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 7328 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 7329 DRM_DEBUG("si_irq_set: sw int cp1\n"); 7330 if (ring->me == 1) { 7331 switch (ring->pipe) { 7332 case 0: 7333 cp_m1p0 |= TIME_STAMP_INT_ENABLE; 7334 break; 7335 case 1: 7336 cp_m1p1 |= TIME_STAMP_INT_ENABLE; 7337 break; 7338 case 2: 7339 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 7340 break; 7341 case 3: 7342 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 7343 break; 7344 default: 7345 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); 7346 break; 7347 } 7348 } else if (ring->me == 2) { 7349 switch (ring->pipe) { 7350 case 0: 7351 cp_m2p0 |= TIME_STAMP_INT_ENABLE; 7352 break; 7353 case 1: 7354 cp_m2p1 |= TIME_STAMP_INT_ENABLE; 7355 break; 7356 case 2: 7357 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 7358 break; 7359 case 3: 7360 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 7361 break; 7362 default: 7363 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); 7364 break; 7365 } 7366 } else { 7367 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me); 7368 } 7369 } 7370 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 7371 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 7372 DRM_DEBUG("si_irq_set: sw int cp2\n"); 7373 if (ring->me == 1) { 7374 switch (ring->pipe) { 7375 case 0: 7376 cp_m1p0 |= TIME_STAMP_INT_ENABLE; 7377 break; 7378 case 1: 7379 cp_m1p1 |= TIME_STAMP_INT_ENABLE; 7380 break; 7381 case 2: 7382 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 7383 break; 7384 case 3: 7385 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 7386 break; 7387 default: 7388 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); 7389 break; 7390 } 7391 } else if (ring->me == 2) { 7392 switch (ring->pipe) { 7393 case 0: 7394 cp_m2p0 |= TIME_STAMP_INT_ENABLE; 7395 break; 7396 case 1: 7397 cp_m2p1 |= TIME_STAMP_INT_ENABLE; 7398 break; 7399 case 2: 7400 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 7401 break; 7402 case 3: 7403 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 7404 break; 7405 default: 7406 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); 7407 break; 7408 } 7409 } else { 7410 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me); 7411 } 7412 } 7413 7414 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 7415 DRM_DEBUG("cik_irq_set: sw int dma\n"); 7416 dma_cntl |= TRAP_ENABLE; 7417 } 7418 7419 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 7420 DRM_DEBUG("cik_irq_set: sw int dma1\n"); 7421 dma_cntl1 |= TRAP_ENABLE; 7422 } 7423 7424 if (rdev->irq.crtc_vblank_int[0] || 7425 atomic_read(&rdev->irq.pflip[0])) { 7426 DRM_DEBUG("cik_irq_set: vblank 0\n"); 7427 crtc1 |= VBLANK_INTERRUPT_MASK; 7428 } 7429 if (rdev->irq.crtc_vblank_int[1] || 7430 atomic_read(&rdev->irq.pflip[1])) { 7431 DRM_DEBUG("cik_irq_set: vblank 1\n"); 7432 crtc2 |= VBLANK_INTERRUPT_MASK; 7433 } 7434 if (rdev->irq.crtc_vblank_int[2] || 7435 atomic_read(&rdev->irq.pflip[2])) { 7436 DRM_DEBUG("cik_irq_set: vblank 2\n"); 7437 crtc3 |= VBLANK_INTERRUPT_MASK; 7438 } 7439 if (rdev->irq.crtc_vblank_int[3] || 7440 atomic_read(&rdev->irq.pflip[3])) { 7441 DRM_DEBUG("cik_irq_set: vblank 3\n"); 7442 crtc4 |= VBLANK_INTERRUPT_MASK; 7443 } 7444 if (rdev->irq.crtc_vblank_int[4] || 7445 atomic_read(&rdev->irq.pflip[4])) { 7446 DRM_DEBUG("cik_irq_set: vblank 4\n"); 7447 crtc5 |= VBLANK_INTERRUPT_MASK; 7448 } 7449 if (rdev->irq.crtc_vblank_int[5] || 7450 atomic_read(&rdev->irq.pflip[5])) { 7451 DRM_DEBUG("cik_irq_set: vblank 5\n"); 7452 crtc6 |= VBLANK_INTERRUPT_MASK; 7453 } 7454 if (rdev->irq.hpd[0]) { 7455 DRM_DEBUG("cik_irq_set: hpd 1\n"); 7456 hpd1 |= DC_HPDx_INT_EN; 7457 } 7458 if (rdev->irq.hpd[1]) { 7459 DRM_DEBUG("cik_irq_set: hpd 2\n"); 7460 hpd2 |= DC_HPDx_INT_EN; 7461 } 7462 if (rdev->irq.hpd[2]) { 7463 DRM_DEBUG("cik_irq_set: hpd 3\n"); 7464 hpd3 |= DC_HPDx_INT_EN; 7465 } 7466 if (rdev->irq.hpd[3]) { 7467 DRM_DEBUG("cik_irq_set: hpd 4\n"); 7468 hpd4 |= DC_HPDx_INT_EN; 7469 } 7470 if (rdev->irq.hpd[4]) { 7471 DRM_DEBUG("cik_irq_set: hpd 5\n"); 7472 hpd5 |= DC_HPDx_INT_EN; 7473 } 7474 if (rdev->irq.hpd[5]) { 7475 DRM_DEBUG("cik_irq_set: hpd 6\n"); 7476 hpd6 |= DC_HPDx_INT_EN; 7477 } 7478 7479 if (rdev->irq.dpm_thermal) { 7480 DRM_DEBUG("dpm thermal\n"); 7481 if (rdev->flags & RADEON_IS_IGP) 7482 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK; 7483 else 7484 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; 7485 } 7486 7487 WREG32(CP_INT_CNTL_RING0, cp_int_cntl); 7488 7489 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl); 7490 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1); 7491 7492 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0); 7493 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1); 7494 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2); 7495 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3); 7496 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0); 7497 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1); 7498 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2); 7499 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3); 7500 7501 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 7502 7503 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 7504 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 7505 if (rdev->num_crtc >= 4) { 7506 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 7507 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 7508 } 7509 if (rdev->num_crtc >= 6) { 7510 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 7511 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 7512 } 7513 7514 if (rdev->num_crtc >= 2) { 7515 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 7516 GRPH_PFLIP_INT_MASK); 7517 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 7518 GRPH_PFLIP_INT_MASK); 7519 } 7520 if (rdev->num_crtc >= 4) { 7521 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 7522 GRPH_PFLIP_INT_MASK); 7523 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 7524 GRPH_PFLIP_INT_MASK); 7525 } 7526 if (rdev->num_crtc >= 6) { 7527 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 7528 GRPH_PFLIP_INT_MASK); 7529 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 7530 GRPH_PFLIP_INT_MASK); 7531 } 7532 7533 WREG32(DC_HPD1_INT_CONTROL, hpd1); 7534 WREG32(DC_HPD2_INT_CONTROL, hpd2); 7535 WREG32(DC_HPD3_INT_CONTROL, hpd3); 7536 WREG32(DC_HPD4_INT_CONTROL, hpd4); 7537 WREG32(DC_HPD5_INT_CONTROL, hpd5); 7538 WREG32(DC_HPD6_INT_CONTROL, hpd6); 7539 7540 if (rdev->flags & RADEON_IS_IGP) 7541 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int); 7542 else 7543 WREG32_SMC(CG_THERMAL_INT, thermal_int); 7544 7545 return 0; 7546 } 7547 7548 /** 7549 * cik_irq_ack - ack interrupt sources 7550 * 7551 * @rdev: radeon_device pointer 7552 * 7553 * Ack interrupt sources on the GPU (vblanks, hpd, 7554 * etc.) (CIK). Certain interrupts sources are sw 7555 * generated and do not require an explicit ack. 7556 */ 7557 static inline void cik_irq_ack(struct radeon_device *rdev) 7558 { 7559 u32 tmp; 7560 7561 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS); 7562 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 7563 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 7564 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 7565 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 7566 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 7567 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6); 7568 7569 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS + 7570 EVERGREEN_CRTC0_REGISTER_OFFSET); 7571 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS + 7572 EVERGREEN_CRTC1_REGISTER_OFFSET); 7573 if (rdev->num_crtc >= 4) { 7574 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS + 7575 EVERGREEN_CRTC2_REGISTER_OFFSET); 7576 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS + 7577 EVERGREEN_CRTC3_REGISTER_OFFSET); 7578 } 7579 if (rdev->num_crtc >= 6) { 7580 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS + 7581 EVERGREEN_CRTC4_REGISTER_OFFSET); 7582 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS + 7583 EVERGREEN_CRTC5_REGISTER_OFFSET); 7584 } 7585 7586 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 7587 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, 7588 GRPH_PFLIP_INT_CLEAR); 7589 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 7590 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, 7591 GRPH_PFLIP_INT_CLEAR); 7592 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) 7593 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 7594 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) 7595 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 7596 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 7597 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 7598 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) 7599 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 7600 7601 if (rdev->num_crtc >= 4) { 7602 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 7603 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, 7604 GRPH_PFLIP_INT_CLEAR); 7605 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 7606 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, 7607 GRPH_PFLIP_INT_CLEAR); 7608 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 7609 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 7610 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 7611 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 7612 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 7613 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 7614 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 7615 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 7616 } 7617 7618 if (rdev->num_crtc >= 6) { 7619 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 7620 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, 7621 GRPH_PFLIP_INT_CLEAR); 7622 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 7623 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, 7624 GRPH_PFLIP_INT_CLEAR); 7625 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 7626 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 7627 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 7628 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 7629 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 7630 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 7631 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 7632 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 7633 } 7634 7635 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { 7636 tmp = RREG32(DC_HPD1_INT_CONTROL); 7637 tmp |= DC_HPDx_INT_ACK; 7638 WREG32(DC_HPD1_INT_CONTROL, tmp); 7639 } 7640 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { 7641 tmp = RREG32(DC_HPD2_INT_CONTROL); 7642 tmp |= DC_HPDx_INT_ACK; 7643 WREG32(DC_HPD2_INT_CONTROL, tmp); 7644 } 7645 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { 7646 tmp = RREG32(DC_HPD3_INT_CONTROL); 7647 tmp |= DC_HPDx_INT_ACK; 7648 WREG32(DC_HPD3_INT_CONTROL, tmp); 7649 } 7650 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { 7651 tmp = RREG32(DC_HPD4_INT_CONTROL); 7652 tmp |= DC_HPDx_INT_ACK; 7653 WREG32(DC_HPD4_INT_CONTROL, tmp); 7654 } 7655 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { 7656 tmp = RREG32(DC_HPD5_INT_CONTROL); 7657 tmp |= DC_HPDx_INT_ACK; 7658 WREG32(DC_HPD5_INT_CONTROL, tmp); 7659 } 7660 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { 7661 tmp = RREG32(DC_HPD5_INT_CONTROL); 7662 tmp |= DC_HPDx_INT_ACK; 7663 WREG32(DC_HPD6_INT_CONTROL, tmp); 7664 } 7665 } 7666 7667 /** 7668 * cik_irq_disable - disable interrupts 7669 * 7670 * @rdev: radeon_device pointer 7671 * 7672 * Disable interrupts on the hw (CIK). 7673 */ 7674 static void cik_irq_disable(struct radeon_device *rdev) 7675 { 7676 cik_disable_interrupts(rdev); 7677 /* Wait and acknowledge irq */ 7678 mdelay(1); 7679 cik_irq_ack(rdev); 7680 cik_disable_interrupt_state(rdev); 7681 } 7682 7683 /** 7684 * cik_irq_disable - disable interrupts for suspend 7685 * 7686 * @rdev: radeon_device pointer 7687 * 7688 * Disable interrupts and stop the RLC (CIK). 7689 * Used for suspend. 7690 */ 7691 static void cik_irq_suspend(struct radeon_device *rdev) 7692 { 7693 cik_irq_disable(rdev); 7694 cik_rlc_stop(rdev); 7695 } 7696 7697 /** 7698 * cik_irq_fini - tear down interrupt support 7699 * 7700 * @rdev: radeon_device pointer 7701 * 7702 * Disable interrupts on the hw and free the IH ring 7703 * buffer (CIK). 7704 * Used for driver unload. 7705 */ 7706 static void cik_irq_fini(struct radeon_device *rdev) 7707 { 7708 cik_irq_suspend(rdev); 7709 r600_ih_ring_fini(rdev); 7710 } 7711 7712 /** 7713 * cik_get_ih_wptr - get the IH ring buffer wptr 7714 * 7715 * @rdev: radeon_device pointer 7716 * 7717 * Get the IH ring buffer wptr from either the register 7718 * or the writeback memory buffer (CIK). Also check for 7719 * ring buffer overflow and deal with it. 7720 * Used by cik_irq_process(). 7721 * Returns the value of the wptr. 7722 */ 7723 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev) 7724 { 7725 u32 wptr, tmp; 7726 7727 if (rdev->wb.enabled) 7728 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 7729 else 7730 wptr = RREG32(IH_RB_WPTR); 7731 7732 if (wptr & RB_OVERFLOW) { 7733 wptr &= ~RB_OVERFLOW; 7734 /* When a ring buffer overflow happen start parsing interrupt 7735 * from the last not overwritten vector (wptr + 16). Hopefully 7736 * this should allow us to catchup. 7737 */ 7738 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", 7739 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); 7740 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 7741 tmp = RREG32(IH_RB_CNTL); 7742 tmp |= IH_WPTR_OVERFLOW_CLEAR; 7743 WREG32(IH_RB_CNTL, tmp); 7744 } 7745 return (wptr & rdev->ih.ptr_mask); 7746 } 7747 7748 /* CIK IV Ring 7749 * Each IV ring entry is 128 bits: 7750 * [7:0] - interrupt source id 7751 * [31:8] - reserved 7752 * [59:32] - interrupt source data 7753 * [63:60] - reserved 7754 * [71:64] - RINGID 7755 * CP: 7756 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0] 7757 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher 7758 * - for gfx, hw shader state (0=PS...5=LS, 6=CS) 7759 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes 7760 * PIPE_ID - ME0 0=3D 7761 * - ME1&2 compute dispatcher (4 pipes each) 7762 * SDMA: 7763 * INSTANCE_ID [1:0], QUEUE_ID[1:0] 7764 * INSTANCE_ID - 0 = sdma0, 1 = sdma1 7765 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1 7766 * [79:72] - VMID 7767 * [95:80] - PASID 7768 * [127:96] - reserved 7769 */ 7770 /** 7771 * cik_irq_process - interrupt handler 7772 * 7773 * @rdev: radeon_device pointer 7774 * 7775 * Interrupt hander (CIK). Walk the IH ring, 7776 * ack interrupts and schedule work to handle 7777 * interrupt events. 7778 * Returns irq process return code. 7779 */ 7780 irqreturn_t cik_irq_process(struct radeon_device *rdev) 7781 { 7782 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 7783 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 7784 u32 wptr; 7785 u32 rptr; 7786 u32 src_id, src_data, ring_id; 7787 u8 me_id, pipe_id, queue_id; 7788 u32 ring_index; 7789 bool queue_hotplug = false; 7790 bool queue_reset = false; 7791 u32 addr, status, mc_client; 7792 bool queue_thermal = false; 7793 7794 if (!rdev->ih.enabled || rdev->shutdown) 7795 return IRQ_NONE; 7796 7797 wptr = cik_get_ih_wptr(rdev); 7798 7799 restart_ih: 7800 /* is somebody else already processing irqs? */ 7801 if (atomic_xchg(&rdev->ih.lock, 1)) 7802 return IRQ_NONE; 7803 7804 rptr = rdev->ih.rptr; 7805 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 7806 7807 /* Order reading of wptr vs. reading of IH ring data */ 7808 rmb(); 7809 7810 /* display interrupts */ 7811 cik_irq_ack(rdev); 7812 7813 while (rptr != wptr) { 7814 /* wptr/rptr are in bytes! */ 7815 ring_index = rptr / 4; 7816 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 7817 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 7818 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; 7819 7820 switch (src_id) { 7821 case 1: /* D1 vblank/vline */ 7822 switch (src_data) { 7823 case 0: /* D1 vblank */ 7824 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) { 7825 if (rdev->irq.crtc_vblank_int[0]) { 7826 drm_handle_vblank(rdev->ddev, 0); 7827 rdev->pm.vblank_sync = true; 7828 wake_up(&rdev->irq.vblank_queue); 7829 } 7830 if (atomic_read(&rdev->irq.pflip[0])) 7831 radeon_crtc_handle_vblank(rdev, 0); 7832 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 7833 DRM_DEBUG("IH: D1 vblank\n"); 7834 } 7835 break; 7836 case 1: /* D1 vline */ 7837 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) { 7838 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT; 7839 DRM_DEBUG("IH: D1 vline\n"); 7840 } 7841 break; 7842 default: 7843 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7844 break; 7845 } 7846 break; 7847 case 2: /* D2 vblank/vline */ 7848 switch (src_data) { 7849 case 0: /* D2 vblank */ 7850 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 7851 if (rdev->irq.crtc_vblank_int[1]) { 7852 drm_handle_vblank(rdev->ddev, 1); 7853 rdev->pm.vblank_sync = true; 7854 wake_up(&rdev->irq.vblank_queue); 7855 } 7856 if (atomic_read(&rdev->irq.pflip[1])) 7857 radeon_crtc_handle_vblank(rdev, 1); 7858 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 7859 DRM_DEBUG("IH: D2 vblank\n"); 7860 } 7861 break; 7862 case 1: /* D2 vline */ 7863 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) { 7864 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 7865 DRM_DEBUG("IH: D2 vline\n"); 7866 } 7867 break; 7868 default: 7869 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7870 break; 7871 } 7872 break; 7873 case 3: /* D3 vblank/vline */ 7874 switch (src_data) { 7875 case 0: /* D3 vblank */ 7876 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 7877 if (rdev->irq.crtc_vblank_int[2]) { 7878 drm_handle_vblank(rdev->ddev, 2); 7879 rdev->pm.vblank_sync = true; 7880 wake_up(&rdev->irq.vblank_queue); 7881 } 7882 if (atomic_read(&rdev->irq.pflip[2])) 7883 radeon_crtc_handle_vblank(rdev, 2); 7884 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 7885 DRM_DEBUG("IH: D3 vblank\n"); 7886 } 7887 break; 7888 case 1: /* D3 vline */ 7889 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 7890 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 7891 DRM_DEBUG("IH: D3 vline\n"); 7892 } 7893 break; 7894 default: 7895 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7896 break; 7897 } 7898 break; 7899 case 4: /* D4 vblank/vline */ 7900 switch (src_data) { 7901 case 0: /* D4 vblank */ 7902 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 7903 if (rdev->irq.crtc_vblank_int[3]) { 7904 drm_handle_vblank(rdev->ddev, 3); 7905 rdev->pm.vblank_sync = true; 7906 wake_up(&rdev->irq.vblank_queue); 7907 } 7908 if (atomic_read(&rdev->irq.pflip[3])) 7909 radeon_crtc_handle_vblank(rdev, 3); 7910 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 7911 DRM_DEBUG("IH: D4 vblank\n"); 7912 } 7913 break; 7914 case 1: /* D4 vline */ 7915 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 7916 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 7917 DRM_DEBUG("IH: D4 vline\n"); 7918 } 7919 break; 7920 default: 7921 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7922 break; 7923 } 7924 break; 7925 case 5: /* D5 vblank/vline */ 7926 switch (src_data) { 7927 case 0: /* D5 vblank */ 7928 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 7929 if (rdev->irq.crtc_vblank_int[4]) { 7930 drm_handle_vblank(rdev->ddev, 4); 7931 rdev->pm.vblank_sync = true; 7932 wake_up(&rdev->irq.vblank_queue); 7933 } 7934 if (atomic_read(&rdev->irq.pflip[4])) 7935 radeon_crtc_handle_vblank(rdev, 4); 7936 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 7937 DRM_DEBUG("IH: D5 vblank\n"); 7938 } 7939 break; 7940 case 1: /* D5 vline */ 7941 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 7942 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 7943 DRM_DEBUG("IH: D5 vline\n"); 7944 } 7945 break; 7946 default: 7947 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7948 break; 7949 } 7950 break; 7951 case 6: /* D6 vblank/vline */ 7952 switch (src_data) { 7953 case 0: /* D6 vblank */ 7954 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 7955 if (rdev->irq.crtc_vblank_int[5]) { 7956 drm_handle_vblank(rdev->ddev, 5); 7957 rdev->pm.vblank_sync = true; 7958 wake_up(&rdev->irq.vblank_queue); 7959 } 7960 if (atomic_read(&rdev->irq.pflip[5])) 7961 radeon_crtc_handle_vblank(rdev, 5); 7962 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 7963 DRM_DEBUG("IH: D6 vblank\n"); 7964 } 7965 break; 7966 case 1: /* D6 vline */ 7967 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 7968 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 7969 DRM_DEBUG("IH: D6 vline\n"); 7970 } 7971 break; 7972 default: 7973 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7974 break; 7975 } 7976 break; 7977 case 8: /* D1 page flip */ 7978 case 10: /* D2 page flip */ 7979 case 12: /* D3 page flip */ 7980 case 14: /* D4 page flip */ 7981 case 16: /* D5 page flip */ 7982 case 18: /* D6 page flip */ 7983 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); 7984 if (radeon_use_pflipirq > 0) 7985 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); 7986 break; 7987 case 42: /* HPD hotplug */ 7988 switch (src_data) { 7989 case 0: 7990 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { 7991 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT; 7992 queue_hotplug = true; 7993 DRM_DEBUG("IH: HPD1\n"); 7994 } 7995 break; 7996 case 1: 7997 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { 7998 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT; 7999 queue_hotplug = true; 8000 DRM_DEBUG("IH: HPD2\n"); 8001 } 8002 break; 8003 case 2: 8004 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { 8005 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 8006 queue_hotplug = true; 8007 DRM_DEBUG("IH: HPD3\n"); 8008 } 8009 break; 8010 case 3: 8011 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { 8012 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 8013 queue_hotplug = true; 8014 DRM_DEBUG("IH: HPD4\n"); 8015 } 8016 break; 8017 case 4: 8018 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { 8019 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 8020 queue_hotplug = true; 8021 DRM_DEBUG("IH: HPD5\n"); 8022 } 8023 break; 8024 case 5: 8025 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { 8026 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 8027 queue_hotplug = true; 8028 DRM_DEBUG("IH: HPD6\n"); 8029 } 8030 break; 8031 default: 8032 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 8033 break; 8034 } 8035 break; 8036 case 124: /* UVD */ 8037 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 8038 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 8039 break; 8040 case 146: 8041 case 147: 8042 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 8043 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 8044 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT); 8045 /* reset addr and status */ 8046 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 8047 if (addr == 0x0 && status == 0x0) 8048 break; 8049 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 8050 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 8051 addr); 8052 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 8053 status); 8054 cik_vm_decode_fault(rdev, status, addr, mc_client); 8055 break; 8056 case 167: /* VCE */ 8057 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data); 8058 switch (src_data) { 8059 case 0: 8060 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX); 8061 break; 8062 case 1: 8063 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX); 8064 break; 8065 default: 8066 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); 8067 break; 8068 } 8069 break; 8070 case 176: /* GFX RB CP_INT */ 8071 case 177: /* GFX IB CP_INT */ 8072 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 8073 break; 8074 case 181: /* CP EOP event */ 8075 DRM_DEBUG("IH: CP EOP\n"); 8076 /* XXX check the bitfield order! */ 8077 me_id = (ring_id & 0x60) >> 5; 8078 pipe_id = (ring_id & 0x18) >> 3; 8079 queue_id = (ring_id & 0x7) >> 0; 8080 switch (me_id) { 8081 case 0: 8082 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 8083 break; 8084 case 1: 8085 case 2: 8086 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id)) 8087 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 8088 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id)) 8089 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 8090 break; 8091 } 8092 break; 8093 case 184: /* CP Privileged reg access */ 8094 DRM_ERROR("Illegal register access in command stream\n"); 8095 /* XXX check the bitfield order! */ 8096 me_id = (ring_id & 0x60) >> 5; 8097 pipe_id = (ring_id & 0x18) >> 3; 8098 queue_id = (ring_id & 0x7) >> 0; 8099 switch (me_id) { 8100 case 0: 8101 /* This results in a full GPU reset, but all we need to do is soft 8102 * reset the CP for gfx 8103 */ 8104 queue_reset = true; 8105 break; 8106 case 1: 8107 /* XXX compute */ 8108 queue_reset = true; 8109 break; 8110 case 2: 8111 /* XXX compute */ 8112 queue_reset = true; 8113 break; 8114 } 8115 break; 8116 case 185: /* CP Privileged inst */ 8117 DRM_ERROR("Illegal instruction in command stream\n"); 8118 /* XXX check the bitfield order! */ 8119 me_id = (ring_id & 0x60) >> 5; 8120 pipe_id = (ring_id & 0x18) >> 3; 8121 queue_id = (ring_id & 0x7) >> 0; 8122 switch (me_id) { 8123 case 0: 8124 /* This results in a full GPU reset, but all we need to do is soft 8125 * reset the CP for gfx 8126 */ 8127 queue_reset = true; 8128 break; 8129 case 1: 8130 /* XXX compute */ 8131 queue_reset = true; 8132 break; 8133 case 2: 8134 /* XXX compute */ 8135 queue_reset = true; 8136 break; 8137 } 8138 break; 8139 case 224: /* SDMA trap event */ 8140 /* XXX check the bitfield order! */ 8141 me_id = (ring_id & 0x3) >> 0; 8142 queue_id = (ring_id & 0xc) >> 2; 8143 DRM_DEBUG("IH: SDMA trap\n"); 8144 switch (me_id) { 8145 case 0: 8146 switch (queue_id) { 8147 case 0: 8148 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 8149 break; 8150 case 1: 8151 /* XXX compute */ 8152 break; 8153 case 2: 8154 /* XXX compute */ 8155 break; 8156 } 8157 break; 8158 case 1: 8159 switch (queue_id) { 8160 case 0: 8161 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 8162 break; 8163 case 1: 8164 /* XXX compute */ 8165 break; 8166 case 2: 8167 /* XXX compute */ 8168 break; 8169 } 8170 break; 8171 } 8172 break; 8173 case 230: /* thermal low to high */ 8174 DRM_DEBUG("IH: thermal low to high\n"); 8175 rdev->pm.dpm.thermal.high_to_low = false; 8176 queue_thermal = true; 8177 break; 8178 case 231: /* thermal high to low */ 8179 DRM_DEBUG("IH: thermal high to low\n"); 8180 rdev->pm.dpm.thermal.high_to_low = true; 8181 queue_thermal = true; 8182 break; 8183 case 233: /* GUI IDLE */ 8184 DRM_DEBUG("IH: GUI idle\n"); 8185 break; 8186 case 241: /* SDMA Privileged inst */ 8187 case 247: /* SDMA Privileged inst */ 8188 DRM_ERROR("Illegal instruction in SDMA command stream\n"); 8189 /* XXX check the bitfield order! */ 8190 me_id = (ring_id & 0x3) >> 0; 8191 queue_id = (ring_id & 0xc) >> 2; 8192 switch (me_id) { 8193 case 0: 8194 switch (queue_id) { 8195 case 0: 8196 queue_reset = true; 8197 break; 8198 case 1: 8199 /* XXX compute */ 8200 queue_reset = true; 8201 break; 8202 case 2: 8203 /* XXX compute */ 8204 queue_reset = true; 8205 break; 8206 } 8207 break; 8208 case 1: 8209 switch (queue_id) { 8210 case 0: 8211 queue_reset = true; 8212 break; 8213 case 1: 8214 /* XXX compute */ 8215 queue_reset = true; 8216 break; 8217 case 2: 8218 /* XXX compute */ 8219 queue_reset = true; 8220 break; 8221 } 8222 break; 8223 } 8224 break; 8225 default: 8226 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 8227 break; 8228 } 8229 8230 /* wptr/rptr are in bytes! */ 8231 rptr += 16; 8232 rptr &= rdev->ih.ptr_mask; 8233 WREG32(IH_RB_RPTR, rptr); 8234 } 8235 if (queue_hotplug) 8236 schedule_work(&rdev->hotplug_work); 8237 if (queue_reset) { 8238 rdev->needs_reset = true; 8239 wake_up_all(&rdev->fence_queue); 8240 } 8241 if (queue_thermal) 8242 taskqueue_enqueue(rdev->tq, &rdev->pm.dpm.thermal.work); 8243 rdev->ih.rptr = rptr; 8244 atomic_set(&rdev->ih.lock, 0); 8245 8246 /* make sure wptr hasn't changed while processing */ 8247 wptr = cik_get_ih_wptr(rdev); 8248 if (wptr != rptr) 8249 goto restart_ih; 8250 8251 return IRQ_HANDLED; 8252 } 8253 8254 /* 8255 * startup/shutdown callbacks 8256 */ 8257 /** 8258 * cik_startup - program the asic to a functional state 8259 * 8260 * @rdev: radeon_device pointer 8261 * 8262 * Programs the asic to a functional state (CIK). 8263 * Called by cik_init() and cik_resume(). 8264 * Returns 0 for success, error for failure. 8265 */ 8266 static int cik_startup(struct radeon_device *rdev) 8267 { 8268 struct radeon_ring *ring; 8269 u32 nop; 8270 int r; 8271 8272 /* enable pcie gen2/3 link */ 8273 cik_pcie_gen3_enable(rdev); 8274 /* enable aspm */ 8275 cik_program_aspm(rdev); 8276 8277 /* scratch needs to be initialized before MC */ 8278 r = r600_vram_scratch_init(rdev); 8279 if (r) 8280 return r; 8281 8282 cik_mc_program(rdev); 8283 8284 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) { 8285 r = ci_mc_load_microcode(rdev); 8286 if (r) { 8287 DRM_ERROR("Failed to load MC firmware!\n"); 8288 return r; 8289 } 8290 } 8291 8292 r = cik_pcie_gart_enable(rdev); 8293 if (r) 8294 return r; 8295 cik_gpu_init(rdev); 8296 8297 /* allocate rlc buffers */ 8298 if (rdev->flags & RADEON_IS_IGP) { 8299 if (rdev->family == CHIP_KAVERI) { 8300 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list; 8301 rdev->rlc.reg_list_size = 8302 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list); 8303 } else { 8304 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list; 8305 rdev->rlc.reg_list_size = 8306 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list); 8307 } 8308 } 8309 rdev->rlc.cs_data = ci_cs_data; 8310 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4; 8311 r = sumo_rlc_init(rdev); 8312 if (r) { 8313 DRM_ERROR("Failed to init rlc BOs!\n"); 8314 return r; 8315 } 8316 8317 /* allocate wb buffer */ 8318 r = radeon_wb_init(rdev); 8319 if (r) 8320 return r; 8321 8322 /* allocate mec buffers */ 8323 r = cik_mec_init(rdev); 8324 if (r) { 8325 DRM_ERROR("Failed to init MEC BOs!\n"); 8326 return r; 8327 } 8328 8329 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 8330 if (r) { 8331 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8332 return r; 8333 } 8334 8335 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 8336 if (r) { 8337 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8338 return r; 8339 } 8340 8341 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 8342 if (r) { 8343 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8344 return r; 8345 } 8346 8347 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 8348 if (r) { 8349 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 8350 return r; 8351 } 8352 8353 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 8354 if (r) { 8355 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 8356 return r; 8357 } 8358 8359 r = radeon_uvd_resume(rdev); 8360 if (!r) { 8361 r = uvd_v4_2_resume(rdev); 8362 if (!r) { 8363 r = radeon_fence_driver_start_ring(rdev, 8364 R600_RING_TYPE_UVD_INDEX); 8365 if (r) 8366 dev_err(rdev->dev, "UVD fences init error (%d).\n", r); 8367 } 8368 } 8369 if (r) 8370 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 8371 8372 r = radeon_vce_resume(rdev); 8373 if (!r) { 8374 r = vce_v2_0_resume(rdev); 8375 if (!r) 8376 r = radeon_fence_driver_start_ring(rdev, 8377 TN_RING_TYPE_VCE1_INDEX); 8378 if (!r) 8379 r = radeon_fence_driver_start_ring(rdev, 8380 TN_RING_TYPE_VCE2_INDEX); 8381 } 8382 if (r) { 8383 dev_err(rdev->dev, "VCE init error (%d).\n", r); 8384 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0; 8385 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0; 8386 } 8387 8388 /* Enable IRQ */ 8389 if (!rdev->irq.installed) { 8390 r = radeon_irq_kms_init(rdev); 8391 if (r) 8392 return r; 8393 } 8394 8395 r = cik_irq_init(rdev); 8396 if (r) { 8397 DRM_ERROR("radeon: IH init failed (%d).\n", r); 8398 radeon_irq_kms_fini(rdev); 8399 return r; 8400 } 8401 cik_irq_set(rdev); 8402 8403 if (rdev->family == CHIP_HAWAII) { 8404 if (rdev->new_fw) 8405 nop = PACKET3(PACKET3_NOP, 0x3FFF); 8406 else 8407 nop = RADEON_CP_PACKET2; 8408 } else { 8409 nop = PACKET3(PACKET3_NOP, 0x3FFF); 8410 } 8411 8412 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 8413 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 8414 nop); 8415 if (r) 8416 return r; 8417 8418 /* set up the compute queues */ 8419 /* type-2 packets are deprecated on MEC, use type-3 instead */ 8420 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 8421 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, 8422 nop); 8423 if (r) 8424 return r; 8425 ring->me = 1; /* first MEC */ 8426 ring->pipe = 0; /* first pipe */ 8427 ring->queue = 0; /* first queue */ 8428 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET; 8429 8430 /* type-2 packets are deprecated on MEC, use type-3 instead */ 8431 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 8432 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, 8433 nop); 8434 if (r) 8435 return r; 8436 /* dGPU only have 1 MEC */ 8437 ring->me = 1; /* first MEC */ 8438 ring->pipe = 0; /* first pipe */ 8439 ring->queue = 1; /* second queue */ 8440 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET; 8441 8442 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 8443 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 8444 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); 8445 if (r) 8446 return r; 8447 8448 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 8449 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, 8450 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); 8451 if (r) 8452 return r; 8453 8454 r = cik_cp_resume(rdev); 8455 if (r) 8456 return r; 8457 8458 r = cik_sdma_resume(rdev); 8459 if (r) 8460 return r; 8461 8462 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 8463 if (ring->ring_size) { 8464 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 8465 RADEON_CP_PACKET2); 8466 if (!r) 8467 r = uvd_v1_0_init(rdev); 8468 if (r) 8469 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); 8470 } 8471 8472 r = -ENOENT; 8473 8474 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; 8475 if (ring->ring_size) 8476 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 8477 VCE_CMD_NO_OP); 8478 8479 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; 8480 if (ring->ring_size) 8481 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 8482 VCE_CMD_NO_OP); 8483 8484 if (!r) 8485 r = vce_v1_0_init(rdev); 8486 else if (r != -ENOENT) 8487 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r); 8488 8489 r = radeon_ib_pool_init(rdev); 8490 if (r) { 8491 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 8492 return r; 8493 } 8494 8495 r = radeon_vm_manager_init(rdev); 8496 if (r) { 8497 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); 8498 return r; 8499 } 8500 8501 r = dce6_audio_init(rdev); 8502 if (r) 8503 return r; 8504 8505 return 0; 8506 } 8507 8508 /** 8509 * cik_resume - resume the asic to a functional state 8510 * 8511 * @rdev: radeon_device pointer 8512 * 8513 * Programs the asic to a functional state (CIK). 8514 * Called at resume. 8515 * Returns 0 for success, error for failure. 8516 */ 8517 int cik_resume(struct radeon_device *rdev) 8518 { 8519 int r; 8520 8521 /* post card */ 8522 atom_asic_init(rdev->mode_info.atom_context); 8523 8524 /* init golden registers */ 8525 cik_init_golden_registers(rdev); 8526 8527 if (rdev->pm.pm_method == PM_METHOD_DPM) 8528 radeon_pm_resume(rdev); 8529 8530 rdev->accel_working = true; 8531 r = cik_startup(rdev); 8532 if (r) { 8533 DRM_ERROR("cik startup failed on resume\n"); 8534 rdev->accel_working = false; 8535 return r; 8536 } 8537 8538 return r; 8539 8540 } 8541 8542 /** 8543 * cik_suspend - suspend the asic 8544 * 8545 * @rdev: radeon_device pointer 8546 * 8547 * Bring the chip into a state suitable for suspend (CIK). 8548 * Called at suspend. 8549 * Returns 0 for success. 8550 */ 8551 int cik_suspend(struct radeon_device *rdev) 8552 { 8553 radeon_pm_suspend(rdev); 8554 dce6_audio_fini(rdev); 8555 radeon_vm_manager_fini(rdev); 8556 cik_cp_enable(rdev, false); 8557 cik_sdma_enable(rdev, false); 8558 uvd_v1_0_fini(rdev); 8559 radeon_uvd_suspend(rdev); 8560 radeon_vce_suspend(rdev); 8561 cik_fini_pg(rdev); 8562 cik_fini_cg(rdev); 8563 cik_irq_suspend(rdev); 8564 radeon_wb_disable(rdev); 8565 cik_pcie_gart_disable(rdev); 8566 return 0; 8567 } 8568 8569 /* Plan is to move initialization in that function and use 8570 * helper function so that radeon_device_init pretty much 8571 * do nothing more than calling asic specific function. This 8572 * should also allow to remove a bunch of callback function 8573 * like vram_info. 8574 */ 8575 /** 8576 * cik_init - asic specific driver and hw init 8577 * 8578 * @rdev: radeon_device pointer 8579 * 8580 * Setup asic specific driver variables and program the hw 8581 * to a functional state (CIK). 8582 * Called at driver startup. 8583 * Returns 0 for success, errors for failure. 8584 */ 8585 int cik_init(struct radeon_device *rdev) 8586 { 8587 struct radeon_ring *ring; 8588 int r; 8589 8590 /* Read BIOS */ 8591 if (!radeon_get_bios(rdev)) { 8592 if (ASIC_IS_AVIVO(rdev)) 8593 return -EINVAL; 8594 } 8595 /* Must be an ATOMBIOS */ 8596 if (!rdev->is_atom_bios) { 8597 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); 8598 return -EINVAL; 8599 } 8600 r = radeon_atombios_init(rdev); 8601 if (r) 8602 return r; 8603 8604 /* Post card if necessary */ 8605 if (!radeon_card_posted(rdev)) { 8606 if (!rdev->bios) { 8607 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 8608 return -EINVAL; 8609 } 8610 DRM_INFO("GPU not posted. posting now...\n"); 8611 atom_asic_init(rdev->mode_info.atom_context); 8612 } 8613 /* init golden registers */ 8614 cik_init_golden_registers(rdev); 8615 /* Initialize scratch registers */ 8616 cik_scratch_init(rdev); 8617 /* Initialize surface registers */ 8618 radeon_surface_init(rdev); 8619 /* Initialize clocks */ 8620 radeon_get_clock_info(rdev->ddev); 8621 8622 /* Fence driver */ 8623 r = radeon_fence_driver_init(rdev); 8624 if (r) 8625 return r; 8626 8627 /* initialize memory controller */ 8628 r = cik_mc_init(rdev); 8629 if (r) 8630 return r; 8631 /* Memory manager */ 8632 r = radeon_bo_init(rdev); 8633 if (r) 8634 return r; 8635 8636 if (rdev->flags & RADEON_IS_IGP) { 8637 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 8638 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) { 8639 r = cik_init_microcode(rdev); 8640 if (r) { 8641 DRM_ERROR("Failed to load firmware!\n"); 8642 return r; 8643 } 8644 } 8645 } else { 8646 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 8647 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw || 8648 !rdev->mc_fw) { 8649 r = cik_init_microcode(rdev); 8650 if (r) { 8651 DRM_ERROR("Failed to load firmware!\n"); 8652 return r; 8653 } 8654 } 8655 } 8656 8657 /* Initialize power management */ 8658 radeon_pm_init(rdev); 8659 8660 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 8661 ring->ring_obj = NULL; 8662 r600_ring_init(rdev, ring, 1024 * 1024); 8663 8664 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 8665 ring->ring_obj = NULL; 8666 r600_ring_init(rdev, ring, 1024 * 1024); 8667 r = radeon_doorbell_get(rdev, &ring->doorbell_index); 8668 if (r) 8669 return r; 8670 8671 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 8672 ring->ring_obj = NULL; 8673 r600_ring_init(rdev, ring, 1024 * 1024); 8674 r = radeon_doorbell_get(rdev, &ring->doorbell_index); 8675 if (r) 8676 return r; 8677 8678 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 8679 ring->ring_obj = NULL; 8680 r600_ring_init(rdev, ring, 256 * 1024); 8681 8682 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 8683 ring->ring_obj = NULL; 8684 r600_ring_init(rdev, ring, 256 * 1024); 8685 8686 r = radeon_uvd_init(rdev); 8687 if (!r) { 8688 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 8689 ring->ring_obj = NULL; 8690 r600_ring_init(rdev, ring, 4096); 8691 } 8692 8693 r = radeon_vce_init(rdev); 8694 if (!r) { 8695 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; 8696 ring->ring_obj = NULL; 8697 r600_ring_init(rdev, ring, 4096); 8698 8699 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; 8700 ring->ring_obj = NULL; 8701 r600_ring_init(rdev, ring, 4096); 8702 } 8703 8704 rdev->ih.ring_obj = NULL; 8705 r600_ih_ring_init(rdev, 64 * 1024); 8706 8707 r = r600_pcie_gart_init(rdev); 8708 if (r) 8709 return r; 8710 8711 rdev->accel_working = true; 8712 r = cik_startup(rdev); 8713 if (r) { 8714 dev_err(rdev->dev, "disabling GPU acceleration\n"); 8715 cik_cp_fini(rdev); 8716 cik_sdma_fini(rdev); 8717 cik_irq_fini(rdev); 8718 sumo_rlc_fini(rdev); 8719 cik_mec_fini(rdev); 8720 radeon_wb_fini(rdev); 8721 radeon_ib_pool_fini(rdev); 8722 radeon_vm_manager_fini(rdev); 8723 radeon_irq_kms_fini(rdev); 8724 cik_pcie_gart_fini(rdev); 8725 rdev->accel_working = false; 8726 } 8727 8728 /* Don't start up if the MC ucode is missing. 8729 * The default clocks and voltages before the MC ucode 8730 * is loaded are not suffient for advanced operations. 8731 */ 8732 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 8733 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 8734 return -EINVAL; 8735 } 8736 8737 return 0; 8738 } 8739 8740 /** 8741 * cik_fini - asic specific driver and hw fini 8742 * 8743 * @rdev: radeon_device pointer 8744 * 8745 * Tear down the asic specific driver variables and program the hw 8746 * to an idle state (CIK). 8747 * Called at driver unload. 8748 */ 8749 void cik_fini(struct radeon_device *rdev) 8750 { 8751 radeon_pm_fini(rdev); 8752 cik_cp_fini(rdev); 8753 cik_sdma_fini(rdev); 8754 cik_fini_pg(rdev); 8755 cik_fini_cg(rdev); 8756 cik_irq_fini(rdev); 8757 sumo_rlc_fini(rdev); 8758 cik_mec_fini(rdev); 8759 radeon_wb_fini(rdev); 8760 radeon_vm_manager_fini(rdev); 8761 radeon_ib_pool_fini(rdev); 8762 radeon_irq_kms_fini(rdev); 8763 uvd_v1_0_fini(rdev); 8764 radeon_uvd_fini(rdev); 8765 radeon_vce_fini(rdev); 8766 cik_pcie_gart_fini(rdev); 8767 r600_vram_scratch_fini(rdev); 8768 radeon_gem_fini(rdev); 8769 radeon_fence_driver_fini(rdev); 8770 radeon_bo_fini(rdev); 8771 radeon_atombios_fini(rdev); 8772 kfree(rdev->bios); 8773 rdev->bios = NULL; 8774 } 8775 8776 void dce8_program_fmt(struct drm_encoder *encoder) 8777 { 8778 struct drm_device *dev = encoder->dev; 8779 struct radeon_device *rdev = dev->dev_private; 8780 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); 8781 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); 8782 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); 8783 int bpc = 0; 8784 u32 tmp = 0; 8785 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; 8786 8787 if (connector) { 8788 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 8789 bpc = radeon_get_monitor_bpc(connector); 8790 dither = radeon_connector->dither; 8791 } 8792 8793 /* LVDS/eDP FMT is set up by atom */ 8794 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) 8795 return; 8796 8797 /* not needed for analog */ 8798 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || 8799 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) 8800 return; 8801 8802 if (bpc == 0) 8803 return; 8804 8805 switch (bpc) { 8806 case 6: 8807 if (dither == RADEON_FMT_DITHER_ENABLE) 8808 /* XXX sort out optimal dither settings */ 8809 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8810 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0)); 8811 else 8812 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0)); 8813 break; 8814 case 8: 8815 if (dither == RADEON_FMT_DITHER_ENABLE) 8816 /* XXX sort out optimal dither settings */ 8817 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8818 FMT_RGB_RANDOM_ENABLE | 8819 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1)); 8820 else 8821 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1)); 8822 break; 8823 case 10: 8824 if (dither == RADEON_FMT_DITHER_ENABLE) 8825 /* XXX sort out optimal dither settings */ 8826 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8827 FMT_RGB_RANDOM_ENABLE | 8828 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2)); 8829 else 8830 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2)); 8831 break; 8832 default: 8833 /* not needed */ 8834 break; 8835 } 8836 8837 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); 8838 } 8839 8840 /* display watermark setup */ 8841 /** 8842 * dce8_line_buffer_adjust - Set up the line buffer 8843 * 8844 * @rdev: radeon_device pointer 8845 * @radeon_crtc: the selected display controller 8846 * @mode: the current display mode on the selected display 8847 * controller 8848 * 8849 * Setup up the line buffer allocation for 8850 * the selected display controller (CIK). 8851 * Returns the line buffer size in pixels. 8852 */ 8853 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev, 8854 struct radeon_crtc *radeon_crtc, 8855 struct drm_display_mode *mode) 8856 { 8857 u32 tmp, buffer_alloc, i; 8858 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 8859 /* 8860 * Line Buffer Setup 8861 * There are 6 line buffers, one for each display controllers. 8862 * There are 3 partitions per LB. Select the number of partitions 8863 * to enable based on the display width. For display widths larger 8864 * than 4096, you need use to use 2 display controllers and combine 8865 * them using the stereo blender. 8866 */ 8867 if (radeon_crtc->base.enabled && mode) { 8868 if (mode->crtc_hdisplay < 1920) { 8869 tmp = 1; 8870 buffer_alloc = 2; 8871 } else if (mode->crtc_hdisplay < 2560) { 8872 tmp = 2; 8873 buffer_alloc = 2; 8874 } else if (mode->crtc_hdisplay < 4096) { 8875 tmp = 0; 8876 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; 8877 } else { 8878 DRM_DEBUG_KMS("Mode too big for LB!\n"); 8879 tmp = 0; 8880 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; 8881 } 8882 } else { 8883 tmp = 1; 8884 buffer_alloc = 0; 8885 } 8886 8887 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset, 8888 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0)); 8889 8890 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 8891 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 8892 for (i = 0; i < rdev->usec_timeout; i++) { 8893 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 8894 DMIF_BUFFERS_ALLOCATED_COMPLETED) 8895 break; 8896 udelay(1); 8897 } 8898 8899 if (radeon_crtc->base.enabled && mode) { 8900 switch (tmp) { 8901 case 0: 8902 default: 8903 return 4096 * 2; 8904 case 1: 8905 return 1920 * 2; 8906 case 2: 8907 return 2560 * 2; 8908 } 8909 } 8910 8911 /* controller not enabled, so no lb used */ 8912 return 0; 8913 } 8914 8915 /** 8916 * cik_get_number_of_dram_channels - get the number of dram channels 8917 * 8918 * @rdev: radeon_device pointer 8919 * 8920 * Look up the number of video ram channels (CIK). 8921 * Used for display watermark bandwidth calculations 8922 * Returns the number of dram channels 8923 */ 8924 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev) 8925 { 8926 u32 tmp = RREG32(MC_SHARED_CHMAP); 8927 8928 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 8929 case 0: 8930 default: 8931 return 1; 8932 case 1: 8933 return 2; 8934 case 2: 8935 return 4; 8936 case 3: 8937 return 8; 8938 case 4: 8939 return 3; 8940 case 5: 8941 return 6; 8942 case 6: 8943 return 10; 8944 case 7: 8945 return 12; 8946 case 8: 8947 return 16; 8948 } 8949 } 8950 8951 struct dce8_wm_params { 8952 u32 dram_channels; /* number of dram channels */ 8953 u32 yclk; /* bandwidth per dram data pin in kHz */ 8954 u32 sclk; /* engine clock in kHz */ 8955 u32 disp_clk; /* display clock in kHz */ 8956 u32 src_width; /* viewport width */ 8957 u32 active_time; /* active display time in ns */ 8958 u32 blank_time; /* blank time in ns */ 8959 bool interlaced; /* mode is interlaced */ 8960 fixed20_12 vsc; /* vertical scale ratio */ 8961 u32 num_heads; /* number of active crtcs */ 8962 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 8963 u32 lb_size; /* line buffer allocated to pipe */ 8964 u32 vtaps; /* vertical scaler taps */ 8965 }; 8966 8967 /** 8968 * dce8_dram_bandwidth - get the dram bandwidth 8969 * 8970 * @wm: watermark calculation data 8971 * 8972 * Calculate the raw dram bandwidth (CIK). 8973 * Used for display watermark bandwidth calculations 8974 * Returns the dram bandwidth in MBytes/s 8975 */ 8976 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm) 8977 { 8978 /* Calculate raw DRAM Bandwidth */ 8979 fixed20_12 dram_efficiency; /* 0.7 */ 8980 fixed20_12 yclk, dram_channels, bandwidth; 8981 fixed20_12 a; 8982 8983 a.full = dfixed_const(1000); 8984 yclk.full = dfixed_const(wm->yclk); 8985 yclk.full = dfixed_div(yclk, a); 8986 dram_channels.full = dfixed_const(wm->dram_channels * 4); 8987 a.full = dfixed_const(10); 8988 dram_efficiency.full = dfixed_const(7); 8989 dram_efficiency.full = dfixed_div(dram_efficiency, a); 8990 bandwidth.full = dfixed_mul(dram_channels, yclk); 8991 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 8992 8993 return dfixed_trunc(bandwidth); 8994 } 8995 8996 /** 8997 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display 8998 * 8999 * @wm: watermark calculation data 9000 * 9001 * Calculate the dram bandwidth used for display (CIK). 9002 * Used for display watermark bandwidth calculations 9003 * Returns the dram bandwidth for display in MBytes/s 9004 */ 9005 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm) 9006 { 9007 /* Calculate DRAM Bandwidth and the part allocated to display. */ 9008 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 9009 fixed20_12 yclk, dram_channels, bandwidth; 9010 fixed20_12 a; 9011 9012 a.full = dfixed_const(1000); 9013 yclk.full = dfixed_const(wm->yclk); 9014 yclk.full = dfixed_div(yclk, a); 9015 dram_channels.full = dfixed_const(wm->dram_channels * 4); 9016 a.full = dfixed_const(10); 9017 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 9018 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 9019 bandwidth.full = dfixed_mul(dram_channels, yclk); 9020 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 9021 9022 return dfixed_trunc(bandwidth); 9023 } 9024 9025 /** 9026 * dce8_data_return_bandwidth - get the data return bandwidth 9027 * 9028 * @wm: watermark calculation data 9029 * 9030 * Calculate the data return bandwidth used for display (CIK). 9031 * Used for display watermark bandwidth calculations 9032 * Returns the data return bandwidth in MBytes/s 9033 */ 9034 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm) 9035 { 9036 /* Calculate the display Data return Bandwidth */ 9037 fixed20_12 return_efficiency; /* 0.8 */ 9038 fixed20_12 sclk, bandwidth; 9039 fixed20_12 a; 9040 9041 a.full = dfixed_const(1000); 9042 sclk.full = dfixed_const(wm->sclk); 9043 sclk.full = dfixed_div(sclk, a); 9044 a.full = dfixed_const(10); 9045 return_efficiency.full = dfixed_const(8); 9046 return_efficiency.full = dfixed_div(return_efficiency, a); 9047 a.full = dfixed_const(32); 9048 bandwidth.full = dfixed_mul(a, sclk); 9049 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 9050 9051 return dfixed_trunc(bandwidth); 9052 } 9053 9054 /** 9055 * dce8_dmif_request_bandwidth - get the dmif bandwidth 9056 * 9057 * @wm: watermark calculation data 9058 * 9059 * Calculate the dmif bandwidth used for display (CIK). 9060 * Used for display watermark bandwidth calculations 9061 * Returns the dmif bandwidth in MBytes/s 9062 */ 9063 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm) 9064 { 9065 /* Calculate the DMIF Request Bandwidth */ 9066 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 9067 fixed20_12 disp_clk, bandwidth; 9068 fixed20_12 a, b; 9069 9070 a.full = dfixed_const(1000); 9071 disp_clk.full = dfixed_const(wm->disp_clk); 9072 disp_clk.full = dfixed_div(disp_clk, a); 9073 a.full = dfixed_const(32); 9074 b.full = dfixed_mul(a, disp_clk); 9075 9076 a.full = dfixed_const(10); 9077 disp_clk_request_efficiency.full = dfixed_const(8); 9078 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 9079 9080 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency); 9081 9082 return dfixed_trunc(bandwidth); 9083 } 9084 9085 /** 9086 * dce8_available_bandwidth - get the min available bandwidth 9087 * 9088 * @wm: watermark calculation data 9089 * 9090 * Calculate the min available bandwidth used for display (CIK). 9091 * Used for display watermark bandwidth calculations 9092 * Returns the min available bandwidth in MBytes/s 9093 */ 9094 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm) 9095 { 9096 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 9097 u32 dram_bandwidth = dce8_dram_bandwidth(wm); 9098 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm); 9099 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm); 9100 9101 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 9102 } 9103 9104 /** 9105 * dce8_average_bandwidth - get the average available bandwidth 9106 * 9107 * @wm: watermark calculation data 9108 * 9109 * Calculate the average available bandwidth used for display (CIK). 9110 * Used for display watermark bandwidth calculations 9111 * Returns the average available bandwidth in MBytes/s 9112 */ 9113 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm) 9114 { 9115 /* Calculate the display mode Average Bandwidth 9116 * DisplayMode should contain the source and destination dimensions, 9117 * timing, etc. 9118 */ 9119 fixed20_12 bpp; 9120 fixed20_12 line_time; 9121 fixed20_12 src_width; 9122 fixed20_12 bandwidth; 9123 fixed20_12 a; 9124 9125 a.full = dfixed_const(1000); 9126 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 9127 line_time.full = dfixed_div(line_time, a); 9128 bpp.full = dfixed_const(wm->bytes_per_pixel); 9129 src_width.full = dfixed_const(wm->src_width); 9130 bandwidth.full = dfixed_mul(src_width, bpp); 9131 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 9132 bandwidth.full = dfixed_div(bandwidth, line_time); 9133 9134 return dfixed_trunc(bandwidth); 9135 } 9136 9137 /** 9138 * dce8_latency_watermark - get the latency watermark 9139 * 9140 * @wm: watermark calculation data 9141 * 9142 * Calculate the latency watermark (CIK). 9143 * Used for display watermark bandwidth calculations 9144 * Returns the latency watermark in ns 9145 */ 9146 static u32 dce8_latency_watermark(struct dce8_wm_params *wm) 9147 { 9148 /* First calculate the latency in ns */ 9149 u32 mc_latency = 2000; /* 2000 ns. */ 9150 u32 available_bandwidth = dce8_available_bandwidth(wm); 9151 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 9152 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 9153 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 9154 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 9155 (wm->num_heads * cursor_line_pair_return_time); 9156 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 9157 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 9158 u32 tmp, dmif_size = 12288; 9159 fixed20_12 a, b, c; 9160 9161 if (wm->num_heads == 0) 9162 return 0; 9163 9164 a.full = dfixed_const(2); 9165 b.full = dfixed_const(1); 9166 if ((wm->vsc.full > a.full) || 9167 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 9168 (wm->vtaps >= 5) || 9169 ((wm->vsc.full >= a.full) && wm->interlaced)) 9170 max_src_lines_per_dst_line = 4; 9171 else 9172 max_src_lines_per_dst_line = 2; 9173 9174 a.full = dfixed_const(available_bandwidth); 9175 b.full = dfixed_const(wm->num_heads); 9176 a.full = dfixed_div(a, b); 9177 9178 b.full = dfixed_const(mc_latency + 512); 9179 c.full = dfixed_const(wm->disp_clk); 9180 b.full = dfixed_div(b, c); 9181 9182 c.full = dfixed_const(dmif_size); 9183 b.full = dfixed_div(c, b); 9184 9185 tmp = min(dfixed_trunc(a), dfixed_trunc(b)); 9186 9187 b.full = dfixed_const(1000); 9188 c.full = dfixed_const(wm->disp_clk); 9189 b.full = dfixed_div(c, b); 9190 c.full = dfixed_const(wm->bytes_per_pixel); 9191 b.full = dfixed_mul(b, c); 9192 9193 lb_fill_bw = min(tmp, dfixed_trunc(b)); 9194 9195 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 9196 b.full = dfixed_const(1000); 9197 c.full = dfixed_const(lb_fill_bw); 9198 b.full = dfixed_div(c, b); 9199 a.full = dfixed_div(a, b); 9200 line_fill_time = dfixed_trunc(a); 9201 9202 if (line_fill_time < wm->active_time) 9203 return latency; 9204 else 9205 return latency + (line_fill_time - wm->active_time); 9206 9207 } 9208 9209 /** 9210 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check 9211 * average and available dram bandwidth 9212 * 9213 * @wm: watermark calculation data 9214 * 9215 * Check if the display average bandwidth fits in the display 9216 * dram bandwidth (CIK). 9217 * Used for display watermark bandwidth calculations 9218 * Returns true if the display fits, false if not. 9219 */ 9220 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm) 9221 { 9222 if (dce8_average_bandwidth(wm) <= 9223 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads)) 9224 return true; 9225 else 9226 return false; 9227 } 9228 9229 /** 9230 * dce8_average_bandwidth_vs_available_bandwidth - check 9231 * average and available bandwidth 9232 * 9233 * @wm: watermark calculation data 9234 * 9235 * Check if the display average bandwidth fits in the display 9236 * available bandwidth (CIK). 9237 * Used for display watermark bandwidth calculations 9238 * Returns true if the display fits, false if not. 9239 */ 9240 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm) 9241 { 9242 if (dce8_average_bandwidth(wm) <= 9243 (dce8_available_bandwidth(wm) / wm->num_heads)) 9244 return true; 9245 else 9246 return false; 9247 } 9248 9249 /** 9250 * dce8_check_latency_hiding - check latency hiding 9251 * 9252 * @wm: watermark calculation data 9253 * 9254 * Check latency hiding (CIK). 9255 * Used for display watermark bandwidth calculations 9256 * Returns true if the display fits, false if not. 9257 */ 9258 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm) 9259 { 9260 u32 lb_partitions = wm->lb_size / wm->src_width; 9261 u32 line_time = wm->active_time + wm->blank_time; 9262 u32 latency_tolerant_lines; 9263 u32 latency_hiding; 9264 fixed20_12 a; 9265 9266 a.full = dfixed_const(1); 9267 if (wm->vsc.full > a.full) 9268 latency_tolerant_lines = 1; 9269 else { 9270 if (lb_partitions <= (wm->vtaps + 1)) 9271 latency_tolerant_lines = 1; 9272 else 9273 latency_tolerant_lines = 2; 9274 } 9275 9276 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 9277 9278 if (dce8_latency_watermark(wm) <= latency_hiding) 9279 return true; 9280 else 9281 return false; 9282 } 9283 9284 /** 9285 * dce8_program_watermarks - program display watermarks 9286 * 9287 * @rdev: radeon_device pointer 9288 * @radeon_crtc: the selected display controller 9289 * @lb_size: line buffer size 9290 * @num_heads: number of display controllers in use 9291 * 9292 * Calculate and program the display watermarks for the 9293 * selected display controller (CIK). 9294 */ 9295 static void dce8_program_watermarks(struct radeon_device *rdev, 9296 struct radeon_crtc *radeon_crtc, 9297 u32 lb_size, u32 num_heads) 9298 { 9299 struct drm_display_mode *mode = &radeon_crtc->base.mode; 9300 struct dce8_wm_params wm_low, wm_high; 9301 u32 pixel_period; 9302 u32 line_time = 0; 9303 u32 latency_watermark_a = 0, latency_watermark_b = 0; 9304 u32 tmp, wm_mask; 9305 9306 if (radeon_crtc->base.enabled && num_heads && mode) { 9307 pixel_period = 1000000 / (u32)mode->clock; 9308 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 9309 9310 /* watermark for high clocks */ 9311 if ((rdev->pm.pm_method == PM_METHOD_DPM) && 9312 rdev->pm.dpm_enabled) { 9313 wm_high.yclk = 9314 radeon_dpm_get_mclk(rdev, false) * 10; 9315 wm_high.sclk = 9316 radeon_dpm_get_sclk(rdev, false) * 10; 9317 } else { 9318 wm_high.yclk = rdev->pm.current_mclk * 10; 9319 wm_high.sclk = rdev->pm.current_sclk * 10; 9320 } 9321 9322 wm_high.disp_clk = mode->clock; 9323 wm_high.src_width = mode->crtc_hdisplay; 9324 wm_high.active_time = mode->crtc_hdisplay * pixel_period; 9325 wm_high.blank_time = line_time - wm_high.active_time; 9326 wm_high.interlaced = false; 9327 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 9328 wm_high.interlaced = true; 9329 wm_high.vsc = radeon_crtc->vsc; 9330 wm_high.vtaps = 1; 9331 if (radeon_crtc->rmx_type != RMX_OFF) 9332 wm_high.vtaps = 2; 9333 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 9334 wm_high.lb_size = lb_size; 9335 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev); 9336 wm_high.num_heads = num_heads; 9337 9338 /* set for high clocks */ 9339 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535); 9340 9341 /* possibly force display priority to high */ 9342 /* should really do this at mode validation time... */ 9343 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 9344 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) || 9345 !dce8_check_latency_hiding(&wm_high) || 9346 (rdev->disp_priority == 2)) { 9347 DRM_DEBUG_KMS("force priority to high\n"); 9348 } 9349 9350 /* watermark for low clocks */ 9351 if ((rdev->pm.pm_method == PM_METHOD_DPM) && 9352 rdev->pm.dpm_enabled) { 9353 wm_low.yclk = 9354 radeon_dpm_get_mclk(rdev, true) * 10; 9355 wm_low.sclk = 9356 radeon_dpm_get_sclk(rdev, true) * 10; 9357 } else { 9358 wm_low.yclk = rdev->pm.current_mclk * 10; 9359 wm_low.sclk = rdev->pm.current_sclk * 10; 9360 } 9361 9362 wm_low.disp_clk = mode->clock; 9363 wm_low.src_width = mode->crtc_hdisplay; 9364 wm_low.active_time = mode->crtc_hdisplay * pixel_period; 9365 wm_low.blank_time = line_time - wm_low.active_time; 9366 wm_low.interlaced = false; 9367 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 9368 wm_low.interlaced = true; 9369 wm_low.vsc = radeon_crtc->vsc; 9370 wm_low.vtaps = 1; 9371 if (radeon_crtc->rmx_type != RMX_OFF) 9372 wm_low.vtaps = 2; 9373 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 9374 wm_low.lb_size = lb_size; 9375 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev); 9376 wm_low.num_heads = num_heads; 9377 9378 /* set for low clocks */ 9379 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535); 9380 9381 /* possibly force display priority to high */ 9382 /* should really do this at mode validation time... */ 9383 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 9384 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) || 9385 !dce8_check_latency_hiding(&wm_low) || 9386 (rdev->disp_priority == 2)) { 9387 DRM_DEBUG_KMS("force priority to high\n"); 9388 } 9389 } 9390 9391 /* select wm A */ 9392 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); 9393 tmp = wm_mask; 9394 tmp &= ~LATENCY_WATERMARK_MASK(3); 9395 tmp |= LATENCY_WATERMARK_MASK(1); 9396 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); 9397 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 9398 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 9399 LATENCY_HIGH_WATERMARK(line_time))); 9400 /* select wm B */ 9401 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); 9402 tmp &= ~LATENCY_WATERMARK_MASK(3); 9403 tmp |= LATENCY_WATERMARK_MASK(2); 9404 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); 9405 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 9406 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 9407 LATENCY_HIGH_WATERMARK(line_time))); 9408 /* restore original selection */ 9409 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask); 9410 9411 /* save values for DPM */ 9412 radeon_crtc->line_time = line_time; 9413 radeon_crtc->wm_high = latency_watermark_a; 9414 radeon_crtc->wm_low = latency_watermark_b; 9415 } 9416 9417 /** 9418 * dce8_bandwidth_update - program display watermarks 9419 * 9420 * @rdev: radeon_device pointer 9421 * 9422 * Calculate and program the display watermarks and line 9423 * buffer allocation (CIK). 9424 */ 9425 void dce8_bandwidth_update(struct radeon_device *rdev) 9426 { 9427 struct drm_display_mode *mode = NULL; 9428 u32 num_heads = 0, lb_size; 9429 int i; 9430 9431 if (!rdev->mode_info.mode_config_initialized) 9432 return; 9433 9434 radeon_update_display_priority(rdev); 9435 9436 for (i = 0; i < rdev->num_crtc; i++) { 9437 if (rdev->mode_info.crtcs[i]->base.enabled) 9438 num_heads++; 9439 } 9440 for (i = 0; i < rdev->num_crtc; i++) { 9441 mode = &rdev->mode_info.crtcs[i]->base.mode; 9442 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode); 9443 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 9444 } 9445 } 9446 9447 /** 9448 * cik_get_gpu_clock_counter - return GPU clock counter snapshot 9449 * 9450 * @rdev: radeon_device pointer 9451 * 9452 * Fetches a GPU clock counter snapshot (SI). 9453 * Returns the 64 bit clock counter snapshot. 9454 */ 9455 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev) 9456 { 9457 uint64_t clock; 9458 9459 spin_lock(&rdev->gpu_clock_mutex); 9460 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1); 9461 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) | 9462 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 9463 spin_unlock(&rdev->gpu_clock_mutex); 9464 return clock; 9465 } 9466 9467 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock, 9468 u32 cntl_reg, u32 status_reg) 9469 { 9470 int r, i; 9471 struct atom_clock_dividers dividers; 9472 uint32_t tmp; 9473 9474 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, 9475 clock, false, ÷rs); 9476 if (r) 9477 return r; 9478 9479 tmp = RREG32_SMC(cntl_reg); 9480 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK); 9481 tmp |= dividers.post_divider; 9482 WREG32_SMC(cntl_reg, tmp); 9483 9484 for (i = 0; i < 100; i++) { 9485 if (RREG32_SMC(status_reg) & DCLK_STATUS) 9486 break; 9487 mdelay(10); 9488 } 9489 if (i == 100) 9490 return -ETIMEDOUT; 9491 9492 return 0; 9493 } 9494 9495 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 9496 { 9497 int r = 0; 9498 9499 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); 9500 if (r) 9501 return r; 9502 9503 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); 9504 return r; 9505 } 9506 9507 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk) 9508 { 9509 int r, i; 9510 struct atom_clock_dividers dividers; 9511 u32 tmp; 9512 9513 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, 9514 ecclk, false, ÷rs); 9515 if (r) 9516 return r; 9517 9518 for (i = 0; i < 100; i++) { 9519 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) 9520 break; 9521 mdelay(10); 9522 } 9523 if (i == 100) 9524 return -ETIMEDOUT; 9525 9526 tmp = RREG32_SMC(CG_ECLK_CNTL); 9527 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK); 9528 tmp |= dividers.post_divider; 9529 WREG32_SMC(CG_ECLK_CNTL, tmp); 9530 9531 for (i = 0; i < 100; i++) { 9532 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) 9533 break; 9534 mdelay(10); 9535 } 9536 if (i == 100) 9537 return -ETIMEDOUT; 9538 9539 return 0; 9540 } 9541 9542 static void cik_pcie_gen3_enable(struct radeon_device *rdev) 9543 { 9544 struct pci_dev *root = rdev->pdev->bus->self; 9545 int bridge_pos, gpu_pos; 9546 u32 speed_cntl, mask, current_data_rate; 9547 int ret, i; 9548 u16 tmp16; 9549 9550 if (radeon_pcie_gen2 == 0) 9551 return; 9552 9553 if (rdev->flags & RADEON_IS_IGP) 9554 return; 9555 9556 if (!(rdev->flags & RADEON_IS_PCIE)) 9557 return; 9558 9559 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); 9560 if (ret != 0) 9561 return; 9562 9563 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80))) 9564 return; 9565 9566 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9567 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >> 9568 LC_CURRENT_DATA_RATE_SHIFT; 9569 if (mask & DRM_PCIE_SPEED_80) { 9570 if (current_data_rate == 2) { 9571 DRM_INFO("PCIE gen 3 link speeds already enabled\n"); 9572 return; 9573 } 9574 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n"); 9575 } else if (mask & DRM_PCIE_SPEED_50) { 9576 if (current_data_rate == 1) { 9577 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 9578 return; 9579 } 9580 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 9581 } 9582 9583 bridge_pos = pci_get_pciecap_ptr(root->dev); 9584 if (!bridge_pos) 9585 return; 9586 9587 gpu_pos = pci_get_pciecap_ptr(rdev->pdev->dev); 9588 if (!gpu_pos) 9589 return; 9590 9591 if (mask & DRM_PCIE_SPEED_80) { 9592 /* re-try equalization if gen3 is not already enabled */ 9593 if (current_data_rate != 2) { 9594 u16 bridge_cfg, gpu_cfg; 9595 u16 bridge_cfg2, gpu_cfg2; 9596 u32 max_lw, current_lw, tmp; 9597 9598 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 9599 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 9600 9601 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD; 9602 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 9603 9604 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD; 9605 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 9606 9607 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1); 9608 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT; 9609 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT; 9610 9611 if (current_lw < max_lw) { 9612 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 9613 if (tmp & LC_RENEGOTIATION_SUPPORT) { 9614 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS); 9615 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT); 9616 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW; 9617 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp); 9618 } 9619 } 9620 9621 for (i = 0; i < 10; i++) { 9622 /* check status */ 9623 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); 9624 if (tmp16 & PCI_EXP_DEVSTA_TRPND) 9625 break; 9626 9627 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 9628 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 9629 9630 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2); 9631 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); 9632 9633 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9634 tmp |= LC_SET_QUIESCE; 9635 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9636 9637 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9638 tmp |= LC_REDO_EQ; 9639 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9640 9641 mdelay(100); 9642 9643 /* linkctl */ 9644 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16); 9645 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 9646 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD); 9647 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 9648 9649 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); 9650 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 9651 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD); 9652 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 9653 9654 /* linkctl2 */ 9655 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16); 9656 tmp16 &= ~((1 << 4) | (7 << 9)); 9657 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9))); 9658 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16); 9659 9660 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 9661 tmp16 &= ~((1 << 4) | (7 << 9)); 9662 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9))); 9663 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 9664 9665 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9666 tmp &= ~LC_SET_QUIESCE; 9667 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9668 } 9669 } 9670 } 9671 9672 /* set the link speed */ 9673 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE; 9674 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE; 9675 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 9676 9677 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 9678 tmp16 &= ~0xf; 9679 if (mask & DRM_PCIE_SPEED_80) 9680 tmp16 |= 3; /* gen3 */ 9681 else if (mask & DRM_PCIE_SPEED_50) 9682 tmp16 |= 2; /* gen2 */ 9683 else 9684 tmp16 |= 1; /* gen1 */ 9685 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 9686 9687 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9688 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE; 9689 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 9690 9691 for (i = 0; i < rdev->usec_timeout; i++) { 9692 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9693 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0) 9694 break; 9695 udelay(1); 9696 } 9697 } 9698 9699 static void cik_program_aspm(struct radeon_device *rdev) 9700 { 9701 u32 data, orig; 9702 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false; 9703 bool disable_clkreq = false; 9704 9705 if (radeon_aspm == 0) 9706 return; 9707 9708 /* XXX double check IGPs */ 9709 if (rdev->flags & RADEON_IS_IGP) 9710 return; 9711 9712 if (!(rdev->flags & RADEON_IS_PCIE)) 9713 return; 9714 9715 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 9716 data &= ~LC_XMIT_N_FTS_MASK; 9717 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN; 9718 if (orig != data) 9719 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data); 9720 9721 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3); 9722 data |= LC_GO_TO_RECOVERY; 9723 if (orig != data) 9724 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data); 9725 9726 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL); 9727 data |= P_IGNORE_EDB_ERR; 9728 if (orig != data) 9729 WREG32_PCIE_PORT(PCIE_P_CNTL, data); 9730 9731 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 9732 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 9733 data |= LC_PMI_TO_L1_DIS; 9734 if (!disable_l0s) 9735 data |= LC_L0S_INACTIVITY(7); 9736 9737 if (!disable_l1) { 9738 data |= LC_L1_INACTIVITY(7); 9739 data &= ~LC_PMI_TO_L1_DIS; 9740 if (orig != data) 9741 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9742 9743 if (!disable_plloff_in_l1) { 9744 bool clk_req_support; 9745 9746 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0); 9747 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 9748 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 9749 if (orig != data) 9750 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data); 9751 9752 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1); 9753 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 9754 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 9755 if (orig != data) 9756 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data); 9757 9758 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0); 9759 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 9760 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 9761 if (orig != data) 9762 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data); 9763 9764 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1); 9765 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 9766 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 9767 if (orig != data) 9768 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data); 9769 9770 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 9771 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 9772 data |= LC_DYN_LANES_PWR_STATE(3); 9773 if (orig != data) 9774 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 9775 9776 if (!disable_clkreq) { 9777 #ifdef zMN_TODO 9778 struct pci_dev *root = rdev->pdev->bus->self; 9779 u32 lnkcap; 9780 9781 clk_req_support = false; 9782 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap); 9783 if (lnkcap & PCI_EXP_LNKCAP_CLKPM) 9784 clk_req_support = true; 9785 #else 9786 clk_req_support = false; 9787 #endif 9788 } else { 9789 clk_req_support = false; 9790 } 9791 9792 if (clk_req_support) { 9793 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2); 9794 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23; 9795 if (orig != data) 9796 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data); 9797 9798 orig = data = RREG32_SMC(THM_CLK_CNTL); 9799 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK); 9800 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1); 9801 if (orig != data) 9802 WREG32_SMC(THM_CLK_CNTL, data); 9803 9804 orig = data = RREG32_SMC(MISC_CLK_CTRL); 9805 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK); 9806 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1); 9807 if (orig != data) 9808 WREG32_SMC(MISC_CLK_CTRL, data); 9809 9810 orig = data = RREG32_SMC(CG_CLKPIN_CNTL); 9811 data &= ~BCLK_AS_XCLK; 9812 if (orig != data) 9813 WREG32_SMC(CG_CLKPIN_CNTL, data); 9814 9815 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2); 9816 data &= ~FORCE_BIF_REFCLK_EN; 9817 if (orig != data) 9818 WREG32_SMC(CG_CLKPIN_CNTL_2, data); 9819 9820 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL); 9821 data &= ~MPLL_CLKOUT_SEL_MASK; 9822 data |= MPLL_CLKOUT_SEL(4); 9823 if (orig != data) 9824 WREG32_SMC(MPLL_BYPASSCLK_SEL, data); 9825 } 9826 } 9827 } else { 9828 if (orig != data) 9829 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9830 } 9831 9832 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); 9833 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN; 9834 if (orig != data) 9835 WREG32_PCIE_PORT(PCIE_CNTL2, data); 9836 9837 if (!disable_l0s) { 9838 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 9839 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) { 9840 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1); 9841 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) { 9842 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 9843 data &= ~LC_L0S_INACTIVITY_MASK; 9844 if (orig != data) 9845 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9846 } 9847 } 9848 } 9849 } 9850