1 /* 2 * Copyright 2010 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <drm/drmP.h> 26 #include "radeon.h" 27 #include "radeon_asic.h" 28 #include "radeon_audio.h" 29 #include <uapi_drm/radeon_drm.h> 30 #include "evergreend.h" 31 #include "atom.h" 32 #include "avivod.h" 33 #include "evergreen_reg.h" 34 #include "evergreen_blit_shaders.h" 35 #include "radeon_ucode.h" 36 37 /* 38 * Indirect registers accessor 39 */ 40 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg) 41 { 42 unsigned long flags; 43 u32 r; 44 45 spin_lock_irqsave(&rdev->cg_idx_lock, flags); 46 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff)); 47 r = RREG32(EVERGREEN_CG_IND_DATA); 48 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags); 49 return r; 50 } 51 52 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v) 53 { 54 unsigned long flags; 55 56 spin_lock_irqsave(&rdev->cg_idx_lock, flags); 57 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff)); 58 WREG32(EVERGREEN_CG_IND_DATA, (v)); 59 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags); 60 } 61 62 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg) 63 { 64 unsigned long flags; 65 u32 r; 66 67 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 68 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff)); 69 r = RREG32(EVERGREEN_PIF_PHY0_DATA); 70 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 71 return r; 72 } 73 74 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v) 75 { 76 unsigned long flags; 77 78 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 79 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff)); 80 WREG32(EVERGREEN_PIF_PHY0_DATA, (v)); 81 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 82 } 83 84 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg) 85 { 86 unsigned long flags; 87 u32 r; 88 89 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 90 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff)); 91 r = RREG32(EVERGREEN_PIF_PHY1_DATA); 92 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 93 return r; 94 } 95 96 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v) 97 { 98 unsigned long flags; 99 100 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 101 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff)); 102 WREG32(EVERGREEN_PIF_PHY1_DATA, (v)); 103 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 104 } 105 106 static const u32 crtc_offsets[6] = 107 { 108 EVERGREEN_CRTC0_REGISTER_OFFSET, 109 EVERGREEN_CRTC1_REGISTER_OFFSET, 110 EVERGREEN_CRTC2_REGISTER_OFFSET, 111 EVERGREEN_CRTC3_REGISTER_OFFSET, 112 EVERGREEN_CRTC4_REGISTER_OFFSET, 113 EVERGREEN_CRTC5_REGISTER_OFFSET 114 }; 115 116 #include "clearstate_evergreen.h" 117 118 static const u32 sumo_rlc_save_restore_register_list[] = 119 { 120 0x98fc, 121 0x9830, 122 0x9834, 123 0x9838, 124 0x9870, 125 0x9874, 126 0x8a14, 127 0x8b24, 128 0x8bcc, 129 0x8b10, 130 0x8d00, 131 0x8d04, 132 0x8c00, 133 0x8c04, 134 0x8c08, 135 0x8c0c, 136 0x8d8c, 137 0x8c20, 138 0x8c24, 139 0x8c28, 140 0x8c18, 141 0x8c1c, 142 0x8cf0, 143 0x8e2c, 144 0x8e38, 145 0x8c30, 146 0x9508, 147 0x9688, 148 0x9608, 149 0x960c, 150 0x9610, 151 0x9614, 152 0x88c4, 153 0x88d4, 154 0xa008, 155 0x900c, 156 0x9100, 157 0x913c, 158 0x98f8, 159 0x98f4, 160 0x9b7c, 161 0x3f8c, 162 0x8950, 163 0x8954, 164 0x8a18, 165 0x8b28, 166 0x9144, 167 0x9148, 168 0x914c, 169 0x3f90, 170 0x3f94, 171 0x915c, 172 0x9160, 173 0x9178, 174 0x917c, 175 0x9180, 176 0x918c, 177 0x9190, 178 0x9194, 179 0x9198, 180 0x919c, 181 0x91a8, 182 0x91ac, 183 0x91b0, 184 0x91b4, 185 0x91b8, 186 0x91c4, 187 0x91c8, 188 0x91cc, 189 0x91d0, 190 0x91d4, 191 0x91e0, 192 0x91e4, 193 0x91ec, 194 0x91f0, 195 0x91f4, 196 0x9200, 197 0x9204, 198 0x929c, 199 0x9150, 200 0x802c, 201 }; 202 203 static void evergreen_gpu_init(struct radeon_device *rdev); 204 205 static const u32 evergreen_golden_registers[] = 206 { 207 0x3f90, 0xffff0000, 0xff000000, 208 0x9148, 0xffff0000, 0xff000000, 209 0x3f94, 0xffff0000, 0xff000000, 210 0x914c, 0xffff0000, 0xff000000, 211 0x9b7c, 0xffffffff, 0x00000000, 212 0x8a14, 0xffffffff, 0x00000007, 213 0x8b10, 0xffffffff, 0x00000000, 214 0x960c, 0xffffffff, 0x54763210, 215 0x88c4, 0xffffffff, 0x000000c2, 216 0x88d4, 0xffffffff, 0x00000010, 217 0x8974, 0xffffffff, 0x00000000, 218 0xc78, 0x00000080, 0x00000080, 219 0x5eb4, 0xffffffff, 0x00000002, 220 0x5e78, 0xffffffff, 0x001000f0, 221 0x6104, 0x01000300, 0x00000000, 222 0x5bc0, 0x00300000, 0x00000000, 223 0x7030, 0xffffffff, 0x00000011, 224 0x7c30, 0xffffffff, 0x00000011, 225 0x10830, 0xffffffff, 0x00000011, 226 0x11430, 0xffffffff, 0x00000011, 227 0x12030, 0xffffffff, 0x00000011, 228 0x12c30, 0xffffffff, 0x00000011, 229 0xd02c, 0xffffffff, 0x08421000, 230 0x240c, 0xffffffff, 0x00000380, 231 0x8b24, 0xffffffff, 0x00ff0fff, 232 0x28a4c, 0x06000000, 0x06000000, 233 0x10c, 0x00000001, 0x00000001, 234 0x8d00, 0xffffffff, 0x100e4848, 235 0x8d04, 0xffffffff, 0x00164745, 236 0x8c00, 0xffffffff, 0xe4000003, 237 0x8c04, 0xffffffff, 0x40600060, 238 0x8c08, 0xffffffff, 0x001c001c, 239 0x8cf0, 0xffffffff, 0x08e00620, 240 0x8c20, 0xffffffff, 0x00800080, 241 0x8c24, 0xffffffff, 0x00800080, 242 0x8c18, 0xffffffff, 0x20202078, 243 0x8c1c, 0xffffffff, 0x00001010, 244 0x28350, 0xffffffff, 0x00000000, 245 0xa008, 0xffffffff, 0x00010000, 246 0x5c4, 0xffffffff, 0x00000001, 247 0x9508, 0xffffffff, 0x00000002, 248 0x913c, 0x0000000f, 0x0000000a 249 }; 250 251 static const u32 evergreen_golden_registers2[] = 252 { 253 0x2f4c, 0xffffffff, 0x00000000, 254 0x54f4, 0xffffffff, 0x00000000, 255 0x54f0, 0xffffffff, 0x00000000, 256 0x5498, 0xffffffff, 0x00000000, 257 0x549c, 0xffffffff, 0x00000000, 258 0x5494, 0xffffffff, 0x00000000, 259 0x53cc, 0xffffffff, 0x00000000, 260 0x53c8, 0xffffffff, 0x00000000, 261 0x53c4, 0xffffffff, 0x00000000, 262 0x53c0, 0xffffffff, 0x00000000, 263 0x53bc, 0xffffffff, 0x00000000, 264 0x53b8, 0xffffffff, 0x00000000, 265 0x53b4, 0xffffffff, 0x00000000, 266 0x53b0, 0xffffffff, 0x00000000 267 }; 268 269 static const u32 cypress_mgcg_init[] = 270 { 271 0x802c, 0xffffffff, 0xc0000000, 272 0x5448, 0xffffffff, 0x00000100, 273 0x55e4, 0xffffffff, 0x00000100, 274 0x160c, 0xffffffff, 0x00000100, 275 0x5644, 0xffffffff, 0x00000100, 276 0xc164, 0xffffffff, 0x00000100, 277 0x8a18, 0xffffffff, 0x00000100, 278 0x897c, 0xffffffff, 0x06000100, 279 0x8b28, 0xffffffff, 0x00000100, 280 0x9144, 0xffffffff, 0x00000100, 281 0x9a60, 0xffffffff, 0x00000100, 282 0x9868, 0xffffffff, 0x00000100, 283 0x8d58, 0xffffffff, 0x00000100, 284 0x9510, 0xffffffff, 0x00000100, 285 0x949c, 0xffffffff, 0x00000100, 286 0x9654, 0xffffffff, 0x00000100, 287 0x9030, 0xffffffff, 0x00000100, 288 0x9034, 0xffffffff, 0x00000100, 289 0x9038, 0xffffffff, 0x00000100, 290 0x903c, 0xffffffff, 0x00000100, 291 0x9040, 0xffffffff, 0x00000100, 292 0xa200, 0xffffffff, 0x00000100, 293 0xa204, 0xffffffff, 0x00000100, 294 0xa208, 0xffffffff, 0x00000100, 295 0xa20c, 0xffffffff, 0x00000100, 296 0x971c, 0xffffffff, 0x00000100, 297 0x977c, 0xffffffff, 0x00000100, 298 0x3f80, 0xffffffff, 0x00000100, 299 0xa210, 0xffffffff, 0x00000100, 300 0xa214, 0xffffffff, 0x00000100, 301 0x4d8, 0xffffffff, 0x00000100, 302 0x9784, 0xffffffff, 0x00000100, 303 0x9698, 0xffffffff, 0x00000100, 304 0x4d4, 0xffffffff, 0x00000200, 305 0x30cc, 0xffffffff, 0x00000100, 306 0xd0c0, 0xffffffff, 0xff000100, 307 0x802c, 0xffffffff, 0x40000000, 308 0x915c, 0xffffffff, 0x00010000, 309 0x9160, 0xffffffff, 0x00030002, 310 0x9178, 0xffffffff, 0x00070000, 311 0x917c, 0xffffffff, 0x00030002, 312 0x9180, 0xffffffff, 0x00050004, 313 0x918c, 0xffffffff, 0x00010006, 314 0x9190, 0xffffffff, 0x00090008, 315 0x9194, 0xffffffff, 0x00070000, 316 0x9198, 0xffffffff, 0x00030002, 317 0x919c, 0xffffffff, 0x00050004, 318 0x91a8, 0xffffffff, 0x00010006, 319 0x91ac, 0xffffffff, 0x00090008, 320 0x91b0, 0xffffffff, 0x00070000, 321 0x91b4, 0xffffffff, 0x00030002, 322 0x91b8, 0xffffffff, 0x00050004, 323 0x91c4, 0xffffffff, 0x00010006, 324 0x91c8, 0xffffffff, 0x00090008, 325 0x91cc, 0xffffffff, 0x00070000, 326 0x91d0, 0xffffffff, 0x00030002, 327 0x91d4, 0xffffffff, 0x00050004, 328 0x91e0, 0xffffffff, 0x00010006, 329 0x91e4, 0xffffffff, 0x00090008, 330 0x91e8, 0xffffffff, 0x00000000, 331 0x91ec, 0xffffffff, 0x00070000, 332 0x91f0, 0xffffffff, 0x00030002, 333 0x91f4, 0xffffffff, 0x00050004, 334 0x9200, 0xffffffff, 0x00010006, 335 0x9204, 0xffffffff, 0x00090008, 336 0x9208, 0xffffffff, 0x00070000, 337 0x920c, 0xffffffff, 0x00030002, 338 0x9210, 0xffffffff, 0x00050004, 339 0x921c, 0xffffffff, 0x00010006, 340 0x9220, 0xffffffff, 0x00090008, 341 0x9224, 0xffffffff, 0x00070000, 342 0x9228, 0xffffffff, 0x00030002, 343 0x922c, 0xffffffff, 0x00050004, 344 0x9238, 0xffffffff, 0x00010006, 345 0x923c, 0xffffffff, 0x00090008, 346 0x9240, 0xffffffff, 0x00070000, 347 0x9244, 0xffffffff, 0x00030002, 348 0x9248, 0xffffffff, 0x00050004, 349 0x9254, 0xffffffff, 0x00010006, 350 0x9258, 0xffffffff, 0x00090008, 351 0x925c, 0xffffffff, 0x00070000, 352 0x9260, 0xffffffff, 0x00030002, 353 0x9264, 0xffffffff, 0x00050004, 354 0x9270, 0xffffffff, 0x00010006, 355 0x9274, 0xffffffff, 0x00090008, 356 0x9278, 0xffffffff, 0x00070000, 357 0x927c, 0xffffffff, 0x00030002, 358 0x9280, 0xffffffff, 0x00050004, 359 0x928c, 0xffffffff, 0x00010006, 360 0x9290, 0xffffffff, 0x00090008, 361 0x9294, 0xffffffff, 0x00000000, 362 0x929c, 0xffffffff, 0x00000001, 363 0x802c, 0xffffffff, 0x40010000, 364 0x915c, 0xffffffff, 0x00010000, 365 0x9160, 0xffffffff, 0x00030002, 366 0x9178, 0xffffffff, 0x00070000, 367 0x917c, 0xffffffff, 0x00030002, 368 0x9180, 0xffffffff, 0x00050004, 369 0x918c, 0xffffffff, 0x00010006, 370 0x9190, 0xffffffff, 0x00090008, 371 0x9194, 0xffffffff, 0x00070000, 372 0x9198, 0xffffffff, 0x00030002, 373 0x919c, 0xffffffff, 0x00050004, 374 0x91a8, 0xffffffff, 0x00010006, 375 0x91ac, 0xffffffff, 0x00090008, 376 0x91b0, 0xffffffff, 0x00070000, 377 0x91b4, 0xffffffff, 0x00030002, 378 0x91b8, 0xffffffff, 0x00050004, 379 0x91c4, 0xffffffff, 0x00010006, 380 0x91c8, 0xffffffff, 0x00090008, 381 0x91cc, 0xffffffff, 0x00070000, 382 0x91d0, 0xffffffff, 0x00030002, 383 0x91d4, 0xffffffff, 0x00050004, 384 0x91e0, 0xffffffff, 0x00010006, 385 0x91e4, 0xffffffff, 0x00090008, 386 0x91e8, 0xffffffff, 0x00000000, 387 0x91ec, 0xffffffff, 0x00070000, 388 0x91f0, 0xffffffff, 0x00030002, 389 0x91f4, 0xffffffff, 0x00050004, 390 0x9200, 0xffffffff, 0x00010006, 391 0x9204, 0xffffffff, 0x00090008, 392 0x9208, 0xffffffff, 0x00070000, 393 0x920c, 0xffffffff, 0x00030002, 394 0x9210, 0xffffffff, 0x00050004, 395 0x921c, 0xffffffff, 0x00010006, 396 0x9220, 0xffffffff, 0x00090008, 397 0x9224, 0xffffffff, 0x00070000, 398 0x9228, 0xffffffff, 0x00030002, 399 0x922c, 0xffffffff, 0x00050004, 400 0x9238, 0xffffffff, 0x00010006, 401 0x923c, 0xffffffff, 0x00090008, 402 0x9240, 0xffffffff, 0x00070000, 403 0x9244, 0xffffffff, 0x00030002, 404 0x9248, 0xffffffff, 0x00050004, 405 0x9254, 0xffffffff, 0x00010006, 406 0x9258, 0xffffffff, 0x00090008, 407 0x925c, 0xffffffff, 0x00070000, 408 0x9260, 0xffffffff, 0x00030002, 409 0x9264, 0xffffffff, 0x00050004, 410 0x9270, 0xffffffff, 0x00010006, 411 0x9274, 0xffffffff, 0x00090008, 412 0x9278, 0xffffffff, 0x00070000, 413 0x927c, 0xffffffff, 0x00030002, 414 0x9280, 0xffffffff, 0x00050004, 415 0x928c, 0xffffffff, 0x00010006, 416 0x9290, 0xffffffff, 0x00090008, 417 0x9294, 0xffffffff, 0x00000000, 418 0x929c, 0xffffffff, 0x00000001, 419 0x802c, 0xffffffff, 0xc0000000 420 }; 421 422 static const u32 redwood_mgcg_init[] = 423 { 424 0x802c, 0xffffffff, 0xc0000000, 425 0x5448, 0xffffffff, 0x00000100, 426 0x55e4, 0xffffffff, 0x00000100, 427 0x160c, 0xffffffff, 0x00000100, 428 0x5644, 0xffffffff, 0x00000100, 429 0xc164, 0xffffffff, 0x00000100, 430 0x8a18, 0xffffffff, 0x00000100, 431 0x897c, 0xffffffff, 0x06000100, 432 0x8b28, 0xffffffff, 0x00000100, 433 0x9144, 0xffffffff, 0x00000100, 434 0x9a60, 0xffffffff, 0x00000100, 435 0x9868, 0xffffffff, 0x00000100, 436 0x8d58, 0xffffffff, 0x00000100, 437 0x9510, 0xffffffff, 0x00000100, 438 0x949c, 0xffffffff, 0x00000100, 439 0x9654, 0xffffffff, 0x00000100, 440 0x9030, 0xffffffff, 0x00000100, 441 0x9034, 0xffffffff, 0x00000100, 442 0x9038, 0xffffffff, 0x00000100, 443 0x903c, 0xffffffff, 0x00000100, 444 0x9040, 0xffffffff, 0x00000100, 445 0xa200, 0xffffffff, 0x00000100, 446 0xa204, 0xffffffff, 0x00000100, 447 0xa208, 0xffffffff, 0x00000100, 448 0xa20c, 0xffffffff, 0x00000100, 449 0x971c, 0xffffffff, 0x00000100, 450 0x977c, 0xffffffff, 0x00000100, 451 0x3f80, 0xffffffff, 0x00000100, 452 0xa210, 0xffffffff, 0x00000100, 453 0xa214, 0xffffffff, 0x00000100, 454 0x4d8, 0xffffffff, 0x00000100, 455 0x9784, 0xffffffff, 0x00000100, 456 0x9698, 0xffffffff, 0x00000100, 457 0x4d4, 0xffffffff, 0x00000200, 458 0x30cc, 0xffffffff, 0x00000100, 459 0xd0c0, 0xffffffff, 0xff000100, 460 0x802c, 0xffffffff, 0x40000000, 461 0x915c, 0xffffffff, 0x00010000, 462 0x9160, 0xffffffff, 0x00030002, 463 0x9178, 0xffffffff, 0x00070000, 464 0x917c, 0xffffffff, 0x00030002, 465 0x9180, 0xffffffff, 0x00050004, 466 0x918c, 0xffffffff, 0x00010006, 467 0x9190, 0xffffffff, 0x00090008, 468 0x9194, 0xffffffff, 0x00070000, 469 0x9198, 0xffffffff, 0x00030002, 470 0x919c, 0xffffffff, 0x00050004, 471 0x91a8, 0xffffffff, 0x00010006, 472 0x91ac, 0xffffffff, 0x00090008, 473 0x91b0, 0xffffffff, 0x00070000, 474 0x91b4, 0xffffffff, 0x00030002, 475 0x91b8, 0xffffffff, 0x00050004, 476 0x91c4, 0xffffffff, 0x00010006, 477 0x91c8, 0xffffffff, 0x00090008, 478 0x91cc, 0xffffffff, 0x00070000, 479 0x91d0, 0xffffffff, 0x00030002, 480 0x91d4, 0xffffffff, 0x00050004, 481 0x91e0, 0xffffffff, 0x00010006, 482 0x91e4, 0xffffffff, 0x00090008, 483 0x91e8, 0xffffffff, 0x00000000, 484 0x91ec, 0xffffffff, 0x00070000, 485 0x91f0, 0xffffffff, 0x00030002, 486 0x91f4, 0xffffffff, 0x00050004, 487 0x9200, 0xffffffff, 0x00010006, 488 0x9204, 0xffffffff, 0x00090008, 489 0x9294, 0xffffffff, 0x00000000, 490 0x929c, 0xffffffff, 0x00000001, 491 0x802c, 0xffffffff, 0xc0000000 492 }; 493 494 static const u32 cedar_golden_registers[] = 495 { 496 0x3f90, 0xffff0000, 0xff000000, 497 0x9148, 0xffff0000, 0xff000000, 498 0x3f94, 0xffff0000, 0xff000000, 499 0x914c, 0xffff0000, 0xff000000, 500 0x9b7c, 0xffffffff, 0x00000000, 501 0x8a14, 0xffffffff, 0x00000007, 502 0x8b10, 0xffffffff, 0x00000000, 503 0x960c, 0xffffffff, 0x54763210, 504 0x88c4, 0xffffffff, 0x000000c2, 505 0x88d4, 0xffffffff, 0x00000000, 506 0x8974, 0xffffffff, 0x00000000, 507 0xc78, 0x00000080, 0x00000080, 508 0x5eb4, 0xffffffff, 0x00000002, 509 0x5e78, 0xffffffff, 0x001000f0, 510 0x6104, 0x01000300, 0x00000000, 511 0x5bc0, 0x00300000, 0x00000000, 512 0x7030, 0xffffffff, 0x00000011, 513 0x7c30, 0xffffffff, 0x00000011, 514 0x10830, 0xffffffff, 0x00000011, 515 0x11430, 0xffffffff, 0x00000011, 516 0xd02c, 0xffffffff, 0x08421000, 517 0x240c, 0xffffffff, 0x00000380, 518 0x8b24, 0xffffffff, 0x00ff0fff, 519 0x28a4c, 0x06000000, 0x06000000, 520 0x10c, 0x00000001, 0x00000001, 521 0x8d00, 0xffffffff, 0x100e4848, 522 0x8d04, 0xffffffff, 0x00164745, 523 0x8c00, 0xffffffff, 0xe4000003, 524 0x8c04, 0xffffffff, 0x40600060, 525 0x8c08, 0xffffffff, 0x001c001c, 526 0x8cf0, 0xffffffff, 0x08e00410, 527 0x8c20, 0xffffffff, 0x00800080, 528 0x8c24, 0xffffffff, 0x00800080, 529 0x8c18, 0xffffffff, 0x20202078, 530 0x8c1c, 0xffffffff, 0x00001010, 531 0x28350, 0xffffffff, 0x00000000, 532 0xa008, 0xffffffff, 0x00010000, 533 0x5c4, 0xffffffff, 0x00000001, 534 0x9508, 0xffffffff, 0x00000002 535 }; 536 537 static const u32 cedar_mgcg_init[] = 538 { 539 0x802c, 0xffffffff, 0xc0000000, 540 0x5448, 0xffffffff, 0x00000100, 541 0x55e4, 0xffffffff, 0x00000100, 542 0x160c, 0xffffffff, 0x00000100, 543 0x5644, 0xffffffff, 0x00000100, 544 0xc164, 0xffffffff, 0x00000100, 545 0x8a18, 0xffffffff, 0x00000100, 546 0x897c, 0xffffffff, 0x06000100, 547 0x8b28, 0xffffffff, 0x00000100, 548 0x9144, 0xffffffff, 0x00000100, 549 0x9a60, 0xffffffff, 0x00000100, 550 0x9868, 0xffffffff, 0x00000100, 551 0x8d58, 0xffffffff, 0x00000100, 552 0x9510, 0xffffffff, 0x00000100, 553 0x949c, 0xffffffff, 0x00000100, 554 0x9654, 0xffffffff, 0x00000100, 555 0x9030, 0xffffffff, 0x00000100, 556 0x9034, 0xffffffff, 0x00000100, 557 0x9038, 0xffffffff, 0x00000100, 558 0x903c, 0xffffffff, 0x00000100, 559 0x9040, 0xffffffff, 0x00000100, 560 0xa200, 0xffffffff, 0x00000100, 561 0xa204, 0xffffffff, 0x00000100, 562 0xa208, 0xffffffff, 0x00000100, 563 0xa20c, 0xffffffff, 0x00000100, 564 0x971c, 0xffffffff, 0x00000100, 565 0x977c, 0xffffffff, 0x00000100, 566 0x3f80, 0xffffffff, 0x00000100, 567 0xa210, 0xffffffff, 0x00000100, 568 0xa214, 0xffffffff, 0x00000100, 569 0x4d8, 0xffffffff, 0x00000100, 570 0x9784, 0xffffffff, 0x00000100, 571 0x9698, 0xffffffff, 0x00000100, 572 0x4d4, 0xffffffff, 0x00000200, 573 0x30cc, 0xffffffff, 0x00000100, 574 0xd0c0, 0xffffffff, 0xff000100, 575 0x802c, 0xffffffff, 0x40000000, 576 0x915c, 0xffffffff, 0x00010000, 577 0x9178, 0xffffffff, 0x00050000, 578 0x917c, 0xffffffff, 0x00030002, 579 0x918c, 0xffffffff, 0x00010004, 580 0x9190, 0xffffffff, 0x00070006, 581 0x9194, 0xffffffff, 0x00050000, 582 0x9198, 0xffffffff, 0x00030002, 583 0x91a8, 0xffffffff, 0x00010004, 584 0x91ac, 0xffffffff, 0x00070006, 585 0x91e8, 0xffffffff, 0x00000000, 586 0x9294, 0xffffffff, 0x00000000, 587 0x929c, 0xffffffff, 0x00000001, 588 0x802c, 0xffffffff, 0xc0000000 589 }; 590 591 static const u32 juniper_mgcg_init[] = 592 { 593 0x802c, 0xffffffff, 0xc0000000, 594 0x5448, 0xffffffff, 0x00000100, 595 0x55e4, 0xffffffff, 0x00000100, 596 0x160c, 0xffffffff, 0x00000100, 597 0x5644, 0xffffffff, 0x00000100, 598 0xc164, 0xffffffff, 0x00000100, 599 0x8a18, 0xffffffff, 0x00000100, 600 0x897c, 0xffffffff, 0x06000100, 601 0x8b28, 0xffffffff, 0x00000100, 602 0x9144, 0xffffffff, 0x00000100, 603 0x9a60, 0xffffffff, 0x00000100, 604 0x9868, 0xffffffff, 0x00000100, 605 0x8d58, 0xffffffff, 0x00000100, 606 0x9510, 0xffffffff, 0x00000100, 607 0x949c, 0xffffffff, 0x00000100, 608 0x9654, 0xffffffff, 0x00000100, 609 0x9030, 0xffffffff, 0x00000100, 610 0x9034, 0xffffffff, 0x00000100, 611 0x9038, 0xffffffff, 0x00000100, 612 0x903c, 0xffffffff, 0x00000100, 613 0x9040, 0xffffffff, 0x00000100, 614 0xa200, 0xffffffff, 0x00000100, 615 0xa204, 0xffffffff, 0x00000100, 616 0xa208, 0xffffffff, 0x00000100, 617 0xa20c, 0xffffffff, 0x00000100, 618 0x971c, 0xffffffff, 0x00000100, 619 0xd0c0, 0xffffffff, 0xff000100, 620 0x802c, 0xffffffff, 0x40000000, 621 0x915c, 0xffffffff, 0x00010000, 622 0x9160, 0xffffffff, 0x00030002, 623 0x9178, 0xffffffff, 0x00070000, 624 0x917c, 0xffffffff, 0x00030002, 625 0x9180, 0xffffffff, 0x00050004, 626 0x918c, 0xffffffff, 0x00010006, 627 0x9190, 0xffffffff, 0x00090008, 628 0x9194, 0xffffffff, 0x00070000, 629 0x9198, 0xffffffff, 0x00030002, 630 0x919c, 0xffffffff, 0x00050004, 631 0x91a8, 0xffffffff, 0x00010006, 632 0x91ac, 0xffffffff, 0x00090008, 633 0x91b0, 0xffffffff, 0x00070000, 634 0x91b4, 0xffffffff, 0x00030002, 635 0x91b8, 0xffffffff, 0x00050004, 636 0x91c4, 0xffffffff, 0x00010006, 637 0x91c8, 0xffffffff, 0x00090008, 638 0x91cc, 0xffffffff, 0x00070000, 639 0x91d0, 0xffffffff, 0x00030002, 640 0x91d4, 0xffffffff, 0x00050004, 641 0x91e0, 0xffffffff, 0x00010006, 642 0x91e4, 0xffffffff, 0x00090008, 643 0x91e8, 0xffffffff, 0x00000000, 644 0x91ec, 0xffffffff, 0x00070000, 645 0x91f0, 0xffffffff, 0x00030002, 646 0x91f4, 0xffffffff, 0x00050004, 647 0x9200, 0xffffffff, 0x00010006, 648 0x9204, 0xffffffff, 0x00090008, 649 0x9208, 0xffffffff, 0x00070000, 650 0x920c, 0xffffffff, 0x00030002, 651 0x9210, 0xffffffff, 0x00050004, 652 0x921c, 0xffffffff, 0x00010006, 653 0x9220, 0xffffffff, 0x00090008, 654 0x9224, 0xffffffff, 0x00070000, 655 0x9228, 0xffffffff, 0x00030002, 656 0x922c, 0xffffffff, 0x00050004, 657 0x9238, 0xffffffff, 0x00010006, 658 0x923c, 0xffffffff, 0x00090008, 659 0x9240, 0xffffffff, 0x00070000, 660 0x9244, 0xffffffff, 0x00030002, 661 0x9248, 0xffffffff, 0x00050004, 662 0x9254, 0xffffffff, 0x00010006, 663 0x9258, 0xffffffff, 0x00090008, 664 0x925c, 0xffffffff, 0x00070000, 665 0x9260, 0xffffffff, 0x00030002, 666 0x9264, 0xffffffff, 0x00050004, 667 0x9270, 0xffffffff, 0x00010006, 668 0x9274, 0xffffffff, 0x00090008, 669 0x9278, 0xffffffff, 0x00070000, 670 0x927c, 0xffffffff, 0x00030002, 671 0x9280, 0xffffffff, 0x00050004, 672 0x928c, 0xffffffff, 0x00010006, 673 0x9290, 0xffffffff, 0x00090008, 674 0x9294, 0xffffffff, 0x00000000, 675 0x929c, 0xffffffff, 0x00000001, 676 0x802c, 0xffffffff, 0xc0000000, 677 0x977c, 0xffffffff, 0x00000100, 678 0x3f80, 0xffffffff, 0x00000100, 679 0xa210, 0xffffffff, 0x00000100, 680 0xa214, 0xffffffff, 0x00000100, 681 0x4d8, 0xffffffff, 0x00000100, 682 0x9784, 0xffffffff, 0x00000100, 683 0x9698, 0xffffffff, 0x00000100, 684 0x4d4, 0xffffffff, 0x00000200, 685 0x30cc, 0xffffffff, 0x00000100, 686 0x802c, 0xffffffff, 0xc0000000 687 }; 688 689 static const u32 supersumo_golden_registers[] = 690 { 691 0x5eb4, 0xffffffff, 0x00000002, 692 0x5c4, 0xffffffff, 0x00000001, 693 0x7030, 0xffffffff, 0x00000011, 694 0x7c30, 0xffffffff, 0x00000011, 695 0x6104, 0x01000300, 0x00000000, 696 0x5bc0, 0x00300000, 0x00000000, 697 0x8c04, 0xffffffff, 0x40600060, 698 0x8c08, 0xffffffff, 0x001c001c, 699 0x8c20, 0xffffffff, 0x00800080, 700 0x8c24, 0xffffffff, 0x00800080, 701 0x8c18, 0xffffffff, 0x20202078, 702 0x8c1c, 0xffffffff, 0x00001010, 703 0x918c, 0xffffffff, 0x00010006, 704 0x91a8, 0xffffffff, 0x00010006, 705 0x91c4, 0xffffffff, 0x00010006, 706 0x91e0, 0xffffffff, 0x00010006, 707 0x9200, 0xffffffff, 0x00010006, 708 0x9150, 0xffffffff, 0x6e944040, 709 0x917c, 0xffffffff, 0x00030002, 710 0x9180, 0xffffffff, 0x00050004, 711 0x9198, 0xffffffff, 0x00030002, 712 0x919c, 0xffffffff, 0x00050004, 713 0x91b4, 0xffffffff, 0x00030002, 714 0x91b8, 0xffffffff, 0x00050004, 715 0x91d0, 0xffffffff, 0x00030002, 716 0x91d4, 0xffffffff, 0x00050004, 717 0x91f0, 0xffffffff, 0x00030002, 718 0x91f4, 0xffffffff, 0x00050004, 719 0x915c, 0xffffffff, 0x00010000, 720 0x9160, 0xffffffff, 0x00030002, 721 0x3f90, 0xffff0000, 0xff000000, 722 0x9178, 0xffffffff, 0x00070000, 723 0x9194, 0xffffffff, 0x00070000, 724 0x91b0, 0xffffffff, 0x00070000, 725 0x91cc, 0xffffffff, 0x00070000, 726 0x91ec, 0xffffffff, 0x00070000, 727 0x9148, 0xffff0000, 0xff000000, 728 0x9190, 0xffffffff, 0x00090008, 729 0x91ac, 0xffffffff, 0x00090008, 730 0x91c8, 0xffffffff, 0x00090008, 731 0x91e4, 0xffffffff, 0x00090008, 732 0x9204, 0xffffffff, 0x00090008, 733 0x3f94, 0xffff0000, 0xff000000, 734 0x914c, 0xffff0000, 0xff000000, 735 0x929c, 0xffffffff, 0x00000001, 736 0x8a18, 0xffffffff, 0x00000100, 737 0x8b28, 0xffffffff, 0x00000100, 738 0x9144, 0xffffffff, 0x00000100, 739 0x5644, 0xffffffff, 0x00000100, 740 0x9b7c, 0xffffffff, 0x00000000, 741 0x8030, 0xffffffff, 0x0000100a, 742 0x8a14, 0xffffffff, 0x00000007, 743 0x8b24, 0xffffffff, 0x00ff0fff, 744 0x8b10, 0xffffffff, 0x00000000, 745 0x28a4c, 0x06000000, 0x06000000, 746 0x4d8, 0xffffffff, 0x00000100, 747 0x913c, 0xffff000f, 0x0100000a, 748 0x960c, 0xffffffff, 0x54763210, 749 0x88c4, 0xffffffff, 0x000000c2, 750 0x88d4, 0xffffffff, 0x00000010, 751 0x8974, 0xffffffff, 0x00000000, 752 0xc78, 0x00000080, 0x00000080, 753 0x5e78, 0xffffffff, 0x001000f0, 754 0xd02c, 0xffffffff, 0x08421000, 755 0xa008, 0xffffffff, 0x00010000, 756 0x8d00, 0xffffffff, 0x100e4848, 757 0x8d04, 0xffffffff, 0x00164745, 758 0x8c00, 0xffffffff, 0xe4000003, 759 0x8cf0, 0x1fffffff, 0x08e00620, 760 0x28350, 0xffffffff, 0x00000000, 761 0x9508, 0xffffffff, 0x00000002 762 }; 763 764 static const u32 sumo_golden_registers[] = 765 { 766 0x900c, 0x00ffffff, 0x0017071f, 767 0x8c18, 0xffffffff, 0x10101060, 768 0x8c1c, 0xffffffff, 0x00001010, 769 0x8c30, 0x0000000f, 0x00000005, 770 0x9688, 0x0000000f, 0x00000007 771 }; 772 773 static const u32 wrestler_golden_registers[] = 774 { 775 0x5eb4, 0xffffffff, 0x00000002, 776 0x5c4, 0xffffffff, 0x00000001, 777 0x7030, 0xffffffff, 0x00000011, 778 0x7c30, 0xffffffff, 0x00000011, 779 0x6104, 0x01000300, 0x00000000, 780 0x5bc0, 0x00300000, 0x00000000, 781 0x918c, 0xffffffff, 0x00010006, 782 0x91a8, 0xffffffff, 0x00010006, 783 0x9150, 0xffffffff, 0x6e944040, 784 0x917c, 0xffffffff, 0x00030002, 785 0x9198, 0xffffffff, 0x00030002, 786 0x915c, 0xffffffff, 0x00010000, 787 0x3f90, 0xffff0000, 0xff000000, 788 0x9178, 0xffffffff, 0x00070000, 789 0x9194, 0xffffffff, 0x00070000, 790 0x9148, 0xffff0000, 0xff000000, 791 0x9190, 0xffffffff, 0x00090008, 792 0x91ac, 0xffffffff, 0x00090008, 793 0x3f94, 0xffff0000, 0xff000000, 794 0x914c, 0xffff0000, 0xff000000, 795 0x929c, 0xffffffff, 0x00000001, 796 0x8a18, 0xffffffff, 0x00000100, 797 0x8b28, 0xffffffff, 0x00000100, 798 0x9144, 0xffffffff, 0x00000100, 799 0x9b7c, 0xffffffff, 0x00000000, 800 0x8030, 0xffffffff, 0x0000100a, 801 0x8a14, 0xffffffff, 0x00000001, 802 0x8b24, 0xffffffff, 0x00ff0fff, 803 0x8b10, 0xffffffff, 0x00000000, 804 0x28a4c, 0x06000000, 0x06000000, 805 0x4d8, 0xffffffff, 0x00000100, 806 0x913c, 0xffff000f, 0x0100000a, 807 0x960c, 0xffffffff, 0x54763210, 808 0x88c4, 0xffffffff, 0x000000c2, 809 0x88d4, 0xffffffff, 0x00000010, 810 0x8974, 0xffffffff, 0x00000000, 811 0xc78, 0x00000080, 0x00000080, 812 0x5e78, 0xffffffff, 0x001000f0, 813 0xd02c, 0xffffffff, 0x08421000, 814 0xa008, 0xffffffff, 0x00010000, 815 0x8d00, 0xffffffff, 0x100e4848, 816 0x8d04, 0xffffffff, 0x00164745, 817 0x8c00, 0xffffffff, 0xe4000003, 818 0x8cf0, 0x1fffffff, 0x08e00410, 819 0x28350, 0xffffffff, 0x00000000, 820 0x9508, 0xffffffff, 0x00000002, 821 0x900c, 0xffffffff, 0x0017071f, 822 0x8c18, 0xffffffff, 0x10101060, 823 0x8c1c, 0xffffffff, 0x00001010 824 }; 825 826 static const u32 barts_golden_registers[] = 827 { 828 0x5eb4, 0xffffffff, 0x00000002, 829 0x5e78, 0x8f311ff1, 0x001000f0, 830 0x3f90, 0xffff0000, 0xff000000, 831 0x9148, 0xffff0000, 0xff000000, 832 0x3f94, 0xffff0000, 0xff000000, 833 0x914c, 0xffff0000, 0xff000000, 834 0xc78, 0x00000080, 0x00000080, 835 0xbd4, 0x70073777, 0x00010001, 836 0xd02c, 0xbfffff1f, 0x08421000, 837 0xd0b8, 0x03773777, 0x02011003, 838 0x5bc0, 0x00200000, 0x50100000, 839 0x98f8, 0x33773777, 0x02011003, 840 0x98fc, 0xffffffff, 0x76543210, 841 0x7030, 0x31000311, 0x00000011, 842 0x2f48, 0x00000007, 0x02011003, 843 0x6b28, 0x00000010, 0x00000012, 844 0x7728, 0x00000010, 0x00000012, 845 0x10328, 0x00000010, 0x00000012, 846 0x10f28, 0x00000010, 0x00000012, 847 0x11b28, 0x00000010, 0x00000012, 848 0x12728, 0x00000010, 0x00000012, 849 0x240c, 0x000007ff, 0x00000380, 850 0x8a14, 0xf000001f, 0x00000007, 851 0x8b24, 0x3fff3fff, 0x00ff0fff, 852 0x8b10, 0x0000ff0f, 0x00000000, 853 0x28a4c, 0x07ffffff, 0x06000000, 854 0x10c, 0x00000001, 0x00010003, 855 0xa02c, 0xffffffff, 0x0000009b, 856 0x913c, 0x0000000f, 0x0100000a, 857 0x8d00, 0xffff7f7f, 0x100e4848, 858 0x8d04, 0x00ffffff, 0x00164745, 859 0x8c00, 0xfffc0003, 0xe4000003, 860 0x8c04, 0xf8ff00ff, 0x40600060, 861 0x8c08, 0x00ff00ff, 0x001c001c, 862 0x8cf0, 0x1fff1fff, 0x08e00620, 863 0x8c20, 0x0fff0fff, 0x00800080, 864 0x8c24, 0x0fff0fff, 0x00800080, 865 0x8c18, 0xffffffff, 0x20202078, 866 0x8c1c, 0x0000ffff, 0x00001010, 867 0x28350, 0x00000f01, 0x00000000, 868 0x9508, 0x3700001f, 0x00000002, 869 0x960c, 0xffffffff, 0x54763210, 870 0x88c4, 0x001f3ae3, 0x000000c2, 871 0x88d4, 0x0000001f, 0x00000010, 872 0x8974, 0xffffffff, 0x00000000 873 }; 874 875 static const u32 turks_golden_registers[] = 876 { 877 0x5eb4, 0xffffffff, 0x00000002, 878 0x5e78, 0x8f311ff1, 0x001000f0, 879 0x8c8, 0x00003000, 0x00001070, 880 0x8cc, 0x000fffff, 0x00040035, 881 0x3f90, 0xffff0000, 0xfff00000, 882 0x9148, 0xffff0000, 0xfff00000, 883 0x3f94, 0xffff0000, 0xfff00000, 884 0x914c, 0xffff0000, 0xfff00000, 885 0xc78, 0x00000080, 0x00000080, 886 0xbd4, 0x00073007, 0x00010002, 887 0xd02c, 0xbfffff1f, 0x08421000, 888 0xd0b8, 0x03773777, 0x02010002, 889 0x5bc0, 0x00200000, 0x50100000, 890 0x98f8, 0x33773777, 0x00010002, 891 0x98fc, 0xffffffff, 0x33221100, 892 0x7030, 0x31000311, 0x00000011, 893 0x2f48, 0x33773777, 0x00010002, 894 0x6b28, 0x00000010, 0x00000012, 895 0x7728, 0x00000010, 0x00000012, 896 0x10328, 0x00000010, 0x00000012, 897 0x10f28, 0x00000010, 0x00000012, 898 0x11b28, 0x00000010, 0x00000012, 899 0x12728, 0x00000010, 0x00000012, 900 0x240c, 0x000007ff, 0x00000380, 901 0x8a14, 0xf000001f, 0x00000007, 902 0x8b24, 0x3fff3fff, 0x00ff0fff, 903 0x8b10, 0x0000ff0f, 0x00000000, 904 0x28a4c, 0x07ffffff, 0x06000000, 905 0x10c, 0x00000001, 0x00010003, 906 0xa02c, 0xffffffff, 0x0000009b, 907 0x913c, 0x0000000f, 0x0100000a, 908 0x8d00, 0xffff7f7f, 0x100e4848, 909 0x8d04, 0x00ffffff, 0x00164745, 910 0x8c00, 0xfffc0003, 0xe4000003, 911 0x8c04, 0xf8ff00ff, 0x40600060, 912 0x8c08, 0x00ff00ff, 0x001c001c, 913 0x8cf0, 0x1fff1fff, 0x08e00410, 914 0x8c20, 0x0fff0fff, 0x00800080, 915 0x8c24, 0x0fff0fff, 0x00800080, 916 0x8c18, 0xffffffff, 0x20202078, 917 0x8c1c, 0x0000ffff, 0x00001010, 918 0x28350, 0x00000f01, 0x00000000, 919 0x9508, 0x3700001f, 0x00000002, 920 0x960c, 0xffffffff, 0x54763210, 921 0x88c4, 0x001f3ae3, 0x000000c2, 922 0x88d4, 0x0000001f, 0x00000010, 923 0x8974, 0xffffffff, 0x00000000 924 }; 925 926 static const u32 caicos_golden_registers[] = 927 { 928 0x5eb4, 0xffffffff, 0x00000002, 929 0x5e78, 0x8f311ff1, 0x001000f0, 930 0x8c8, 0x00003420, 0x00001450, 931 0x8cc, 0x000fffff, 0x00040035, 932 0x3f90, 0xffff0000, 0xfffc0000, 933 0x9148, 0xffff0000, 0xfffc0000, 934 0x3f94, 0xffff0000, 0xfffc0000, 935 0x914c, 0xffff0000, 0xfffc0000, 936 0xc78, 0x00000080, 0x00000080, 937 0xbd4, 0x00073007, 0x00010001, 938 0xd02c, 0xbfffff1f, 0x08421000, 939 0xd0b8, 0x03773777, 0x02010001, 940 0x5bc0, 0x00200000, 0x50100000, 941 0x98f8, 0x33773777, 0x02010001, 942 0x98fc, 0xffffffff, 0x33221100, 943 0x7030, 0x31000311, 0x00000011, 944 0x2f48, 0x33773777, 0x02010001, 945 0x6b28, 0x00000010, 0x00000012, 946 0x7728, 0x00000010, 0x00000012, 947 0x10328, 0x00000010, 0x00000012, 948 0x10f28, 0x00000010, 0x00000012, 949 0x11b28, 0x00000010, 0x00000012, 950 0x12728, 0x00000010, 0x00000012, 951 0x240c, 0x000007ff, 0x00000380, 952 0x8a14, 0xf000001f, 0x00000001, 953 0x8b24, 0x3fff3fff, 0x00ff0fff, 954 0x8b10, 0x0000ff0f, 0x00000000, 955 0x28a4c, 0x07ffffff, 0x06000000, 956 0x10c, 0x00000001, 0x00010003, 957 0xa02c, 0xffffffff, 0x0000009b, 958 0x913c, 0x0000000f, 0x0100000a, 959 0x8d00, 0xffff7f7f, 0x100e4848, 960 0x8d04, 0x00ffffff, 0x00164745, 961 0x8c00, 0xfffc0003, 0xe4000003, 962 0x8c04, 0xf8ff00ff, 0x40600060, 963 0x8c08, 0x00ff00ff, 0x001c001c, 964 0x8cf0, 0x1fff1fff, 0x08e00410, 965 0x8c20, 0x0fff0fff, 0x00800080, 966 0x8c24, 0x0fff0fff, 0x00800080, 967 0x8c18, 0xffffffff, 0x20202078, 968 0x8c1c, 0x0000ffff, 0x00001010, 969 0x28350, 0x00000f01, 0x00000000, 970 0x9508, 0x3700001f, 0x00000002, 971 0x960c, 0xffffffff, 0x54763210, 972 0x88c4, 0x001f3ae3, 0x000000c2, 973 0x88d4, 0x0000001f, 0x00000010, 974 0x8974, 0xffffffff, 0x00000000 975 }; 976 977 static void evergreen_init_golden_registers(struct radeon_device *rdev) 978 { 979 switch (rdev->family) { 980 case CHIP_CYPRESS: 981 case CHIP_HEMLOCK: 982 radeon_program_register_sequence(rdev, 983 evergreen_golden_registers, 984 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 985 radeon_program_register_sequence(rdev, 986 evergreen_golden_registers2, 987 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 988 radeon_program_register_sequence(rdev, 989 cypress_mgcg_init, 990 (const u32)ARRAY_SIZE(cypress_mgcg_init)); 991 break; 992 case CHIP_JUNIPER: 993 radeon_program_register_sequence(rdev, 994 evergreen_golden_registers, 995 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 996 radeon_program_register_sequence(rdev, 997 evergreen_golden_registers2, 998 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 999 radeon_program_register_sequence(rdev, 1000 juniper_mgcg_init, 1001 (const u32)ARRAY_SIZE(juniper_mgcg_init)); 1002 break; 1003 case CHIP_REDWOOD: 1004 radeon_program_register_sequence(rdev, 1005 evergreen_golden_registers, 1006 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 1007 radeon_program_register_sequence(rdev, 1008 evergreen_golden_registers2, 1009 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 1010 radeon_program_register_sequence(rdev, 1011 redwood_mgcg_init, 1012 (const u32)ARRAY_SIZE(redwood_mgcg_init)); 1013 break; 1014 case CHIP_CEDAR: 1015 radeon_program_register_sequence(rdev, 1016 cedar_golden_registers, 1017 (const u32)ARRAY_SIZE(cedar_golden_registers)); 1018 radeon_program_register_sequence(rdev, 1019 evergreen_golden_registers2, 1020 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 1021 radeon_program_register_sequence(rdev, 1022 cedar_mgcg_init, 1023 (const u32)ARRAY_SIZE(cedar_mgcg_init)); 1024 break; 1025 case CHIP_PALM: 1026 radeon_program_register_sequence(rdev, 1027 wrestler_golden_registers, 1028 (const u32)ARRAY_SIZE(wrestler_golden_registers)); 1029 break; 1030 case CHIP_SUMO: 1031 radeon_program_register_sequence(rdev, 1032 supersumo_golden_registers, 1033 (const u32)ARRAY_SIZE(supersumo_golden_registers)); 1034 break; 1035 case CHIP_SUMO2: 1036 radeon_program_register_sequence(rdev, 1037 supersumo_golden_registers, 1038 (const u32)ARRAY_SIZE(supersumo_golden_registers)); 1039 radeon_program_register_sequence(rdev, 1040 sumo_golden_registers, 1041 (const u32)ARRAY_SIZE(sumo_golden_registers)); 1042 break; 1043 case CHIP_BARTS: 1044 radeon_program_register_sequence(rdev, 1045 barts_golden_registers, 1046 (const u32)ARRAY_SIZE(barts_golden_registers)); 1047 break; 1048 case CHIP_TURKS: 1049 radeon_program_register_sequence(rdev, 1050 turks_golden_registers, 1051 (const u32)ARRAY_SIZE(turks_golden_registers)); 1052 break; 1053 case CHIP_CAICOS: 1054 radeon_program_register_sequence(rdev, 1055 caicos_golden_registers, 1056 (const u32)ARRAY_SIZE(caicos_golden_registers)); 1057 break; 1058 default: 1059 break; 1060 } 1061 } 1062 1063 /** 1064 * evergreen_get_allowed_info_register - fetch the register for the info ioctl 1065 * 1066 * @rdev: radeon_device pointer 1067 * @reg: register offset in bytes 1068 * @val: register value 1069 * 1070 * Returns 0 for success or -EINVAL for an invalid register 1071 * 1072 */ 1073 int evergreen_get_allowed_info_register(struct radeon_device *rdev, 1074 u32 reg, u32 *val) 1075 { 1076 switch (reg) { 1077 case GRBM_STATUS: 1078 case GRBM_STATUS_SE0: 1079 case GRBM_STATUS_SE1: 1080 case SRBM_STATUS: 1081 case SRBM_STATUS2: 1082 case DMA_STATUS_REG: 1083 case UVD_STATUS: 1084 *val = RREG32(reg); 1085 return 0; 1086 default: 1087 return -EINVAL; 1088 } 1089 } 1090 1091 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw, 1092 unsigned *bankh, unsigned *mtaspect, 1093 unsigned *tile_split) 1094 { 1095 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK; 1096 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK; 1097 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK; 1098 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK; 1099 switch (*bankw) { 1100 default: 1101 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break; 1102 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break; 1103 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break; 1104 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break; 1105 } 1106 switch (*bankh) { 1107 default: 1108 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break; 1109 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break; 1110 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break; 1111 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break; 1112 } 1113 switch (*mtaspect) { 1114 default: 1115 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break; 1116 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break; 1117 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break; 1118 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break; 1119 } 1120 } 1121 1122 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock, 1123 u32 cntl_reg, u32 status_reg) 1124 { 1125 int r, i; 1126 struct atom_clock_dividers dividers; 1127 1128 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, 1129 clock, false, ÷rs); 1130 if (r) 1131 return r; 1132 1133 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK)); 1134 1135 for (i = 0; i < 100; i++) { 1136 if (RREG32(status_reg) & DCLK_STATUS) 1137 break; 1138 mdelay(10); 1139 } 1140 if (i == 100) 1141 return -ETIMEDOUT; 1142 1143 return 0; 1144 } 1145 1146 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 1147 { 1148 int r = 0; 1149 u32 cg_scratch = RREG32(CG_SCRATCH1); 1150 1151 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); 1152 if (r) 1153 goto done; 1154 cg_scratch &= 0xffff0000; 1155 cg_scratch |= vclk / 100; /* Mhz */ 1156 1157 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); 1158 if (r) 1159 goto done; 1160 cg_scratch &= 0x0000ffff; 1161 cg_scratch |= (dclk / 100) << 16; /* Mhz */ 1162 1163 done: 1164 WREG32(CG_SCRATCH1, cg_scratch); 1165 1166 return r; 1167 } 1168 1169 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 1170 { 1171 /* start off with something large */ 1172 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0; 1173 int r; 1174 1175 /* bypass vclk and dclk with bclk */ 1176 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1177 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1), 1178 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 1179 1180 /* put PLL in bypass mode */ 1181 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK); 1182 1183 if (!vclk || !dclk) { 1184 /* keep the Bypass mode, put PLL to sleep */ 1185 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 1186 return 0; 1187 } 1188 1189 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000, 1190 16384, 0x03FFFFFF, 0, 128, 5, 1191 &fb_div, &vclk_div, &dclk_div); 1192 if (r) 1193 return r; 1194 1195 /* set VCO_MODE to 1 */ 1196 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK); 1197 1198 /* toggle UPLL_SLEEP to 1 then back to 0 */ 1199 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 1200 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK); 1201 1202 /* deassert UPLL_RESET */ 1203 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 1204 1205 mdelay(1); 1206 1207 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 1208 if (r) 1209 return r; 1210 1211 /* assert UPLL_RESET again */ 1212 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK); 1213 1214 /* disable spread spectrum. */ 1215 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK); 1216 1217 /* set feedback divider */ 1218 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK); 1219 1220 /* set ref divider to 0 */ 1221 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK); 1222 1223 if (fb_div < 307200) 1224 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9); 1225 else 1226 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9); 1227 1228 /* set PDIV_A and PDIV_B */ 1229 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1230 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div), 1231 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK)); 1232 1233 /* give the PLL some time to settle */ 1234 mdelay(15); 1235 1236 /* deassert PLL_RESET */ 1237 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 1238 1239 mdelay(15); 1240 1241 /* switch from bypass mode to normal mode */ 1242 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK); 1243 1244 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 1245 if (r) 1246 return r; 1247 1248 /* switch VCLK and DCLK selection */ 1249 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1250 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2), 1251 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 1252 1253 mdelay(100); 1254 1255 return 0; 1256 } 1257 1258 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev) 1259 { 1260 int readrq; 1261 u16 v; 1262 1263 readrq = pcie_get_readrq(rdev->pdev); 1264 v = ffs(readrq) - 8; 1265 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it 1266 * to avoid hangs or perfomance issues 1267 */ 1268 if ((v == 0) || (v == 6) || (v == 7)) 1269 pcie_set_readrq(rdev->pdev, 512); 1270 } 1271 1272 void dce4_program_fmt(struct drm_encoder *encoder) 1273 { 1274 struct drm_device *dev = encoder->dev; 1275 struct radeon_device *rdev = dev->dev_private; 1276 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); 1277 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); 1278 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); 1279 int bpc = 0; 1280 u32 tmp = 0; 1281 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; 1282 1283 if (connector) { 1284 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1285 bpc = radeon_get_monitor_bpc(connector); 1286 dither = radeon_connector->dither; 1287 } 1288 1289 /* LVDS/eDP FMT is set up by atom */ 1290 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) 1291 return; 1292 1293 /* not needed for analog */ 1294 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || 1295 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) 1296 return; 1297 1298 if (bpc == 0) 1299 return; 1300 1301 switch (bpc) { 1302 case 6: 1303 if (dither == RADEON_FMT_DITHER_ENABLE) 1304 /* XXX sort out optimal dither settings */ 1305 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 1306 FMT_SPATIAL_DITHER_EN); 1307 else 1308 tmp |= FMT_TRUNCATE_EN; 1309 break; 1310 case 8: 1311 if (dither == RADEON_FMT_DITHER_ENABLE) 1312 /* XXX sort out optimal dither settings */ 1313 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 1314 FMT_RGB_RANDOM_ENABLE | 1315 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH); 1316 else 1317 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH); 1318 break; 1319 case 10: 1320 default: 1321 /* not needed */ 1322 break; 1323 } 1324 1325 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); 1326 } 1327 1328 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc) 1329 { 1330 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK) 1331 return true; 1332 else 1333 return false; 1334 } 1335 1336 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc) 1337 { 1338 u32 pos1, pos2; 1339 1340 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]); 1341 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]); 1342 1343 if (pos1 != pos2) 1344 return true; 1345 else 1346 return false; 1347 } 1348 1349 /** 1350 * dce4_wait_for_vblank - vblank wait asic callback. 1351 * 1352 * @rdev: radeon_device pointer 1353 * @crtc: crtc to wait for vblank on 1354 * 1355 * Wait for vblank on the requested crtc (evergreen+). 1356 */ 1357 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc) 1358 { 1359 unsigned i = 0; 1360 1361 if (crtc >= rdev->num_crtc) 1362 return; 1363 1364 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN)) 1365 return; 1366 1367 /* depending on when we hit vblank, we may be close to active; if so, 1368 * wait for another frame. 1369 */ 1370 while (dce4_is_in_vblank(rdev, crtc)) { 1371 if (i++ % 100 == 0) { 1372 if (!dce4_is_counter_moving(rdev, crtc)) 1373 break; 1374 } 1375 } 1376 1377 while (!dce4_is_in_vblank(rdev, crtc)) { 1378 if (i++ % 100 == 0) { 1379 if (!dce4_is_counter_moving(rdev, crtc)) 1380 break; 1381 } 1382 } 1383 } 1384 1385 /** 1386 * evergreen_page_flip - pageflip callback. 1387 * 1388 * @rdev: radeon_device pointer 1389 * @crtc_id: crtc to cleanup pageflip on 1390 * @crtc_base: new address of the crtc (GPU MC address) 1391 * 1392 * Triggers the actual pageflip by updating the primary 1393 * surface base address (evergreen+). 1394 */ 1395 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, 1396 bool async) 1397 { 1398 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; 1399 1400 /* update the scanout addresses */ 1401 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset, 1402 async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0); 1403 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, 1404 upper_32_bits(crtc_base)); 1405 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, 1406 (u32)crtc_base); 1407 /* post the write */ 1408 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset); 1409 } 1410 1411 /** 1412 * evergreen_page_flip_pending - check if page flip is still pending 1413 * 1414 * @rdev: radeon_device pointer 1415 * @crtc_id: crtc to check 1416 * 1417 * Returns the current update pending status. 1418 */ 1419 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id) 1420 { 1421 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; 1422 1423 /* Return current update_pending status: */ 1424 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & 1425 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING); 1426 } 1427 1428 /* get temperature in millidegrees */ 1429 int evergreen_get_temp(struct radeon_device *rdev) 1430 { 1431 u32 temp, toffset; 1432 int actual_temp = 0; 1433 1434 if (rdev->family == CHIP_JUNIPER) { 1435 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >> 1436 TOFFSET_SHIFT; 1437 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >> 1438 TS0_ADC_DOUT_SHIFT; 1439 1440 if (toffset & 0x100) 1441 actual_temp = temp / 2 - (0x200 - toffset); 1442 else 1443 actual_temp = temp / 2 + toffset; 1444 1445 actual_temp = actual_temp * 1000; 1446 1447 } else { 1448 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >> 1449 ASIC_T_SHIFT; 1450 1451 if (temp & 0x400) 1452 actual_temp = -256; 1453 else if (temp & 0x200) 1454 actual_temp = 255; 1455 else if (temp & 0x100) { 1456 actual_temp = temp & 0x1ff; 1457 actual_temp |= ~0x1ff; 1458 } else 1459 actual_temp = temp & 0xff; 1460 1461 actual_temp = (actual_temp * 1000) / 2; 1462 } 1463 1464 return actual_temp; 1465 } 1466 1467 int sumo_get_temp(struct radeon_device *rdev) 1468 { 1469 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff; 1470 int actual_temp = temp - 49; 1471 1472 return actual_temp * 1000; 1473 } 1474 1475 /** 1476 * sumo_pm_init_profile - Initialize power profiles callback. 1477 * 1478 * @rdev: radeon_device pointer 1479 * 1480 * Initialize the power states used in profile mode 1481 * (sumo, trinity, SI). 1482 * Used for profile mode only. 1483 */ 1484 void sumo_pm_init_profile(struct radeon_device *rdev) 1485 { 1486 int idx; 1487 1488 /* default */ 1489 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index; 1490 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index; 1491 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0; 1492 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0; 1493 1494 /* low,mid sh/mh */ 1495 if (rdev->flags & RADEON_IS_MOBILITY) 1496 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0); 1497 else 1498 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1499 1500 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx; 1501 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx; 1502 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0; 1503 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0; 1504 1505 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx; 1506 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx; 1507 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0; 1508 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0; 1509 1510 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx; 1511 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx; 1512 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0; 1513 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0; 1514 1515 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx; 1516 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx; 1517 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0; 1518 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0; 1519 1520 /* high sh/mh */ 1521 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1522 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx; 1523 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx; 1524 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0; 1525 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 1526 rdev->pm.power_state[idx].num_clock_modes - 1; 1527 1528 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx; 1529 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx; 1530 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0; 1531 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 1532 rdev->pm.power_state[idx].num_clock_modes - 1; 1533 } 1534 1535 /** 1536 * btc_pm_init_profile - Initialize power profiles callback. 1537 * 1538 * @rdev: radeon_device pointer 1539 * 1540 * Initialize the power states used in profile mode 1541 * (BTC, cayman). 1542 * Used for profile mode only. 1543 */ 1544 void btc_pm_init_profile(struct radeon_device *rdev) 1545 { 1546 int idx; 1547 1548 /* default */ 1549 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index; 1550 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index; 1551 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0; 1552 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2; 1553 /* starting with BTC, there is one state that is used for both 1554 * MH and SH. Difference is that we always use the high clock index for 1555 * mclk. 1556 */ 1557 if (rdev->flags & RADEON_IS_MOBILITY) 1558 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0); 1559 else 1560 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1561 /* low sh */ 1562 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx; 1563 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx; 1564 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0; 1565 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0; 1566 /* mid sh */ 1567 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx; 1568 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx; 1569 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0; 1570 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1; 1571 /* high sh */ 1572 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx; 1573 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx; 1574 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0; 1575 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2; 1576 /* low mh */ 1577 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx; 1578 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx; 1579 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0; 1580 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0; 1581 /* mid mh */ 1582 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx; 1583 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx; 1584 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0; 1585 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1; 1586 /* high mh */ 1587 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx; 1588 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx; 1589 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0; 1590 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2; 1591 } 1592 1593 /** 1594 * evergreen_pm_misc - set additional pm hw parameters callback. 1595 * 1596 * @rdev: radeon_device pointer 1597 * 1598 * Set non-clock parameters associated with a power state 1599 * (voltage, etc.) (evergreen+). 1600 */ 1601 void evergreen_pm_misc(struct radeon_device *rdev) 1602 { 1603 int req_ps_idx = rdev->pm.requested_power_state_index; 1604 int req_cm_idx = rdev->pm.requested_clock_mode_index; 1605 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx]; 1606 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage; 1607 1608 if (voltage->type == VOLTAGE_SW) { 1609 /* 0xff0x are flags rather then an actual voltage */ 1610 if ((voltage->voltage & 0xff00) == 0xff00) 1611 return; 1612 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) { 1613 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC); 1614 rdev->pm.current_vddc = voltage->voltage; 1615 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage); 1616 } 1617 1618 /* starting with BTC, there is one state that is used for both 1619 * MH and SH. Difference is that we always use the high clock index for 1620 * mclk and vddci. 1621 */ 1622 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) && 1623 (rdev->family >= CHIP_BARTS) && 1624 rdev->pm.active_crtc_count && 1625 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) || 1626 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX))) 1627 voltage = &rdev->pm.power_state[req_ps_idx]. 1628 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage; 1629 1630 /* 0xff0x are flags rather then an actual voltage */ 1631 if ((voltage->vddci & 0xff00) == 0xff00) 1632 return; 1633 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) { 1634 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI); 1635 rdev->pm.current_vddci = voltage->vddci; 1636 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci); 1637 } 1638 } 1639 } 1640 1641 /** 1642 * evergreen_pm_prepare - pre-power state change callback. 1643 * 1644 * @rdev: radeon_device pointer 1645 * 1646 * Prepare for a power state change (evergreen+). 1647 */ 1648 void evergreen_pm_prepare(struct radeon_device *rdev) 1649 { 1650 struct drm_device *ddev = rdev->ddev; 1651 struct drm_crtc *crtc; 1652 struct radeon_crtc *radeon_crtc; 1653 u32 tmp; 1654 1655 /* disable any active CRTCs */ 1656 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 1657 radeon_crtc = to_radeon_crtc(crtc); 1658 if (radeon_crtc->enabled) { 1659 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 1660 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 1661 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 1662 } 1663 } 1664 } 1665 1666 /** 1667 * evergreen_pm_finish - post-power state change callback. 1668 * 1669 * @rdev: radeon_device pointer 1670 * 1671 * Clean up after a power state change (evergreen+). 1672 */ 1673 void evergreen_pm_finish(struct radeon_device *rdev) 1674 { 1675 struct drm_device *ddev = rdev->ddev; 1676 struct drm_crtc *crtc; 1677 struct radeon_crtc *radeon_crtc; 1678 u32 tmp; 1679 1680 /* enable any active CRTCs */ 1681 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 1682 radeon_crtc = to_radeon_crtc(crtc); 1683 if (radeon_crtc->enabled) { 1684 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 1685 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 1686 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 1687 } 1688 } 1689 } 1690 1691 /** 1692 * evergreen_hpd_sense - hpd sense callback. 1693 * 1694 * @rdev: radeon_device pointer 1695 * @hpd: hpd (hotplug detect) pin 1696 * 1697 * Checks if a digital monitor is connected (evergreen+). 1698 * Returns true if connected, false if not connected. 1699 */ 1700 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd) 1701 { 1702 bool connected = false; 1703 1704 switch (hpd) { 1705 case RADEON_HPD_1: 1706 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE) 1707 connected = true; 1708 break; 1709 case RADEON_HPD_2: 1710 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE) 1711 connected = true; 1712 break; 1713 case RADEON_HPD_3: 1714 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE) 1715 connected = true; 1716 break; 1717 case RADEON_HPD_4: 1718 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE) 1719 connected = true; 1720 break; 1721 case RADEON_HPD_5: 1722 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE) 1723 connected = true; 1724 break; 1725 case RADEON_HPD_6: 1726 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE) 1727 connected = true; 1728 break; 1729 default: 1730 break; 1731 } 1732 1733 return connected; 1734 } 1735 1736 /** 1737 * evergreen_hpd_set_polarity - hpd set polarity callback. 1738 * 1739 * @rdev: radeon_device pointer 1740 * @hpd: hpd (hotplug detect) pin 1741 * 1742 * Set the polarity of the hpd pin (evergreen+). 1743 */ 1744 void evergreen_hpd_set_polarity(struct radeon_device *rdev, 1745 enum radeon_hpd_id hpd) 1746 { 1747 u32 tmp; 1748 bool connected = evergreen_hpd_sense(rdev, hpd); 1749 1750 switch (hpd) { 1751 case RADEON_HPD_1: 1752 tmp = RREG32(DC_HPD1_INT_CONTROL); 1753 if (connected) 1754 tmp &= ~DC_HPDx_INT_POLARITY; 1755 else 1756 tmp |= DC_HPDx_INT_POLARITY; 1757 WREG32(DC_HPD1_INT_CONTROL, tmp); 1758 break; 1759 case RADEON_HPD_2: 1760 tmp = RREG32(DC_HPD2_INT_CONTROL); 1761 if (connected) 1762 tmp &= ~DC_HPDx_INT_POLARITY; 1763 else 1764 tmp |= DC_HPDx_INT_POLARITY; 1765 WREG32(DC_HPD2_INT_CONTROL, tmp); 1766 break; 1767 case RADEON_HPD_3: 1768 tmp = RREG32(DC_HPD3_INT_CONTROL); 1769 if (connected) 1770 tmp &= ~DC_HPDx_INT_POLARITY; 1771 else 1772 tmp |= DC_HPDx_INT_POLARITY; 1773 WREG32(DC_HPD3_INT_CONTROL, tmp); 1774 break; 1775 case RADEON_HPD_4: 1776 tmp = RREG32(DC_HPD4_INT_CONTROL); 1777 if (connected) 1778 tmp &= ~DC_HPDx_INT_POLARITY; 1779 else 1780 tmp |= DC_HPDx_INT_POLARITY; 1781 WREG32(DC_HPD4_INT_CONTROL, tmp); 1782 break; 1783 case RADEON_HPD_5: 1784 tmp = RREG32(DC_HPD5_INT_CONTROL); 1785 if (connected) 1786 tmp &= ~DC_HPDx_INT_POLARITY; 1787 else 1788 tmp |= DC_HPDx_INT_POLARITY; 1789 WREG32(DC_HPD5_INT_CONTROL, tmp); 1790 break; 1791 case RADEON_HPD_6: 1792 tmp = RREG32(DC_HPD6_INT_CONTROL); 1793 if (connected) 1794 tmp &= ~DC_HPDx_INT_POLARITY; 1795 else 1796 tmp |= DC_HPDx_INT_POLARITY; 1797 WREG32(DC_HPD6_INT_CONTROL, tmp); 1798 break; 1799 default: 1800 break; 1801 } 1802 } 1803 1804 /** 1805 * evergreen_hpd_init - hpd setup callback. 1806 * 1807 * @rdev: radeon_device pointer 1808 * 1809 * Setup the hpd pins used by the card (evergreen+). 1810 * Enable the pin, set the polarity, and enable the hpd interrupts. 1811 */ 1812 void evergreen_hpd_init(struct radeon_device *rdev) 1813 { 1814 struct drm_device *dev = rdev->ddev; 1815 struct drm_connector *connector; 1816 unsigned enabled = 0; 1817 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | 1818 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN; 1819 1820 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 1821 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1822 1823 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP || 1824 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) { 1825 /* don't try to enable hpd on eDP or LVDS avoid breaking the 1826 * aux dp channel on imac and help (but not completely fix) 1827 * https://bugzilla.redhat.com/show_bug.cgi?id=726143 1828 * also avoid interrupt storms during dpms. 1829 */ 1830 continue; 1831 } 1832 switch (radeon_connector->hpd.hpd) { 1833 case RADEON_HPD_1: 1834 WREG32(DC_HPD1_CONTROL, tmp); 1835 break; 1836 case RADEON_HPD_2: 1837 WREG32(DC_HPD2_CONTROL, tmp); 1838 break; 1839 case RADEON_HPD_3: 1840 WREG32(DC_HPD3_CONTROL, tmp); 1841 break; 1842 case RADEON_HPD_4: 1843 WREG32(DC_HPD4_CONTROL, tmp); 1844 break; 1845 case RADEON_HPD_5: 1846 WREG32(DC_HPD5_CONTROL, tmp); 1847 break; 1848 case RADEON_HPD_6: 1849 WREG32(DC_HPD6_CONTROL, tmp); 1850 break; 1851 default: 1852 break; 1853 } 1854 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd); 1855 if (radeon_connector->hpd.hpd != RADEON_HPD_NONE) 1856 enabled |= 1 << radeon_connector->hpd.hpd; 1857 } 1858 radeon_irq_kms_enable_hpd(rdev, enabled); 1859 } 1860 1861 /** 1862 * evergreen_hpd_fini - hpd tear down callback. 1863 * 1864 * @rdev: radeon_device pointer 1865 * 1866 * Tear down the hpd pins used by the card (evergreen+). 1867 * Disable the hpd interrupts. 1868 */ 1869 void evergreen_hpd_fini(struct radeon_device *rdev) 1870 { 1871 struct drm_device *dev = rdev->ddev; 1872 struct drm_connector *connector; 1873 unsigned disabled = 0; 1874 1875 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 1876 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1877 switch (radeon_connector->hpd.hpd) { 1878 case RADEON_HPD_1: 1879 WREG32(DC_HPD1_CONTROL, 0); 1880 break; 1881 case RADEON_HPD_2: 1882 WREG32(DC_HPD2_CONTROL, 0); 1883 break; 1884 case RADEON_HPD_3: 1885 WREG32(DC_HPD3_CONTROL, 0); 1886 break; 1887 case RADEON_HPD_4: 1888 WREG32(DC_HPD4_CONTROL, 0); 1889 break; 1890 case RADEON_HPD_5: 1891 WREG32(DC_HPD5_CONTROL, 0); 1892 break; 1893 case RADEON_HPD_6: 1894 WREG32(DC_HPD6_CONTROL, 0); 1895 break; 1896 default: 1897 break; 1898 } 1899 if (radeon_connector->hpd.hpd != RADEON_HPD_NONE) 1900 disabled |= 1 << radeon_connector->hpd.hpd; 1901 } 1902 radeon_irq_kms_disable_hpd(rdev, disabled); 1903 } 1904 1905 /* watermark setup */ 1906 1907 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev, 1908 struct radeon_crtc *radeon_crtc, 1909 struct drm_display_mode *mode, 1910 struct drm_display_mode *other_mode) 1911 { 1912 u32 tmp, buffer_alloc, i; 1913 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 1914 /* 1915 * Line Buffer Setup 1916 * There are 3 line buffers, each one shared by 2 display controllers. 1917 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between 1918 * the display controllers. The paritioning is done via one of four 1919 * preset allocations specified in bits 2:0: 1920 * first display controller 1921 * 0 - first half of lb (3840 * 2) 1922 * 1 - first 3/4 of lb (5760 * 2) 1923 * 2 - whole lb (7680 * 2), other crtc must be disabled 1924 * 3 - first 1/4 of lb (1920 * 2) 1925 * second display controller 1926 * 4 - second half of lb (3840 * 2) 1927 * 5 - second 3/4 of lb (5760 * 2) 1928 * 6 - whole lb (7680 * 2), other crtc must be disabled 1929 * 7 - last 1/4 of lb (1920 * 2) 1930 */ 1931 /* this can get tricky if we have two large displays on a paired group 1932 * of crtcs. Ideally for multiple large displays we'd assign them to 1933 * non-linked crtcs for maximum line buffer allocation. 1934 */ 1935 if (radeon_crtc->base.enabled && mode) { 1936 if (other_mode) { 1937 tmp = 0; /* 1/2 */ 1938 buffer_alloc = 1; 1939 } else { 1940 tmp = 2; /* whole */ 1941 buffer_alloc = 2; 1942 } 1943 } else { 1944 tmp = 0; 1945 buffer_alloc = 0; 1946 } 1947 1948 /* second controller of the pair uses second half of the lb */ 1949 if (radeon_crtc->crtc_id % 2) 1950 tmp += 4; 1951 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp); 1952 1953 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) { 1954 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 1955 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 1956 for (i = 0; i < rdev->usec_timeout; i++) { 1957 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 1958 DMIF_BUFFERS_ALLOCATED_COMPLETED) 1959 break; 1960 udelay(1); 1961 } 1962 } 1963 1964 if (radeon_crtc->base.enabled && mode) { 1965 switch (tmp) { 1966 case 0: 1967 case 4: 1968 default: 1969 if (ASIC_IS_DCE5(rdev)) 1970 return 4096 * 2; 1971 else 1972 return 3840 * 2; 1973 case 1: 1974 case 5: 1975 if (ASIC_IS_DCE5(rdev)) 1976 return 6144 * 2; 1977 else 1978 return 5760 * 2; 1979 case 2: 1980 case 6: 1981 if (ASIC_IS_DCE5(rdev)) 1982 return 8192 * 2; 1983 else 1984 return 7680 * 2; 1985 case 3: 1986 case 7: 1987 if (ASIC_IS_DCE5(rdev)) 1988 return 2048 * 2; 1989 else 1990 return 1920 * 2; 1991 } 1992 } 1993 1994 /* controller not enabled, so no lb used */ 1995 return 0; 1996 } 1997 1998 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev) 1999 { 2000 u32 tmp = RREG32(MC_SHARED_CHMAP); 2001 2002 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 2003 case 0: 2004 default: 2005 return 1; 2006 case 1: 2007 return 2; 2008 case 2: 2009 return 4; 2010 case 3: 2011 return 8; 2012 } 2013 } 2014 2015 struct evergreen_wm_params { 2016 u32 dram_channels; /* number of dram channels */ 2017 u32 yclk; /* bandwidth per dram data pin in kHz */ 2018 u32 sclk; /* engine clock in kHz */ 2019 u32 disp_clk; /* display clock in kHz */ 2020 u32 src_width; /* viewport width */ 2021 u32 active_time; /* active display time in ns */ 2022 u32 blank_time; /* blank time in ns */ 2023 bool interlaced; /* mode is interlaced */ 2024 fixed20_12 vsc; /* vertical scale ratio */ 2025 u32 num_heads; /* number of active crtcs */ 2026 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 2027 u32 lb_size; /* line buffer allocated to pipe */ 2028 u32 vtaps; /* vertical scaler taps */ 2029 }; 2030 2031 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm) 2032 { 2033 /* Calculate DRAM Bandwidth and the part allocated to display. */ 2034 fixed20_12 dram_efficiency; /* 0.7 */ 2035 fixed20_12 yclk, dram_channels, bandwidth; 2036 fixed20_12 a; 2037 2038 a.full = dfixed_const(1000); 2039 yclk.full = dfixed_const(wm->yclk); 2040 yclk.full = dfixed_div(yclk, a); 2041 dram_channels.full = dfixed_const(wm->dram_channels * 4); 2042 a.full = dfixed_const(10); 2043 dram_efficiency.full = dfixed_const(7); 2044 dram_efficiency.full = dfixed_div(dram_efficiency, a); 2045 bandwidth.full = dfixed_mul(dram_channels, yclk); 2046 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 2047 2048 return dfixed_trunc(bandwidth); 2049 } 2050 2051 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 2052 { 2053 /* Calculate DRAM Bandwidth and the part allocated to display. */ 2054 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 2055 fixed20_12 yclk, dram_channels, bandwidth; 2056 fixed20_12 a; 2057 2058 a.full = dfixed_const(1000); 2059 yclk.full = dfixed_const(wm->yclk); 2060 yclk.full = dfixed_div(yclk, a); 2061 dram_channels.full = dfixed_const(wm->dram_channels * 4); 2062 a.full = dfixed_const(10); 2063 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 2064 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 2065 bandwidth.full = dfixed_mul(dram_channels, yclk); 2066 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 2067 2068 return dfixed_trunc(bandwidth); 2069 } 2070 2071 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm) 2072 { 2073 /* Calculate the display Data return Bandwidth */ 2074 fixed20_12 return_efficiency; /* 0.8 */ 2075 fixed20_12 sclk, bandwidth; 2076 fixed20_12 a; 2077 2078 a.full = dfixed_const(1000); 2079 sclk.full = dfixed_const(wm->sclk); 2080 sclk.full = dfixed_div(sclk, a); 2081 a.full = dfixed_const(10); 2082 return_efficiency.full = dfixed_const(8); 2083 return_efficiency.full = dfixed_div(return_efficiency, a); 2084 a.full = dfixed_const(32); 2085 bandwidth.full = dfixed_mul(a, sclk); 2086 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 2087 2088 return dfixed_trunc(bandwidth); 2089 } 2090 2091 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm) 2092 { 2093 /* Calculate the DMIF Request Bandwidth */ 2094 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 2095 fixed20_12 disp_clk, bandwidth; 2096 fixed20_12 a; 2097 2098 a.full = dfixed_const(1000); 2099 disp_clk.full = dfixed_const(wm->disp_clk); 2100 disp_clk.full = dfixed_div(disp_clk, a); 2101 a.full = dfixed_const(10); 2102 disp_clk_request_efficiency.full = dfixed_const(8); 2103 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 2104 a.full = dfixed_const(32); 2105 bandwidth.full = dfixed_mul(a, disp_clk); 2106 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency); 2107 2108 return dfixed_trunc(bandwidth); 2109 } 2110 2111 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm) 2112 { 2113 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 2114 u32 dram_bandwidth = evergreen_dram_bandwidth(wm); 2115 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm); 2116 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm); 2117 2118 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 2119 } 2120 2121 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm) 2122 { 2123 /* Calculate the display mode Average Bandwidth 2124 * DisplayMode should contain the source and destination dimensions, 2125 * timing, etc. 2126 */ 2127 fixed20_12 bpp; 2128 fixed20_12 line_time; 2129 fixed20_12 src_width; 2130 fixed20_12 bandwidth; 2131 fixed20_12 a; 2132 2133 a.full = dfixed_const(1000); 2134 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 2135 line_time.full = dfixed_div(line_time, a); 2136 bpp.full = dfixed_const(wm->bytes_per_pixel); 2137 src_width.full = dfixed_const(wm->src_width); 2138 bandwidth.full = dfixed_mul(src_width, bpp); 2139 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 2140 bandwidth.full = dfixed_div(bandwidth, line_time); 2141 2142 return dfixed_trunc(bandwidth); 2143 } 2144 2145 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm) 2146 { 2147 /* First calcualte the latency in ns */ 2148 u32 mc_latency = 2000; /* 2000 ns. */ 2149 u32 available_bandwidth = evergreen_available_bandwidth(wm); 2150 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 2151 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 2152 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 2153 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 2154 (wm->num_heads * cursor_line_pair_return_time); 2155 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 2156 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 2157 fixed20_12 a, b, c; 2158 2159 if (wm->num_heads == 0) 2160 return 0; 2161 2162 a.full = dfixed_const(2); 2163 b.full = dfixed_const(1); 2164 if ((wm->vsc.full > a.full) || 2165 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 2166 (wm->vtaps >= 5) || 2167 ((wm->vsc.full >= a.full) && wm->interlaced)) 2168 max_src_lines_per_dst_line = 4; 2169 else 2170 max_src_lines_per_dst_line = 2; 2171 2172 a.full = dfixed_const(available_bandwidth); 2173 b.full = dfixed_const(wm->num_heads); 2174 a.full = dfixed_div(a, b); 2175 2176 b.full = dfixed_const(1000); 2177 c.full = dfixed_const(wm->disp_clk); 2178 b.full = dfixed_div(c, b); 2179 c.full = dfixed_const(wm->bytes_per_pixel); 2180 b.full = dfixed_mul(b, c); 2181 2182 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b)); 2183 2184 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 2185 b.full = dfixed_const(1000); 2186 c.full = dfixed_const(lb_fill_bw); 2187 b.full = dfixed_div(c, b); 2188 a.full = dfixed_div(a, b); 2189 line_fill_time = dfixed_trunc(a); 2190 2191 if (line_fill_time < wm->active_time) 2192 return latency; 2193 else 2194 return latency + (line_fill_time - wm->active_time); 2195 2196 } 2197 2198 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 2199 { 2200 if (evergreen_average_bandwidth(wm) <= 2201 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads)) 2202 return true; 2203 else 2204 return false; 2205 }; 2206 2207 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm) 2208 { 2209 if (evergreen_average_bandwidth(wm) <= 2210 (evergreen_available_bandwidth(wm) / wm->num_heads)) 2211 return true; 2212 else 2213 return false; 2214 }; 2215 2216 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm) 2217 { 2218 u32 lb_partitions = wm->lb_size / wm->src_width; 2219 u32 line_time = wm->active_time + wm->blank_time; 2220 u32 latency_tolerant_lines; 2221 u32 latency_hiding; 2222 fixed20_12 a; 2223 2224 a.full = dfixed_const(1); 2225 if (wm->vsc.full > a.full) 2226 latency_tolerant_lines = 1; 2227 else { 2228 if (lb_partitions <= (wm->vtaps + 1)) 2229 latency_tolerant_lines = 1; 2230 else 2231 latency_tolerant_lines = 2; 2232 } 2233 2234 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 2235 2236 if (evergreen_latency_watermark(wm) <= latency_hiding) 2237 return true; 2238 else 2239 return false; 2240 } 2241 2242 static void evergreen_program_watermarks(struct radeon_device *rdev, 2243 struct radeon_crtc *radeon_crtc, 2244 u32 lb_size, u32 num_heads) 2245 { 2246 struct drm_display_mode *mode = &radeon_crtc->base.mode; 2247 struct evergreen_wm_params wm_low, wm_high; 2248 u32 dram_channels; 2249 u32 pixel_period; 2250 u32 line_time = 0; 2251 u32 latency_watermark_a = 0, latency_watermark_b = 0; 2252 u32 priority_a_mark = 0, priority_b_mark = 0; 2253 u32 priority_a_cnt = PRIORITY_OFF; 2254 u32 priority_b_cnt = PRIORITY_OFF; 2255 u32 pipe_offset = radeon_crtc->crtc_id * 16; 2256 u32 tmp, arb_control3; 2257 fixed20_12 a, b, c; 2258 2259 if (radeon_crtc->base.enabled && num_heads && mode) { 2260 pixel_period = 1000000 / (u32)mode->clock; 2261 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 2262 priority_a_cnt = 0; 2263 priority_b_cnt = 0; 2264 dram_channels = evergreen_get_number_of_dram_channels(rdev); 2265 2266 /* watermark for high clocks */ 2267 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2268 wm_high.yclk = 2269 radeon_dpm_get_mclk(rdev, false) * 10; 2270 wm_high.sclk = 2271 radeon_dpm_get_sclk(rdev, false) * 10; 2272 } else { 2273 wm_high.yclk = rdev->pm.current_mclk * 10; 2274 wm_high.sclk = rdev->pm.current_sclk * 10; 2275 } 2276 2277 wm_high.disp_clk = mode->clock; 2278 wm_high.src_width = mode->crtc_hdisplay; 2279 wm_high.active_time = mode->crtc_hdisplay * pixel_period; 2280 wm_high.blank_time = line_time - wm_high.active_time; 2281 wm_high.interlaced = false; 2282 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2283 wm_high.interlaced = true; 2284 wm_high.vsc = radeon_crtc->vsc; 2285 wm_high.vtaps = 1; 2286 if (radeon_crtc->rmx_type != RMX_OFF) 2287 wm_high.vtaps = 2; 2288 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2289 wm_high.lb_size = lb_size; 2290 wm_high.dram_channels = dram_channels; 2291 wm_high.num_heads = num_heads; 2292 2293 /* watermark for low clocks */ 2294 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2295 wm_low.yclk = 2296 radeon_dpm_get_mclk(rdev, true) * 10; 2297 wm_low.sclk = 2298 radeon_dpm_get_sclk(rdev, true) * 10; 2299 } else { 2300 wm_low.yclk = rdev->pm.current_mclk * 10; 2301 wm_low.sclk = rdev->pm.current_sclk * 10; 2302 } 2303 2304 wm_low.disp_clk = mode->clock; 2305 wm_low.src_width = mode->crtc_hdisplay; 2306 wm_low.active_time = mode->crtc_hdisplay * pixel_period; 2307 wm_low.blank_time = line_time - wm_low.active_time; 2308 wm_low.interlaced = false; 2309 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2310 wm_low.interlaced = true; 2311 wm_low.vsc = radeon_crtc->vsc; 2312 wm_low.vtaps = 1; 2313 if (radeon_crtc->rmx_type != RMX_OFF) 2314 wm_low.vtaps = 2; 2315 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2316 wm_low.lb_size = lb_size; 2317 wm_low.dram_channels = dram_channels; 2318 wm_low.num_heads = num_heads; 2319 2320 /* set for high clocks */ 2321 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535); 2322 /* set for low clocks */ 2323 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535); 2324 2325 /* possibly force display priority to high */ 2326 /* should really do this at mode validation time... */ 2327 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 2328 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) || 2329 !evergreen_check_latency_hiding(&wm_high) || 2330 (rdev->disp_priority == 2)) { 2331 DRM_DEBUG_KMS("force priority a to high\n"); 2332 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2333 } 2334 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 2335 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) || 2336 !evergreen_check_latency_hiding(&wm_low) || 2337 (rdev->disp_priority == 2)) { 2338 DRM_DEBUG_KMS("force priority b to high\n"); 2339 priority_b_cnt |= PRIORITY_ALWAYS_ON; 2340 } 2341 2342 a.full = dfixed_const(1000); 2343 b.full = dfixed_const(mode->clock); 2344 b.full = dfixed_div(b, a); 2345 c.full = dfixed_const(latency_watermark_a); 2346 c.full = dfixed_mul(c, b); 2347 c.full = dfixed_mul(c, radeon_crtc->hsc); 2348 c.full = dfixed_div(c, a); 2349 a.full = dfixed_const(16); 2350 c.full = dfixed_div(c, a); 2351 priority_a_mark = dfixed_trunc(c); 2352 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK; 2353 2354 a.full = dfixed_const(1000); 2355 b.full = dfixed_const(mode->clock); 2356 b.full = dfixed_div(b, a); 2357 c.full = dfixed_const(latency_watermark_b); 2358 c.full = dfixed_mul(c, b); 2359 c.full = dfixed_mul(c, radeon_crtc->hsc); 2360 c.full = dfixed_div(c, a); 2361 a.full = dfixed_const(16); 2362 c.full = dfixed_div(c, a); 2363 priority_b_mark = dfixed_trunc(c); 2364 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; 2365 2366 /* Save number of lines the linebuffer leads before the scanout */ 2367 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay); 2368 } 2369 2370 /* select wm A */ 2371 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 2372 tmp = arb_control3; 2373 tmp &= ~LATENCY_WATERMARK_MASK(3); 2374 tmp |= LATENCY_WATERMARK_MASK(1); 2375 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 2376 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 2377 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 2378 LATENCY_HIGH_WATERMARK(line_time))); 2379 /* select wm B */ 2380 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 2381 tmp &= ~LATENCY_WATERMARK_MASK(3); 2382 tmp |= LATENCY_WATERMARK_MASK(2); 2383 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 2384 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 2385 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 2386 LATENCY_HIGH_WATERMARK(line_time))); 2387 /* restore original selection */ 2388 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3); 2389 2390 /* write the priority marks */ 2391 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); 2392 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); 2393 2394 /* save values for DPM */ 2395 radeon_crtc->line_time = line_time; 2396 radeon_crtc->wm_high = latency_watermark_a; 2397 radeon_crtc->wm_low = latency_watermark_b; 2398 } 2399 2400 /** 2401 * evergreen_bandwidth_update - update display watermarks callback. 2402 * 2403 * @rdev: radeon_device pointer 2404 * 2405 * Update the display watermarks based on the requested mode(s) 2406 * (evergreen+). 2407 */ 2408 void evergreen_bandwidth_update(struct radeon_device *rdev) 2409 { 2410 struct drm_display_mode *mode0 = NULL; 2411 struct drm_display_mode *mode1 = NULL; 2412 u32 num_heads = 0, lb_size; 2413 int i; 2414 2415 if (!rdev->mode_info.mode_config_initialized) 2416 return; 2417 2418 radeon_update_display_priority(rdev); 2419 2420 for (i = 0; i < rdev->num_crtc; i++) { 2421 if (rdev->mode_info.crtcs[i]->base.enabled) 2422 num_heads++; 2423 } 2424 for (i = 0; i < rdev->num_crtc; i += 2) { 2425 mode0 = &rdev->mode_info.crtcs[i]->base.mode; 2426 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode; 2427 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1); 2428 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 2429 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0); 2430 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads); 2431 } 2432 } 2433 2434 /** 2435 * evergreen_mc_wait_for_idle - wait for MC idle callback. 2436 * 2437 * @rdev: radeon_device pointer 2438 * 2439 * Wait for the MC (memory controller) to be idle. 2440 * (evergreen+). 2441 * Returns 0 if the MC is idle, -1 if not. 2442 */ 2443 int evergreen_mc_wait_for_idle(struct radeon_device *rdev) 2444 { 2445 unsigned i; 2446 u32 tmp; 2447 2448 for (i = 0; i < rdev->usec_timeout; i++) { 2449 /* read MC_STATUS */ 2450 tmp = RREG32(SRBM_STATUS) & 0x1F00; 2451 if (!tmp) 2452 return 0; 2453 udelay(1); 2454 } 2455 return -1; 2456 } 2457 2458 /* 2459 * GART 2460 */ 2461 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev) 2462 { 2463 unsigned i; 2464 u32 tmp; 2465 2466 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 2467 2468 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1)); 2469 for (i = 0; i < rdev->usec_timeout; i++) { 2470 /* read MC_STATUS */ 2471 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE); 2472 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT; 2473 if (tmp == 2) { 2474 printk(KERN_WARNING "[drm] r600 flush TLB failed\n"); 2475 return; 2476 } 2477 if (tmp) { 2478 return; 2479 } 2480 udelay(1); 2481 } 2482 } 2483 2484 static int evergreen_pcie_gart_enable(struct radeon_device *rdev) 2485 { 2486 u32 tmp; 2487 int r; 2488 2489 if (rdev->gart.robj == NULL) { 2490 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 2491 return -EINVAL; 2492 } 2493 r = radeon_gart_table_vram_pin(rdev); 2494 if (r) 2495 return r; 2496 /* Setup L2 cache */ 2497 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 2498 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 2499 EFFECTIVE_L2_QUEUE_SIZE(7)); 2500 WREG32(VM_L2_CNTL2, 0); 2501 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2502 /* Setup TLB control */ 2503 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 2504 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 2505 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 2506 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2507 if (rdev->flags & RADEON_IS_IGP) { 2508 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp); 2509 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp); 2510 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp); 2511 } else { 2512 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2513 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2514 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2515 if ((rdev->family == CHIP_JUNIPER) || 2516 (rdev->family == CHIP_CYPRESS) || 2517 (rdev->family == CHIP_HEMLOCK) || 2518 (rdev->family == CHIP_BARTS)) 2519 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp); 2520 } 2521 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2522 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2523 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2524 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2525 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 2526 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 2527 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 2528 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 2529 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT); 2530 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 2531 (u32)(rdev->dummy_page.addr >> 12)); 2532 WREG32(VM_CONTEXT1_CNTL, 0); 2533 2534 evergreen_pcie_gart_tlb_flush(rdev); 2535 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 2536 (unsigned)(rdev->mc.gtt_size >> 20), 2537 (unsigned long long)rdev->gart.table_addr); 2538 rdev->gart.ready = true; 2539 return 0; 2540 } 2541 2542 static void evergreen_pcie_gart_disable(struct radeon_device *rdev) 2543 { 2544 u32 tmp; 2545 2546 /* Disable all tables */ 2547 WREG32(VM_CONTEXT0_CNTL, 0); 2548 WREG32(VM_CONTEXT1_CNTL, 0); 2549 2550 /* Setup L2 cache */ 2551 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING | 2552 EFFECTIVE_L2_QUEUE_SIZE(7)); 2553 WREG32(VM_L2_CNTL2, 0); 2554 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2555 /* Setup TLB control */ 2556 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2557 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2558 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2559 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2560 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2561 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2562 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2563 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2564 radeon_gart_table_vram_unpin(rdev); 2565 } 2566 2567 static void evergreen_pcie_gart_fini(struct radeon_device *rdev) 2568 { 2569 evergreen_pcie_gart_disable(rdev); 2570 radeon_gart_table_vram_free(rdev); 2571 radeon_gart_fini(rdev); 2572 } 2573 2574 2575 static void evergreen_agp_enable(struct radeon_device *rdev) 2576 { 2577 u32 tmp; 2578 2579 /* Setup L2 cache */ 2580 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 2581 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 2582 EFFECTIVE_L2_QUEUE_SIZE(7)); 2583 WREG32(VM_L2_CNTL2, 0); 2584 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2585 /* Setup TLB control */ 2586 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 2587 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 2588 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 2589 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2590 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2591 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2592 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2593 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2594 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2595 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2596 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2597 WREG32(VM_CONTEXT0_CNTL, 0); 2598 WREG32(VM_CONTEXT1_CNTL, 0); 2599 } 2600 2601 static const unsigned ni_dig_offsets[] = 2602 { 2603 NI_DIG0_REGISTER_OFFSET, 2604 NI_DIG1_REGISTER_OFFSET, 2605 NI_DIG2_REGISTER_OFFSET, 2606 NI_DIG3_REGISTER_OFFSET, 2607 NI_DIG4_REGISTER_OFFSET, 2608 NI_DIG5_REGISTER_OFFSET 2609 }; 2610 2611 static const unsigned ni_tx_offsets[] = 2612 { 2613 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1, 2614 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1, 2615 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1, 2616 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1, 2617 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1, 2618 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1 2619 }; 2620 2621 static const unsigned evergreen_dp_offsets[] = 2622 { 2623 EVERGREEN_DP0_REGISTER_OFFSET, 2624 EVERGREEN_DP1_REGISTER_OFFSET, 2625 EVERGREEN_DP2_REGISTER_OFFSET, 2626 EVERGREEN_DP3_REGISTER_OFFSET, 2627 EVERGREEN_DP4_REGISTER_OFFSET, 2628 EVERGREEN_DP5_REGISTER_OFFSET 2629 }; 2630 2631 2632 /* 2633 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc 2634 * We go from crtc to connector and it is not relible since it 2635 * should be an opposite direction .If crtc is enable then 2636 * find the dig_fe which selects this crtc and insure that it enable. 2637 * if such dig_fe is found then find dig_be which selects found dig_be and 2638 * insure that it enable and in DP_SST mode. 2639 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing 2640 * from dp symbols clocks . 2641 */ 2642 static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev, 2643 unsigned crtc_id, unsigned *ret_dig_fe) 2644 { 2645 unsigned i; 2646 unsigned dig_fe; 2647 unsigned dig_be; 2648 unsigned dig_en_be; 2649 unsigned uniphy_pll; 2650 unsigned digs_fe_selected; 2651 unsigned dig_be_mode; 2652 unsigned dig_fe_mask; 2653 bool is_enabled = false; 2654 bool found_crtc = false; 2655 2656 /* loop through all running dig_fe to find selected crtc */ 2657 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) { 2658 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]); 2659 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON && 2660 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) { 2661 /* found running pipe */ 2662 found_crtc = true; 2663 dig_fe_mask = 1 << i; 2664 dig_fe = i; 2665 break; 2666 } 2667 } 2668 2669 if (found_crtc) { 2670 /* loop through all running dig_be to find selected dig_fe */ 2671 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) { 2672 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]); 2673 /* if dig_fe_selected by dig_be? */ 2674 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be); 2675 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be); 2676 if (dig_fe_mask & digs_fe_selected && 2677 /* if dig_be in sst mode? */ 2678 dig_be_mode == NI_DIG_BE_DPSST) { 2679 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL + 2680 ni_dig_offsets[i]); 2681 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 + 2682 ni_tx_offsets[i]); 2683 /* dig_be enable and tx is running */ 2684 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE && 2685 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON && 2686 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) { 2687 is_enabled = true; 2688 *ret_dig_fe = dig_fe; 2689 break; 2690 } 2691 } 2692 } 2693 } 2694 2695 return is_enabled; 2696 } 2697 2698 /* 2699 * Blank dig when in dp sst mode 2700 * Dig ignores crtc timing 2701 */ 2702 static void evergreen_blank_dp_output(struct radeon_device *rdev, 2703 unsigned dig_fe) 2704 { 2705 unsigned stream_ctrl; 2706 unsigned fifo_ctrl; 2707 unsigned counter = 0; 2708 2709 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) { 2710 DRM_ERROR("invalid dig_fe %d\n", dig_fe); 2711 return; 2712 } 2713 2714 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2715 evergreen_dp_offsets[dig_fe]); 2716 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) { 2717 DRM_ERROR("dig %d , should be enable\n", dig_fe); 2718 return; 2719 } 2720 2721 stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE; 2722 WREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2723 evergreen_dp_offsets[dig_fe], stream_ctrl); 2724 2725 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2726 evergreen_dp_offsets[dig_fe]); 2727 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) { 2728 msleep(1); 2729 counter++; 2730 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2731 evergreen_dp_offsets[dig_fe]); 2732 } 2733 if (counter >= 32 ) 2734 DRM_ERROR("counter exceeds %d\n", counter); 2735 2736 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]); 2737 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET; 2738 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl); 2739 2740 } 2741 2742 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save) 2743 { 2744 u32 crtc_enabled, tmp, frame_count, blackout; 2745 int i, j; 2746 unsigned dig_fe; 2747 2748 bzero(save, sizeof(*save)); /* avoid gcc warning */ 2749 if (!ASIC_IS_NODCE(rdev)) { 2750 save->vga_render_control = RREG32(VGA_RENDER_CONTROL); 2751 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL); 2752 2753 /* disable VGA render */ 2754 WREG32(VGA_RENDER_CONTROL, 0); 2755 } 2756 /* blank the display controllers */ 2757 for (i = 0; i < rdev->num_crtc; i++) { 2758 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN; 2759 if (crtc_enabled) { 2760 save->crtc_enabled[i] = true; 2761 if (ASIC_IS_DCE6(rdev)) { 2762 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); 2763 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) { 2764 radeon_wait_for_vblank(rdev, i); 2765 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2766 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN; 2767 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); 2768 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2769 } 2770 } else { 2771 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2772 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) { 2773 radeon_wait_for_vblank(rdev, i); 2774 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2775 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 2776 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2777 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2778 } 2779 } 2780 /* wait for the next frame */ 2781 frame_count = radeon_get_vblank_counter(rdev, i); 2782 for (j = 0; j < rdev->usec_timeout; j++) { 2783 if (radeon_get_vblank_counter(rdev, i) != frame_count) 2784 break; 2785 udelay(1); 2786 } 2787 /*we should disable dig if it drives dp sst*/ 2788 /*but we are in radeon_device_init and the topology is unknown*/ 2789 /*and it is available after radeon_modeset_init*/ 2790 /*the following method radeon_atom_encoder_dpms_dig*/ 2791 /*does the job if we initialize it properly*/ 2792 /*for now we do it this manually*/ 2793 /**/ 2794 if (ASIC_IS_DCE5(rdev) && 2795 evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe)) 2796 evergreen_blank_dp_output(rdev, dig_fe); 2797 /*we could remove 6 lines below*/ 2798 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */ 2799 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2800 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2801 tmp &= ~EVERGREEN_CRTC_MASTER_EN; 2802 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2803 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2804 save->crtc_enabled[i] = false; 2805 /* ***** */ 2806 } else { 2807 save->crtc_enabled[i] = false; 2808 } 2809 } 2810 2811 radeon_mc_wait_for_idle(rdev); 2812 2813 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 2814 if ((blackout & BLACKOUT_MODE_MASK) != 1) { 2815 /* Block CPU access */ 2816 WREG32(BIF_FB_EN, 0); 2817 /* blackout the MC */ 2818 blackout &= ~BLACKOUT_MODE_MASK; 2819 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 2820 } 2821 /* wait for the MC to settle */ 2822 udelay(100); 2823 2824 /* lock double buffered regs */ 2825 for (i = 0; i < rdev->num_crtc; i++) { 2826 if (save->crtc_enabled[i]) { 2827 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2828 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) { 2829 tmp |= EVERGREEN_GRPH_UPDATE_LOCK; 2830 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp); 2831 } 2832 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]); 2833 if (!(tmp & 1)) { 2834 tmp |= 1; 2835 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp); 2836 } 2837 } 2838 } 2839 } 2840 2841 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save) 2842 { 2843 u32 tmp, frame_count; 2844 int i, j; 2845 2846 /* update crtc base addresses */ 2847 for (i = 0; i < rdev->num_crtc; i++) { 2848 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], 2849 upper_32_bits(rdev->mc.vram_start)); 2850 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], 2851 upper_32_bits(rdev->mc.vram_start)); 2852 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i], 2853 (u32)rdev->mc.vram_start); 2854 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i], 2855 (u32)rdev->mc.vram_start); 2856 } 2857 2858 if (!ASIC_IS_NODCE(rdev)) { 2859 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start)); 2860 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start); 2861 } 2862 2863 /* unlock regs and wait for update */ 2864 for (i = 0; i < rdev->num_crtc; i++) { 2865 if (save->crtc_enabled[i]) { 2866 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]); 2867 if ((tmp & 0x7) != 3) { 2868 tmp &= ~0x7; 2869 tmp |= 0x3; 2870 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp); 2871 } 2872 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2873 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) { 2874 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK; 2875 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp); 2876 } 2877 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]); 2878 if (tmp & 1) { 2879 tmp &= ~1; 2880 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp); 2881 } 2882 for (j = 0; j < rdev->usec_timeout; j++) { 2883 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2884 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0) 2885 break; 2886 udelay(1); 2887 } 2888 } 2889 } 2890 2891 /* unblackout the MC */ 2892 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL); 2893 tmp &= ~BLACKOUT_MODE_MASK; 2894 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp); 2895 /* allow CPU access */ 2896 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 2897 2898 for (i = 0; i < rdev->num_crtc; i++) { 2899 if (save->crtc_enabled[i]) { 2900 if (ASIC_IS_DCE6(rdev)) { 2901 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); 2902 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN; 2903 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2904 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); 2905 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2906 } else { 2907 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2908 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 2909 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2910 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2911 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2912 } 2913 /* wait for the next frame */ 2914 frame_count = radeon_get_vblank_counter(rdev, i); 2915 for (j = 0; j < rdev->usec_timeout; j++) { 2916 if (radeon_get_vblank_counter(rdev, i) != frame_count) 2917 break; 2918 udelay(1); 2919 } 2920 } 2921 } 2922 if (!ASIC_IS_NODCE(rdev)) { 2923 /* Unlock vga access */ 2924 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control); 2925 mdelay(1); 2926 WREG32(VGA_RENDER_CONTROL, save->vga_render_control); 2927 } 2928 } 2929 2930 void evergreen_mc_program(struct radeon_device *rdev) 2931 { 2932 struct evergreen_mc_save save; 2933 u32 tmp; 2934 int i, j; 2935 2936 /* Initialize HDP */ 2937 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 2938 WREG32((0x2c14 + j), 0x00000000); 2939 WREG32((0x2c18 + j), 0x00000000); 2940 WREG32((0x2c1c + j), 0x00000000); 2941 WREG32((0x2c20 + j), 0x00000000); 2942 WREG32((0x2c24 + j), 0x00000000); 2943 } 2944 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 2945 2946 evergreen_mc_stop(rdev, &save); 2947 if (evergreen_mc_wait_for_idle(rdev)) { 2948 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2949 } 2950 /* Lockout access through VGA aperture*/ 2951 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 2952 /* Update configuration */ 2953 if (rdev->flags & RADEON_IS_AGP) { 2954 if (rdev->mc.vram_start < rdev->mc.gtt_start) { 2955 /* VRAM before AGP */ 2956 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2957 rdev->mc.vram_start >> 12); 2958 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2959 rdev->mc.gtt_end >> 12); 2960 } else { 2961 /* VRAM after AGP */ 2962 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2963 rdev->mc.gtt_start >> 12); 2964 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2965 rdev->mc.vram_end >> 12); 2966 } 2967 } else { 2968 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2969 rdev->mc.vram_start >> 12); 2970 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2971 rdev->mc.vram_end >> 12); 2972 } 2973 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12); 2974 /* llano/ontario only */ 2975 if ((rdev->family == CHIP_PALM) || 2976 (rdev->family == CHIP_SUMO) || 2977 (rdev->family == CHIP_SUMO2)) { 2978 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF; 2979 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24; 2980 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20; 2981 WREG32(MC_FUS_VM_FB_OFFSET, tmp); 2982 } 2983 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 2984 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 2985 WREG32(MC_VM_FB_LOCATION, tmp); 2986 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 2987 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 2988 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 2989 if (rdev->flags & RADEON_IS_AGP) { 2990 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16); 2991 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16); 2992 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22); 2993 } else { 2994 WREG32(MC_VM_AGP_BASE, 0); 2995 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 2996 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 2997 } 2998 if (evergreen_mc_wait_for_idle(rdev)) { 2999 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3000 } 3001 evergreen_mc_resume(rdev, &save); 3002 /* we need to own VRAM, so turn off the VGA renderer here 3003 * to stop it overwriting our objects */ 3004 rv515_vga_render_disable(rdev); 3005 } 3006 3007 /* 3008 * CP. 3009 */ 3010 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 3011 { 3012 struct radeon_ring *ring = &rdev->ring[ib->ring]; 3013 u32 next_rptr; 3014 3015 /* set to DX10/11 mode */ 3016 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0)); 3017 radeon_ring_write(ring, 1); 3018 3019 if (ring->rptr_save_reg) { 3020 next_rptr = ring->wptr + 3 + 4; 3021 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3022 radeon_ring_write(ring, ((ring->rptr_save_reg - 3023 PACKET3_SET_CONFIG_REG_START) >> 2)); 3024 radeon_ring_write(ring, next_rptr); 3025 } else if (rdev->wb.enabled) { 3026 next_rptr = ring->wptr + 5 + 4; 3027 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3)); 3028 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 3029 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18)); 3030 radeon_ring_write(ring, next_rptr); 3031 radeon_ring_write(ring, 0); 3032 } 3033 3034 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 3035 radeon_ring_write(ring, 3036 #ifdef __BIG_ENDIAN 3037 (2 << 0) | 3038 #endif 3039 (ib->gpu_addr & 0xFFFFFFFC)); 3040 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF); 3041 radeon_ring_write(ring, ib->length_dw); 3042 } 3043 3044 3045 static int evergreen_cp_load_microcode(struct radeon_device *rdev) 3046 { 3047 const __be32 *fw_data; 3048 int i; 3049 3050 if (!rdev->me_fw || !rdev->pfp_fw) 3051 return -EINVAL; 3052 3053 r700_cp_stop(rdev); 3054 WREG32(CP_RB_CNTL, 3055 #ifdef __BIG_ENDIAN 3056 BUF_SWAP_32BIT | 3057 #endif 3058 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3)); 3059 3060 fw_data = (const __be32 *)rdev->pfp_fw->data; 3061 WREG32(CP_PFP_UCODE_ADDR, 0); 3062 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++) 3063 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 3064 WREG32(CP_PFP_UCODE_ADDR, 0); 3065 3066 fw_data = (const __be32 *)rdev->me_fw->data; 3067 WREG32(CP_ME_RAM_WADDR, 0); 3068 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++) 3069 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 3070 3071 WREG32(CP_PFP_UCODE_ADDR, 0); 3072 WREG32(CP_ME_RAM_WADDR, 0); 3073 WREG32(CP_ME_RAM_RADDR, 0); 3074 return 0; 3075 } 3076 3077 static int evergreen_cp_start(struct radeon_device *rdev) 3078 { 3079 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3080 int r, i; 3081 uint32_t cp_me; 3082 3083 r = radeon_ring_lock(rdev, ring, 7); 3084 if (r) { 3085 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3086 return r; 3087 } 3088 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5)); 3089 radeon_ring_write(ring, 0x1); 3090 radeon_ring_write(ring, 0x0); 3091 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1); 3092 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 3093 radeon_ring_write(ring, 0); 3094 radeon_ring_write(ring, 0); 3095 radeon_ring_unlock_commit(rdev, ring, false); 3096 3097 cp_me = 0xff; 3098 WREG32(CP_ME_CNTL, cp_me); 3099 3100 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19); 3101 if (r) { 3102 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3103 return r; 3104 } 3105 3106 /* setup clear context state */ 3107 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3108 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3109 3110 for (i = 0; i < evergreen_default_size; i++) 3111 radeon_ring_write(ring, evergreen_default_state[i]); 3112 3113 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3114 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3115 3116 /* set clear context state */ 3117 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3118 radeon_ring_write(ring, 0); 3119 3120 /* SQ_VTX_BASE_VTX_LOC */ 3121 radeon_ring_write(ring, 0xc0026f00); 3122 radeon_ring_write(ring, 0x00000000); 3123 radeon_ring_write(ring, 0x00000000); 3124 radeon_ring_write(ring, 0x00000000); 3125 3126 /* Clear consts */ 3127 radeon_ring_write(ring, 0xc0036f00); 3128 radeon_ring_write(ring, 0x00000bc4); 3129 radeon_ring_write(ring, 0xffffffff); 3130 radeon_ring_write(ring, 0xffffffff); 3131 radeon_ring_write(ring, 0xffffffff); 3132 3133 radeon_ring_write(ring, 0xc0026900); 3134 radeon_ring_write(ring, 0x00000316); 3135 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 3136 radeon_ring_write(ring, 0x00000010); /* */ 3137 3138 radeon_ring_unlock_commit(rdev, ring, false); 3139 3140 return 0; 3141 } 3142 3143 static int evergreen_cp_resume(struct radeon_device *rdev) 3144 { 3145 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3146 u32 tmp; 3147 u32 rb_bufsz; 3148 int r; 3149 3150 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */ 3151 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP | 3152 SOFT_RESET_PA | 3153 SOFT_RESET_SH | 3154 SOFT_RESET_VGT | 3155 SOFT_RESET_SPI | 3156 SOFT_RESET_SX)); 3157 RREG32(GRBM_SOFT_RESET); 3158 mdelay(15); 3159 WREG32(GRBM_SOFT_RESET, 0); 3160 RREG32(GRBM_SOFT_RESET); 3161 3162 /* Set ring buffer size */ 3163 rb_bufsz = order_base_2(ring->ring_size / 8); 3164 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3165 #ifdef __BIG_ENDIAN 3166 tmp |= BUF_SWAP_32BIT; 3167 #endif 3168 WREG32(CP_RB_CNTL, tmp); 3169 WREG32(CP_SEM_WAIT_TIMER, 0x0); 3170 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 3171 3172 /* Set the write pointer delay */ 3173 WREG32(CP_RB_WPTR_DELAY, 0); 3174 3175 /* Initialize the ring buffer's read and write pointers */ 3176 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA); 3177 WREG32(CP_RB_RPTR_WR, 0); 3178 ring->wptr = 0; 3179 WREG32(CP_RB_WPTR, ring->wptr); 3180 3181 /* set the wb address whether it's enabled or not */ 3182 WREG32(CP_RB_RPTR_ADDR, 3183 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC)); 3184 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 3185 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 3186 3187 if (rdev->wb.enabled) 3188 WREG32(SCRATCH_UMSK, 0xff); 3189 else { 3190 tmp |= RB_NO_UPDATE; 3191 WREG32(SCRATCH_UMSK, 0); 3192 } 3193 3194 mdelay(1); 3195 WREG32(CP_RB_CNTL, tmp); 3196 3197 WREG32(CP_RB_BASE, ring->gpu_addr >> 8); 3198 WREG32(CP_DEBUG, (1 << 27) | (1 << 28)); 3199 3200 evergreen_cp_start(rdev); 3201 ring->ready = true; 3202 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring); 3203 if (r) { 3204 ring->ready = false; 3205 return r; 3206 } 3207 return 0; 3208 } 3209 3210 /* 3211 * Core functions 3212 */ 3213 static void evergreen_gpu_init(struct radeon_device *rdev) 3214 { 3215 u32 gb_addr_config; 3216 u32 mc_shared_chmap, mc_arb_ramcfg; 3217 u32 sx_debug_1; 3218 u32 smx_dc_ctl0; 3219 u32 sq_config; 3220 u32 sq_lds_resource_mgmt; 3221 u32 sq_gpr_resource_mgmt_1; 3222 u32 sq_gpr_resource_mgmt_2; 3223 u32 sq_gpr_resource_mgmt_3; 3224 u32 sq_thread_resource_mgmt; 3225 u32 sq_thread_resource_mgmt_2; 3226 u32 sq_stack_resource_mgmt_1; 3227 u32 sq_stack_resource_mgmt_2; 3228 u32 sq_stack_resource_mgmt_3; 3229 u32 vgt_cache_invalidation; 3230 u32 hdp_host_path_cntl, tmp; 3231 u32 disabled_rb_mask; 3232 int i, j, ps_thread_count; 3233 3234 switch (rdev->family) { 3235 case CHIP_CYPRESS: 3236 case CHIP_HEMLOCK: 3237 rdev->config.evergreen.num_ses = 2; 3238 rdev->config.evergreen.max_pipes = 4; 3239 rdev->config.evergreen.max_tile_pipes = 8; 3240 rdev->config.evergreen.max_simds = 10; 3241 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3242 rdev->config.evergreen.max_gprs = 256; 3243 rdev->config.evergreen.max_threads = 248; 3244 rdev->config.evergreen.max_gs_threads = 32; 3245 rdev->config.evergreen.max_stack_entries = 512; 3246 rdev->config.evergreen.sx_num_of_sets = 4; 3247 rdev->config.evergreen.sx_max_export_size = 256; 3248 rdev->config.evergreen.sx_max_export_pos_size = 64; 3249 rdev->config.evergreen.sx_max_export_smx_size = 192; 3250 rdev->config.evergreen.max_hw_contexts = 8; 3251 rdev->config.evergreen.sq_num_cf_insts = 2; 3252 3253 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3254 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3255 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3256 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN; 3257 break; 3258 case CHIP_JUNIPER: 3259 rdev->config.evergreen.num_ses = 1; 3260 rdev->config.evergreen.max_pipes = 4; 3261 rdev->config.evergreen.max_tile_pipes = 4; 3262 rdev->config.evergreen.max_simds = 10; 3263 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3264 rdev->config.evergreen.max_gprs = 256; 3265 rdev->config.evergreen.max_threads = 248; 3266 rdev->config.evergreen.max_gs_threads = 32; 3267 rdev->config.evergreen.max_stack_entries = 512; 3268 rdev->config.evergreen.sx_num_of_sets = 4; 3269 rdev->config.evergreen.sx_max_export_size = 256; 3270 rdev->config.evergreen.sx_max_export_pos_size = 64; 3271 rdev->config.evergreen.sx_max_export_smx_size = 192; 3272 rdev->config.evergreen.max_hw_contexts = 8; 3273 rdev->config.evergreen.sq_num_cf_insts = 2; 3274 3275 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3276 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3277 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3278 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN; 3279 break; 3280 case CHIP_REDWOOD: 3281 rdev->config.evergreen.num_ses = 1; 3282 rdev->config.evergreen.max_pipes = 4; 3283 rdev->config.evergreen.max_tile_pipes = 4; 3284 rdev->config.evergreen.max_simds = 5; 3285 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3286 rdev->config.evergreen.max_gprs = 256; 3287 rdev->config.evergreen.max_threads = 248; 3288 rdev->config.evergreen.max_gs_threads = 32; 3289 rdev->config.evergreen.max_stack_entries = 256; 3290 rdev->config.evergreen.sx_num_of_sets = 4; 3291 rdev->config.evergreen.sx_max_export_size = 256; 3292 rdev->config.evergreen.sx_max_export_pos_size = 64; 3293 rdev->config.evergreen.sx_max_export_smx_size = 192; 3294 rdev->config.evergreen.max_hw_contexts = 8; 3295 rdev->config.evergreen.sq_num_cf_insts = 2; 3296 3297 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3298 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3299 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3300 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN; 3301 break; 3302 case CHIP_CEDAR: 3303 default: 3304 rdev->config.evergreen.num_ses = 1; 3305 rdev->config.evergreen.max_pipes = 2; 3306 rdev->config.evergreen.max_tile_pipes = 2; 3307 rdev->config.evergreen.max_simds = 2; 3308 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3309 rdev->config.evergreen.max_gprs = 256; 3310 rdev->config.evergreen.max_threads = 192; 3311 rdev->config.evergreen.max_gs_threads = 16; 3312 rdev->config.evergreen.max_stack_entries = 256; 3313 rdev->config.evergreen.sx_num_of_sets = 4; 3314 rdev->config.evergreen.sx_max_export_size = 128; 3315 rdev->config.evergreen.sx_max_export_pos_size = 32; 3316 rdev->config.evergreen.sx_max_export_smx_size = 96; 3317 rdev->config.evergreen.max_hw_contexts = 4; 3318 rdev->config.evergreen.sq_num_cf_insts = 1; 3319 3320 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3321 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3322 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3323 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN; 3324 break; 3325 case CHIP_PALM: 3326 rdev->config.evergreen.num_ses = 1; 3327 rdev->config.evergreen.max_pipes = 2; 3328 rdev->config.evergreen.max_tile_pipes = 2; 3329 rdev->config.evergreen.max_simds = 2; 3330 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3331 rdev->config.evergreen.max_gprs = 256; 3332 rdev->config.evergreen.max_threads = 192; 3333 rdev->config.evergreen.max_gs_threads = 16; 3334 rdev->config.evergreen.max_stack_entries = 256; 3335 rdev->config.evergreen.sx_num_of_sets = 4; 3336 rdev->config.evergreen.sx_max_export_size = 128; 3337 rdev->config.evergreen.sx_max_export_pos_size = 32; 3338 rdev->config.evergreen.sx_max_export_smx_size = 96; 3339 rdev->config.evergreen.max_hw_contexts = 4; 3340 rdev->config.evergreen.sq_num_cf_insts = 1; 3341 3342 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3343 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3344 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3345 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN; 3346 break; 3347 case CHIP_SUMO: 3348 rdev->config.evergreen.num_ses = 1; 3349 rdev->config.evergreen.max_pipes = 4; 3350 rdev->config.evergreen.max_tile_pipes = 4; 3351 if (rdev->pdev->device == 0x9648) 3352 rdev->config.evergreen.max_simds = 3; 3353 else if ((rdev->pdev->device == 0x9647) || 3354 (rdev->pdev->device == 0x964a)) 3355 rdev->config.evergreen.max_simds = 4; 3356 else 3357 rdev->config.evergreen.max_simds = 5; 3358 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3359 rdev->config.evergreen.max_gprs = 256; 3360 rdev->config.evergreen.max_threads = 248; 3361 rdev->config.evergreen.max_gs_threads = 32; 3362 rdev->config.evergreen.max_stack_entries = 256; 3363 rdev->config.evergreen.sx_num_of_sets = 4; 3364 rdev->config.evergreen.sx_max_export_size = 256; 3365 rdev->config.evergreen.sx_max_export_pos_size = 64; 3366 rdev->config.evergreen.sx_max_export_smx_size = 192; 3367 rdev->config.evergreen.max_hw_contexts = 8; 3368 rdev->config.evergreen.sq_num_cf_insts = 2; 3369 3370 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3371 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3372 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3373 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN; 3374 break; 3375 case CHIP_SUMO2: 3376 rdev->config.evergreen.num_ses = 1; 3377 rdev->config.evergreen.max_pipes = 4; 3378 rdev->config.evergreen.max_tile_pipes = 4; 3379 rdev->config.evergreen.max_simds = 2; 3380 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3381 rdev->config.evergreen.max_gprs = 256; 3382 rdev->config.evergreen.max_threads = 248; 3383 rdev->config.evergreen.max_gs_threads = 32; 3384 rdev->config.evergreen.max_stack_entries = 512; 3385 rdev->config.evergreen.sx_num_of_sets = 4; 3386 rdev->config.evergreen.sx_max_export_size = 256; 3387 rdev->config.evergreen.sx_max_export_pos_size = 64; 3388 rdev->config.evergreen.sx_max_export_smx_size = 192; 3389 rdev->config.evergreen.max_hw_contexts = 4; 3390 rdev->config.evergreen.sq_num_cf_insts = 2; 3391 3392 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3393 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3394 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3395 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN; 3396 break; 3397 case CHIP_BARTS: 3398 rdev->config.evergreen.num_ses = 2; 3399 rdev->config.evergreen.max_pipes = 4; 3400 rdev->config.evergreen.max_tile_pipes = 8; 3401 rdev->config.evergreen.max_simds = 7; 3402 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3403 rdev->config.evergreen.max_gprs = 256; 3404 rdev->config.evergreen.max_threads = 248; 3405 rdev->config.evergreen.max_gs_threads = 32; 3406 rdev->config.evergreen.max_stack_entries = 512; 3407 rdev->config.evergreen.sx_num_of_sets = 4; 3408 rdev->config.evergreen.sx_max_export_size = 256; 3409 rdev->config.evergreen.sx_max_export_pos_size = 64; 3410 rdev->config.evergreen.sx_max_export_smx_size = 192; 3411 rdev->config.evergreen.max_hw_contexts = 8; 3412 rdev->config.evergreen.sq_num_cf_insts = 2; 3413 3414 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3415 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3416 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3417 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN; 3418 break; 3419 case CHIP_TURKS: 3420 rdev->config.evergreen.num_ses = 1; 3421 rdev->config.evergreen.max_pipes = 4; 3422 rdev->config.evergreen.max_tile_pipes = 4; 3423 rdev->config.evergreen.max_simds = 6; 3424 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3425 rdev->config.evergreen.max_gprs = 256; 3426 rdev->config.evergreen.max_threads = 248; 3427 rdev->config.evergreen.max_gs_threads = 32; 3428 rdev->config.evergreen.max_stack_entries = 256; 3429 rdev->config.evergreen.sx_num_of_sets = 4; 3430 rdev->config.evergreen.sx_max_export_size = 256; 3431 rdev->config.evergreen.sx_max_export_pos_size = 64; 3432 rdev->config.evergreen.sx_max_export_smx_size = 192; 3433 rdev->config.evergreen.max_hw_contexts = 8; 3434 rdev->config.evergreen.sq_num_cf_insts = 2; 3435 3436 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3437 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3438 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3439 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN; 3440 break; 3441 case CHIP_CAICOS: 3442 rdev->config.evergreen.num_ses = 1; 3443 rdev->config.evergreen.max_pipes = 2; 3444 rdev->config.evergreen.max_tile_pipes = 2; 3445 rdev->config.evergreen.max_simds = 2; 3446 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3447 rdev->config.evergreen.max_gprs = 256; 3448 rdev->config.evergreen.max_threads = 192; 3449 rdev->config.evergreen.max_gs_threads = 16; 3450 rdev->config.evergreen.max_stack_entries = 256; 3451 rdev->config.evergreen.sx_num_of_sets = 4; 3452 rdev->config.evergreen.sx_max_export_size = 128; 3453 rdev->config.evergreen.sx_max_export_pos_size = 32; 3454 rdev->config.evergreen.sx_max_export_smx_size = 96; 3455 rdev->config.evergreen.max_hw_contexts = 4; 3456 rdev->config.evergreen.sq_num_cf_insts = 1; 3457 3458 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3459 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3460 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3461 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN; 3462 break; 3463 } 3464 3465 /* Initialize HDP */ 3466 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3467 WREG32((0x2c14 + j), 0x00000000); 3468 WREG32((0x2c18 + j), 0x00000000); 3469 WREG32((0x2c1c + j), 0x00000000); 3470 WREG32((0x2c20 + j), 0x00000000); 3471 WREG32((0x2c24 + j), 0x00000000); 3472 } 3473 3474 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 3475 WREG32(SRBM_INT_CNTL, 0x1); 3476 WREG32(SRBM_INT_ACK, 0x1); 3477 3478 evergreen_fix_pci_max_read_req_size(rdev); 3479 3480 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 3481 if ((rdev->family == CHIP_PALM) || 3482 (rdev->family == CHIP_SUMO) || 3483 (rdev->family == CHIP_SUMO2)) 3484 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG); 3485 else 3486 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3487 3488 /* setup tiling info dword. gb_addr_config is not adequate since it does 3489 * not have bank info, so create a custom tiling dword. 3490 * bits 3:0 num_pipes 3491 * bits 7:4 num_banks 3492 * bits 11:8 group_size 3493 * bits 15:12 row_size 3494 */ 3495 rdev->config.evergreen.tile_config = 0; 3496 switch (rdev->config.evergreen.max_tile_pipes) { 3497 case 1: 3498 default: 3499 rdev->config.evergreen.tile_config |= (0 << 0); 3500 break; 3501 case 2: 3502 rdev->config.evergreen.tile_config |= (1 << 0); 3503 break; 3504 case 4: 3505 rdev->config.evergreen.tile_config |= (2 << 0); 3506 break; 3507 case 8: 3508 rdev->config.evergreen.tile_config |= (3 << 0); 3509 break; 3510 } 3511 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */ 3512 if (rdev->flags & RADEON_IS_IGP) 3513 rdev->config.evergreen.tile_config |= 1 << 4; 3514 else { 3515 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) { 3516 case 0: /* four banks */ 3517 rdev->config.evergreen.tile_config |= 0 << 4; 3518 break; 3519 case 1: /* eight banks */ 3520 rdev->config.evergreen.tile_config |= 1 << 4; 3521 break; 3522 case 2: /* sixteen banks */ 3523 default: 3524 rdev->config.evergreen.tile_config |= 2 << 4; 3525 break; 3526 } 3527 } 3528 rdev->config.evergreen.tile_config |= 0 << 8; 3529 rdev->config.evergreen.tile_config |= 3530 ((gb_addr_config & 0x30000000) >> 28) << 12; 3531 3532 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) { 3533 u32 efuse_straps_4; 3534 u32 efuse_straps_3; 3535 3536 efuse_straps_4 = RREG32_RCU(0x204); 3537 efuse_straps_3 = RREG32_RCU(0x203); 3538 tmp = (((efuse_straps_4 & 0xf) << 4) | 3539 ((efuse_straps_3 & 0xf0000000) >> 28)); 3540 } else { 3541 tmp = 0; 3542 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) { 3543 u32 rb_disable_bitmap; 3544 3545 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3546 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3547 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16; 3548 tmp <<= 4; 3549 tmp |= rb_disable_bitmap; 3550 } 3551 } 3552 /* enabled rb are just the one not disabled :) */ 3553 disabled_rb_mask = tmp; 3554 tmp = 0; 3555 for (i = 0; i < rdev->config.evergreen.max_backends; i++) 3556 tmp |= (1 << i); 3557 /* if all the backends are disabled, fix it up here */ 3558 if ((disabled_rb_mask & tmp) == tmp) { 3559 for (i = 0; i < rdev->config.evergreen.max_backends; i++) 3560 disabled_rb_mask &= ~(1 << i); 3561 } 3562 3563 for (i = 0; i < rdev->config.evergreen.num_ses; i++) { 3564 u32 simd_disable_bitmap; 3565 3566 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3567 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3568 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; 3569 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds; 3570 tmp <<= 16; 3571 tmp |= simd_disable_bitmap; 3572 } 3573 rdev->config.evergreen.active_simds = hweight32(~tmp); 3574 3575 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 3576 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 3577 3578 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3579 WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 3580 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3581 WREG32(DMA_TILING_CONFIG, gb_addr_config); 3582 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3583 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3584 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3585 3586 if ((rdev->config.evergreen.max_backends == 1) && 3587 (rdev->flags & RADEON_IS_IGP)) { 3588 if ((disabled_rb_mask & 3) == 1) { 3589 /* RB0 disabled, RB1 enabled */ 3590 tmp = 0x11111111; 3591 } else { 3592 /* RB1 disabled, RB0 enabled */ 3593 tmp = 0x00000000; 3594 } 3595 } else { 3596 tmp = gb_addr_config & NUM_PIPES_MASK; 3597 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends, 3598 EVERGREEN_MAX_BACKENDS, disabled_rb_mask); 3599 } 3600 WREG32(GB_BACKEND_MAP, tmp); 3601 3602 WREG32(CGTS_SYS_TCC_DISABLE, 0); 3603 WREG32(CGTS_TCC_DISABLE, 0); 3604 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0); 3605 WREG32(CGTS_USER_TCC_DISABLE, 0); 3606 3607 /* set HW defaults for 3D engine */ 3608 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | 3609 ROQ_IB2_START(0x2b))); 3610 3611 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30)); 3612 3613 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | 3614 SYNC_GRADIENT | 3615 SYNC_WALKER | 3616 SYNC_ALIGNER)); 3617 3618 sx_debug_1 = RREG32(SX_DEBUG_1); 3619 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS; 3620 WREG32(SX_DEBUG_1, sx_debug_1); 3621 3622 3623 smx_dc_ctl0 = RREG32(SMX_DC_CTL0); 3624 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff); 3625 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets); 3626 WREG32(SMX_DC_CTL0, smx_dc_ctl0); 3627 3628 if (rdev->family <= CHIP_SUMO2) 3629 WREG32(SMX_SAR_CTL0, 0x00010000); 3630 3631 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) | 3632 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) | 3633 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1))); 3634 3635 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) | 3636 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) | 3637 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size))); 3638 3639 WREG32(VGT_NUM_INSTANCES, 1); 3640 WREG32(SPI_CONFIG_CNTL, 0); 3641 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3642 WREG32(CP_PERFMON_CNTL, 0); 3643 3644 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) | 3645 FETCH_FIFO_HIWATER(0x4) | 3646 DONE_FIFO_HIWATER(0xe0) | 3647 ALU_UPDATE_FIFO_HIWATER(0x8))); 3648 3649 sq_config = RREG32(SQ_CONFIG); 3650 sq_config &= ~(PS_PRIO(3) | 3651 VS_PRIO(3) | 3652 GS_PRIO(3) | 3653 ES_PRIO(3)); 3654 sq_config |= (VC_ENABLE | 3655 EXPORT_SRC_C | 3656 PS_PRIO(0) | 3657 VS_PRIO(1) | 3658 GS_PRIO(2) | 3659 ES_PRIO(3)); 3660 3661 switch (rdev->family) { 3662 case CHIP_CEDAR: 3663 case CHIP_PALM: 3664 case CHIP_SUMO: 3665 case CHIP_SUMO2: 3666 case CHIP_CAICOS: 3667 /* no vertex cache */ 3668 sq_config &= ~VC_ENABLE; 3669 break; 3670 default: 3671 break; 3672 } 3673 3674 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT); 3675 3676 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32); 3677 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32); 3678 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4); 3679 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 3680 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 3681 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 3682 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 3683 3684 switch (rdev->family) { 3685 case CHIP_CEDAR: 3686 case CHIP_PALM: 3687 case CHIP_SUMO: 3688 case CHIP_SUMO2: 3689 ps_thread_count = 96; 3690 break; 3691 default: 3692 ps_thread_count = 128; 3693 break; 3694 } 3695 3696 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count); 3697 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3698 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3699 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3700 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3701 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3702 3703 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3704 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3705 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3706 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3707 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3708 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3709 3710 WREG32(SQ_CONFIG, sq_config); 3711 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1); 3712 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2); 3713 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3); 3714 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt); 3715 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2); 3716 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1); 3717 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2); 3718 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3); 3719 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0); 3720 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt); 3721 3722 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3723 FORCE_EOV_MAX_REZ_CNT(255))); 3724 3725 switch (rdev->family) { 3726 case CHIP_CEDAR: 3727 case CHIP_PALM: 3728 case CHIP_SUMO: 3729 case CHIP_SUMO2: 3730 case CHIP_CAICOS: 3731 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY); 3732 break; 3733 default: 3734 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC); 3735 break; 3736 } 3737 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO); 3738 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation); 3739 3740 WREG32(VGT_GS_VERTEX_REUSE, 16); 3741 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0); 3742 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3743 3744 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14); 3745 WREG32(VGT_OUT_DEALLOC_CNTL, 16); 3746 3747 WREG32(CB_PERF_CTR0_SEL_0, 0); 3748 WREG32(CB_PERF_CTR0_SEL_1, 0); 3749 WREG32(CB_PERF_CTR1_SEL_0, 0); 3750 WREG32(CB_PERF_CTR1_SEL_1, 0); 3751 WREG32(CB_PERF_CTR2_SEL_0, 0); 3752 WREG32(CB_PERF_CTR2_SEL_1, 0); 3753 WREG32(CB_PERF_CTR3_SEL_0, 0); 3754 WREG32(CB_PERF_CTR3_SEL_1, 0); 3755 3756 /* clear render buffer base addresses */ 3757 WREG32(CB_COLOR0_BASE, 0); 3758 WREG32(CB_COLOR1_BASE, 0); 3759 WREG32(CB_COLOR2_BASE, 0); 3760 WREG32(CB_COLOR3_BASE, 0); 3761 WREG32(CB_COLOR4_BASE, 0); 3762 WREG32(CB_COLOR5_BASE, 0); 3763 WREG32(CB_COLOR6_BASE, 0); 3764 WREG32(CB_COLOR7_BASE, 0); 3765 WREG32(CB_COLOR8_BASE, 0); 3766 WREG32(CB_COLOR9_BASE, 0); 3767 WREG32(CB_COLOR10_BASE, 0); 3768 WREG32(CB_COLOR11_BASE, 0); 3769 3770 /* set the shader const cache sizes to 0 */ 3771 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4) 3772 WREG32(i, 0); 3773 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4) 3774 WREG32(i, 0); 3775 3776 tmp = RREG32(HDP_MISC_CNTL); 3777 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3778 WREG32(HDP_MISC_CNTL, tmp); 3779 3780 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3781 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3782 3783 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3784 3785 udelay(50); 3786 3787 } 3788 3789 int evergreen_mc_init(struct radeon_device *rdev) 3790 { 3791 u32 tmp; 3792 int chansize, numchan; 3793 3794 /* Get VRAM informations */ 3795 rdev->mc.vram_is_ddr = true; 3796 if ((rdev->family == CHIP_PALM) || 3797 (rdev->family == CHIP_SUMO) || 3798 (rdev->family == CHIP_SUMO2)) 3799 tmp = RREG32(FUS_MC_ARB_RAMCFG); 3800 else 3801 tmp = RREG32(MC_ARB_RAMCFG); 3802 if (tmp & CHANSIZE_OVERRIDE) { 3803 chansize = 16; 3804 } else if (tmp & CHANSIZE_MASK) { 3805 chansize = 64; 3806 } else { 3807 chansize = 32; 3808 } 3809 tmp = RREG32(MC_SHARED_CHMAP); 3810 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 3811 case 0: 3812 default: 3813 numchan = 1; 3814 break; 3815 case 1: 3816 numchan = 2; 3817 break; 3818 case 2: 3819 numchan = 4; 3820 break; 3821 case 3: 3822 numchan = 8; 3823 break; 3824 } 3825 rdev->mc.vram_width = numchan * chansize; 3826 /* Could aper size report 0 ? */ 3827 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 3828 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 3829 /* Setup GPU memory space */ 3830 if ((rdev->family == CHIP_PALM) || 3831 (rdev->family == CHIP_SUMO) || 3832 (rdev->family == CHIP_SUMO2)) { 3833 /* size in bytes on fusion */ 3834 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE); 3835 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE); 3836 } else { 3837 /* size in MB on evergreen/cayman/tn */ 3838 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3839 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3840 } 3841 rdev->mc.visible_vram_size = rdev->mc.aper_size; 3842 r700_vram_gtt_location(rdev, &rdev->mc); 3843 radeon_update_bandwidth_info(rdev); 3844 3845 return 0; 3846 } 3847 3848 void evergreen_print_gpu_status_regs(struct radeon_device *rdev) 3849 { 3850 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n", 3851 RREG32(GRBM_STATUS)); 3852 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n", 3853 RREG32(GRBM_STATUS_SE0)); 3854 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n", 3855 RREG32(GRBM_STATUS_SE1)); 3856 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n", 3857 RREG32(SRBM_STATUS)); 3858 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n", 3859 RREG32(SRBM_STATUS2)); 3860 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n", 3861 RREG32(CP_STALLED_STAT1)); 3862 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n", 3863 RREG32(CP_STALLED_STAT2)); 3864 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n", 3865 RREG32(CP_BUSY_STAT)); 3866 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n", 3867 RREG32(CP_STAT)); 3868 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", 3869 RREG32(DMA_STATUS_REG)); 3870 if (rdev->family >= CHIP_CAYMAN) { 3871 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n", 3872 RREG32(DMA_STATUS_REG + 0x800)); 3873 } 3874 } 3875 3876 bool evergreen_is_display_hung(struct radeon_device *rdev) 3877 { 3878 u32 crtc_hung = 0; 3879 u32 crtc_status[6]; 3880 u32 i, j, tmp; 3881 3882 for (i = 0; i < rdev->num_crtc; i++) { 3883 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) { 3884 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); 3885 crtc_hung |= (1 << i); 3886 } 3887 } 3888 3889 for (j = 0; j < 10; j++) { 3890 for (i = 0; i < rdev->num_crtc; i++) { 3891 if (crtc_hung & (1 << i)) { 3892 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); 3893 if (tmp != crtc_status[i]) 3894 crtc_hung &= ~(1 << i); 3895 } 3896 } 3897 if (crtc_hung == 0) 3898 return false; 3899 udelay(100); 3900 } 3901 3902 return true; 3903 } 3904 3905 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev) 3906 { 3907 u32 reset_mask = 0; 3908 u32 tmp; 3909 3910 /* GRBM_STATUS */ 3911 tmp = RREG32(GRBM_STATUS); 3912 if (tmp & (PA_BUSY | SC_BUSY | 3913 SH_BUSY | SX_BUSY | 3914 TA_BUSY | VGT_BUSY | 3915 DB_BUSY | CB_BUSY | 3916 SPI_BUSY | VGT_BUSY_NO_DMA)) 3917 reset_mask |= RADEON_RESET_GFX; 3918 3919 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | 3920 CP_BUSY | CP_COHERENCY_BUSY)) 3921 reset_mask |= RADEON_RESET_CP; 3922 3923 if (tmp & GRBM_EE_BUSY) 3924 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; 3925 3926 /* DMA_STATUS_REG */ 3927 tmp = RREG32(DMA_STATUS_REG); 3928 if (!(tmp & DMA_IDLE)) 3929 reset_mask |= RADEON_RESET_DMA; 3930 3931 /* SRBM_STATUS2 */ 3932 tmp = RREG32(SRBM_STATUS2); 3933 if (tmp & DMA_BUSY) 3934 reset_mask |= RADEON_RESET_DMA; 3935 3936 /* SRBM_STATUS */ 3937 tmp = RREG32(SRBM_STATUS); 3938 if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) 3939 reset_mask |= RADEON_RESET_RLC; 3940 3941 if (tmp & IH_BUSY) 3942 reset_mask |= RADEON_RESET_IH; 3943 3944 if (tmp & SEM_BUSY) 3945 reset_mask |= RADEON_RESET_SEM; 3946 3947 if (tmp & GRBM_RQ_PENDING) 3948 reset_mask |= RADEON_RESET_GRBM; 3949 3950 if (tmp & VMC_BUSY) 3951 reset_mask |= RADEON_RESET_VMC; 3952 3953 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 3954 MCC_BUSY | MCD_BUSY)) 3955 reset_mask |= RADEON_RESET_MC; 3956 3957 if (evergreen_is_display_hung(rdev)) 3958 reset_mask |= RADEON_RESET_DISPLAY; 3959 3960 /* VM_L2_STATUS */ 3961 tmp = RREG32(VM_L2_STATUS); 3962 if (tmp & L2_BUSY) 3963 reset_mask |= RADEON_RESET_VMC; 3964 3965 /* Skip MC reset as it's mostly likely not hung, just busy */ 3966 if (reset_mask & RADEON_RESET_MC) { 3967 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 3968 reset_mask &= ~RADEON_RESET_MC; 3969 } 3970 3971 return reset_mask; 3972 } 3973 3974 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 3975 { 3976 struct evergreen_mc_save save; 3977 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 3978 u32 tmp; 3979 3980 if (reset_mask == 0) 3981 return; 3982 3983 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 3984 3985 evergreen_print_gpu_status_regs(rdev); 3986 3987 /* Disable CP parsing/prefetching */ 3988 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 3989 3990 if (reset_mask & RADEON_RESET_DMA) { 3991 /* Disable DMA */ 3992 tmp = RREG32(DMA_RB_CNTL); 3993 tmp &= ~DMA_RB_ENABLE; 3994 WREG32(DMA_RB_CNTL, tmp); 3995 } 3996 3997 udelay(50); 3998 3999 evergreen_mc_stop(rdev, &save); 4000 if (evergreen_mc_wait_for_idle(rdev)) { 4001 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 4002 } 4003 4004 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) { 4005 grbm_soft_reset |= SOFT_RESET_DB | 4006 SOFT_RESET_CB | 4007 SOFT_RESET_PA | 4008 SOFT_RESET_SC | 4009 SOFT_RESET_SPI | 4010 SOFT_RESET_SX | 4011 SOFT_RESET_SH | 4012 SOFT_RESET_TC | 4013 SOFT_RESET_TA | 4014 SOFT_RESET_VC | 4015 SOFT_RESET_VGT; 4016 } 4017 4018 if (reset_mask & RADEON_RESET_CP) { 4019 grbm_soft_reset |= SOFT_RESET_CP | 4020 SOFT_RESET_VGT; 4021 4022 srbm_soft_reset |= SOFT_RESET_GRBM; 4023 } 4024 4025 if (reset_mask & RADEON_RESET_DMA) 4026 srbm_soft_reset |= SOFT_RESET_DMA; 4027 4028 if (reset_mask & RADEON_RESET_DISPLAY) 4029 srbm_soft_reset |= SOFT_RESET_DC; 4030 4031 if (reset_mask & RADEON_RESET_RLC) 4032 srbm_soft_reset |= SOFT_RESET_RLC; 4033 4034 if (reset_mask & RADEON_RESET_SEM) 4035 srbm_soft_reset |= SOFT_RESET_SEM; 4036 4037 if (reset_mask & RADEON_RESET_IH) 4038 srbm_soft_reset |= SOFT_RESET_IH; 4039 4040 if (reset_mask & RADEON_RESET_GRBM) 4041 srbm_soft_reset |= SOFT_RESET_GRBM; 4042 4043 if (reset_mask & RADEON_RESET_VMC) 4044 srbm_soft_reset |= SOFT_RESET_VMC; 4045 4046 if (!(rdev->flags & RADEON_IS_IGP)) { 4047 if (reset_mask & RADEON_RESET_MC) 4048 srbm_soft_reset |= SOFT_RESET_MC; 4049 } 4050 4051 if (grbm_soft_reset) { 4052 tmp = RREG32(GRBM_SOFT_RESET); 4053 tmp |= grbm_soft_reset; 4054 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4055 WREG32(GRBM_SOFT_RESET, tmp); 4056 tmp = RREG32(GRBM_SOFT_RESET); 4057 4058 udelay(50); 4059 4060 tmp &= ~grbm_soft_reset; 4061 WREG32(GRBM_SOFT_RESET, tmp); 4062 tmp = RREG32(GRBM_SOFT_RESET); 4063 } 4064 4065 if (srbm_soft_reset) { 4066 tmp = RREG32(SRBM_SOFT_RESET); 4067 tmp |= srbm_soft_reset; 4068 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 4069 WREG32(SRBM_SOFT_RESET, tmp); 4070 tmp = RREG32(SRBM_SOFT_RESET); 4071 4072 udelay(50); 4073 4074 tmp &= ~srbm_soft_reset; 4075 WREG32(SRBM_SOFT_RESET, tmp); 4076 tmp = RREG32(SRBM_SOFT_RESET); 4077 } 4078 4079 /* Wait a little for things to settle down */ 4080 udelay(50); 4081 4082 evergreen_mc_resume(rdev, &save); 4083 udelay(50); 4084 4085 evergreen_print_gpu_status_regs(rdev); 4086 } 4087 4088 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev) 4089 { 4090 struct evergreen_mc_save save; 4091 u32 tmp, i; 4092 4093 dev_info(rdev->dev, "GPU pci config reset\n"); 4094 4095 /* disable dpm? */ 4096 4097 /* Disable CP parsing/prefetching */ 4098 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 4099 udelay(50); 4100 /* Disable DMA */ 4101 tmp = RREG32(DMA_RB_CNTL); 4102 tmp &= ~DMA_RB_ENABLE; 4103 WREG32(DMA_RB_CNTL, tmp); 4104 /* XXX other engines? */ 4105 4106 /* halt the rlc */ 4107 r600_rlc_stop(rdev); 4108 4109 udelay(50); 4110 4111 /* set mclk/sclk to bypass */ 4112 rv770_set_clk_bypass_mode(rdev); 4113 /* disable BM */ 4114 pci_disable_busmaster(rdev->pdev->dev.bsddev); 4115 /* disable mem access */ 4116 evergreen_mc_stop(rdev, &save); 4117 if (evergreen_mc_wait_for_idle(rdev)) { 4118 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 4119 } 4120 /* reset */ 4121 radeon_pci_config_reset(rdev); 4122 /* wait for asic to come out of reset */ 4123 for (i = 0; i < rdev->usec_timeout; i++) { 4124 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 4125 break; 4126 udelay(1); 4127 } 4128 } 4129 4130 int evergreen_asic_reset(struct radeon_device *rdev, bool hard) 4131 { 4132 u32 reset_mask; 4133 4134 if (hard) { 4135 evergreen_gpu_pci_config_reset(rdev); 4136 return 0; 4137 } 4138 4139 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4140 4141 if (reset_mask) 4142 r600_set_bios_scratch_engine_hung(rdev, true); 4143 4144 /* try soft reset */ 4145 evergreen_gpu_soft_reset(rdev, reset_mask); 4146 4147 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4148 4149 /* try pci config reset */ 4150 if (reset_mask && radeon_hard_reset) 4151 evergreen_gpu_pci_config_reset(rdev); 4152 4153 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4154 4155 if (!reset_mask) 4156 r600_set_bios_scratch_engine_hung(rdev, false); 4157 4158 return 0; 4159 } 4160 4161 /** 4162 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up 4163 * 4164 * @rdev: radeon_device pointer 4165 * @ring: radeon_ring structure holding ring information 4166 * 4167 * Check if the GFX engine is locked up. 4168 * Returns true if the engine appears to be locked up, false if not. 4169 */ 4170 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 4171 { 4172 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4173 4174 if (!(reset_mask & (RADEON_RESET_GFX | 4175 RADEON_RESET_COMPUTE | 4176 RADEON_RESET_CP))) { 4177 radeon_ring_lockup_update(rdev, ring); 4178 return false; 4179 } 4180 return radeon_ring_test_lockup(rdev, ring); 4181 } 4182 4183 /* 4184 * RLC 4185 */ 4186 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000 4187 #define RLC_CLEAR_STATE_END_MARKER 0x00000001 4188 4189 void sumo_rlc_fini(struct radeon_device *rdev) 4190 { 4191 int r; 4192 4193 /* save restore block */ 4194 if (rdev->rlc.save_restore_obj) { 4195 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 4196 if (unlikely(r != 0)) 4197 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r); 4198 radeon_bo_unpin(rdev->rlc.save_restore_obj); 4199 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4200 4201 radeon_bo_unref(&rdev->rlc.save_restore_obj); 4202 rdev->rlc.save_restore_obj = NULL; 4203 } 4204 4205 /* clear state block */ 4206 if (rdev->rlc.clear_state_obj) { 4207 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 4208 if (unlikely(r != 0)) 4209 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r); 4210 radeon_bo_unpin(rdev->rlc.clear_state_obj); 4211 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4212 4213 radeon_bo_unref(&rdev->rlc.clear_state_obj); 4214 rdev->rlc.clear_state_obj = NULL; 4215 } 4216 4217 /* clear state block */ 4218 if (rdev->rlc.cp_table_obj) { 4219 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); 4220 if (unlikely(r != 0)) 4221 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); 4222 radeon_bo_unpin(rdev->rlc.cp_table_obj); 4223 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4224 4225 radeon_bo_unref(&rdev->rlc.cp_table_obj); 4226 rdev->rlc.cp_table_obj = NULL; 4227 } 4228 } 4229 4230 #define CP_ME_TABLE_SIZE 96 4231 4232 int sumo_rlc_init(struct radeon_device *rdev) 4233 { 4234 const u32 *src_ptr; 4235 volatile u32 *dst_ptr; 4236 u32 dws, data, i, j, k, reg_num; 4237 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0; 4238 u64 reg_list_mc_addr; 4239 const struct cs_section_def *cs_data; 4240 int r; 4241 void *vptr; 4242 4243 vptr = NULL; 4244 src_ptr = rdev->rlc.reg_list; 4245 dws = rdev->rlc.reg_list_size; 4246 if (rdev->family >= CHIP_BONAIRE) { 4247 dws += (5 * 16) + 48 + 48 + 64; 4248 } 4249 cs_data = rdev->rlc.cs_data; 4250 4251 if (src_ptr) { 4252 /* save restore block */ 4253 if (rdev->rlc.save_restore_obj == NULL) { 4254 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, 4255 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4256 &rdev->rlc.save_restore_obj); 4257 if (r) { 4258 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r); 4259 return r; 4260 } 4261 } 4262 4263 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 4264 if (unlikely(r != 0)) { 4265 sumo_rlc_fini(rdev); 4266 return r; 4267 } 4268 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM, 4269 &rdev->rlc.save_restore_gpu_addr); 4270 if (r) { 4271 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4272 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r); 4273 sumo_rlc_fini(rdev); 4274 return r; 4275 } 4276 4277 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void**)&vptr); 4278 if (r) { 4279 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r); 4280 sumo_rlc_fini(rdev); 4281 return r; 4282 } 4283 rdev->rlc.sr_ptr = vptr; 4284 /* write the sr buffer */ 4285 dst_ptr = rdev->rlc.sr_ptr; 4286 if (rdev->family >= CHIP_TAHITI) { 4287 /* SI */ 4288 for (i = 0; i < rdev->rlc.reg_list_size; i++) 4289 dst_ptr[i] = cpu_to_le32(src_ptr[i]); 4290 } else { 4291 /* ON/LN/TN */ 4292 /* format: 4293 * dw0: (reg2 << 16) | reg1 4294 * dw1: reg1 save space 4295 * dw2: reg2 save space 4296 */ 4297 for (i = 0; i < dws; i++) { 4298 data = src_ptr[i] >> 2; 4299 i++; 4300 if (i < dws) 4301 data |= (src_ptr[i] >> 2) << 16; 4302 j = (((i - 1) * 3) / 2); 4303 dst_ptr[j] = cpu_to_le32(data); 4304 } 4305 j = ((i * 3) / 2); 4306 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER); 4307 } 4308 radeon_bo_kunmap(rdev->rlc.save_restore_obj); 4309 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4310 } 4311 4312 if (cs_data) { 4313 /* clear state block */ 4314 if (rdev->family >= CHIP_BONAIRE) { 4315 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev); 4316 } else if (rdev->family >= CHIP_TAHITI) { 4317 rdev->rlc.clear_state_size = si_get_csb_size(rdev); 4318 dws = rdev->rlc.clear_state_size + (256 / 4); 4319 } else { 4320 reg_list_num = 0; 4321 dws = 0; 4322 for (i = 0; cs_data[i].section != NULL; i++) { 4323 for (j = 0; cs_data[i].section[j].extent != NULL; j++) { 4324 reg_list_num++; 4325 dws += cs_data[i].section[j].reg_count; 4326 } 4327 } 4328 reg_list_blk_index = (3 * reg_list_num + 2); 4329 dws += reg_list_blk_index; 4330 rdev->rlc.clear_state_size = dws; 4331 } 4332 4333 if (rdev->rlc.clear_state_obj == NULL) { 4334 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, 4335 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4336 &rdev->rlc.clear_state_obj); 4337 if (r) { 4338 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r); 4339 sumo_rlc_fini(rdev); 4340 return r; 4341 } 4342 } 4343 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 4344 if (unlikely(r != 0)) { 4345 sumo_rlc_fini(rdev); 4346 return r; 4347 } 4348 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM, 4349 &rdev->rlc.clear_state_gpu_addr); 4350 if (r) { 4351 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4352 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r); 4353 sumo_rlc_fini(rdev); 4354 return r; 4355 } 4356 4357 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void**)&vptr); 4358 if (r) { 4359 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r); 4360 sumo_rlc_fini(rdev); 4361 return r; 4362 } 4363 rdev->rlc.cs_ptr = vptr; 4364 /* set up the cs buffer */ 4365 dst_ptr = rdev->rlc.cs_ptr; 4366 if (rdev->family >= CHIP_BONAIRE) { 4367 cik_get_csb_buffer(rdev, dst_ptr); 4368 } else if (rdev->family >= CHIP_TAHITI) { 4369 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256; 4370 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr)); 4371 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr)); 4372 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size); 4373 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]); 4374 } else { 4375 reg_list_hdr_blk_index = 0; 4376 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4); 4377 data = upper_32_bits(reg_list_mc_addr); 4378 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4379 reg_list_hdr_blk_index++; 4380 for (i = 0; cs_data[i].section != NULL; i++) { 4381 for (j = 0; cs_data[i].section[j].extent != NULL; j++) { 4382 reg_num = cs_data[i].section[j].reg_count; 4383 data = reg_list_mc_addr & 0xffffffff; 4384 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4385 reg_list_hdr_blk_index++; 4386 4387 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff; 4388 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4389 reg_list_hdr_blk_index++; 4390 4391 data = 0x08000000 | (reg_num * 4); 4392 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4393 reg_list_hdr_blk_index++; 4394 4395 for (k = 0; k < reg_num; k++) { 4396 data = cs_data[i].section[j].extent[k]; 4397 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data); 4398 } 4399 reg_list_mc_addr += reg_num * 4; 4400 reg_list_blk_index += reg_num; 4401 } 4402 } 4403 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER); 4404 } 4405 radeon_bo_kunmap(rdev->rlc.clear_state_obj); 4406 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4407 } 4408 4409 if (rdev->rlc.cp_table_size) { 4410 if (rdev->rlc.cp_table_obj == NULL) { 4411 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, 4412 PAGE_SIZE, true, 4413 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4414 &rdev->rlc.cp_table_obj); 4415 if (r) { 4416 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r); 4417 sumo_rlc_fini(rdev); 4418 return r; 4419 } 4420 } 4421 4422 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); 4423 if (unlikely(r != 0)) { 4424 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); 4425 sumo_rlc_fini(rdev); 4426 return r; 4427 } 4428 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM, 4429 &rdev->rlc.cp_table_gpu_addr); 4430 if (r) { 4431 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4432 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r); 4433 sumo_rlc_fini(rdev); 4434 return r; 4435 } 4436 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void**)&vptr); 4437 if (r) { 4438 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r); 4439 sumo_rlc_fini(rdev); 4440 return r; 4441 } 4442 rdev->rlc.cp_table_ptr = vptr; 4443 4444 cik_init_cp_pg_table(rdev); 4445 4446 radeon_bo_kunmap(rdev->rlc.cp_table_obj); 4447 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4448 4449 } 4450 4451 return 0; 4452 } 4453 4454 static void evergreen_rlc_start(struct radeon_device *rdev) 4455 { 4456 u32 mask = RLC_ENABLE; 4457 4458 if (rdev->flags & RADEON_IS_IGP) { 4459 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC; 4460 } 4461 4462 WREG32(RLC_CNTL, mask); 4463 } 4464 4465 int evergreen_rlc_resume(struct radeon_device *rdev) 4466 { 4467 u32 i; 4468 const __be32 *fw_data; 4469 4470 if (!rdev->rlc_fw) 4471 return -EINVAL; 4472 4473 r600_rlc_stop(rdev); 4474 4475 WREG32(RLC_HB_CNTL, 0); 4476 4477 if (rdev->flags & RADEON_IS_IGP) { 4478 if (rdev->family == CHIP_ARUBA) { 4479 u32 always_on_bitmap = 4480 3 | (3 << (16 * rdev->config.cayman.max_shader_engines)); 4481 /* find out the number of active simds */ 4482 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; 4483 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se; 4484 tmp = hweight32(~tmp); 4485 if (tmp == rdev->config.cayman.max_simds_per_se) { 4486 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap); 4487 WREG32(TN_RLC_LB_PARAMS, 0x00601004); 4488 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff); 4489 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000); 4490 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000); 4491 } 4492 } else { 4493 WREG32(RLC_HB_WPTR_LSB_ADDR, 0); 4494 WREG32(RLC_HB_WPTR_MSB_ADDR, 0); 4495 } 4496 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 4497 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 4498 } else { 4499 WREG32(RLC_HB_BASE, 0); 4500 WREG32(RLC_HB_RPTR, 0); 4501 WREG32(RLC_HB_WPTR, 0); 4502 WREG32(RLC_HB_WPTR_LSB_ADDR, 0); 4503 WREG32(RLC_HB_WPTR_MSB_ADDR, 0); 4504 } 4505 WREG32(RLC_MC_CNTL, 0); 4506 WREG32(RLC_UCODE_CNTL, 0); 4507 4508 fw_data = (const __be32 *)rdev->rlc_fw->data; 4509 if (rdev->family >= CHIP_ARUBA) { 4510 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) { 4511 WREG32(RLC_UCODE_ADDR, i); 4512 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4513 } 4514 } else if (rdev->family >= CHIP_CAYMAN) { 4515 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) { 4516 WREG32(RLC_UCODE_ADDR, i); 4517 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4518 } 4519 } else { 4520 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) { 4521 WREG32(RLC_UCODE_ADDR, i); 4522 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4523 } 4524 } 4525 WREG32(RLC_UCODE_ADDR, 0); 4526 4527 evergreen_rlc_start(rdev); 4528 4529 return 0; 4530 } 4531 4532 /* Interrupts */ 4533 4534 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc) 4535 { 4536 if (crtc >= rdev->num_crtc) 4537 return 0; 4538 else 4539 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]); 4540 } 4541 4542 void evergreen_disable_interrupt_state(struct radeon_device *rdev) 4543 { 4544 u32 tmp; 4545 4546 if (rdev->family >= CHIP_CAYMAN) { 4547 cayman_cp_int_cntl_setup(rdev, 0, 4548 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4549 cayman_cp_int_cntl_setup(rdev, 1, 0); 4550 cayman_cp_int_cntl_setup(rdev, 2, 0); 4551 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; 4552 WREG32(CAYMAN_DMA1_CNTL, tmp); 4553 } else 4554 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4555 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE; 4556 WREG32(DMA_CNTL, tmp); 4557 WREG32(GRBM_INT_CNTL, 0); 4558 WREG32(SRBM_INT_CNTL, 0); 4559 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 4560 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 4561 if (rdev->num_crtc >= 4) { 4562 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 4563 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 4564 } 4565 if (rdev->num_crtc >= 6) { 4566 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 4567 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 4568 } 4569 4570 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 4571 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 4572 if (rdev->num_crtc >= 4) { 4573 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 4574 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 4575 } 4576 if (rdev->num_crtc >= 6) { 4577 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 4578 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 4579 } 4580 4581 /* only one DAC on DCE5 */ 4582 if (!ASIC_IS_DCE5(rdev)) 4583 WREG32(DACA_AUTODETECT_INT_CONTROL, 0); 4584 WREG32(DACB_AUTODETECT_INT_CONTROL, 0); 4585 4586 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4587 WREG32(DC_HPD1_INT_CONTROL, tmp); 4588 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4589 WREG32(DC_HPD2_INT_CONTROL, tmp); 4590 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4591 WREG32(DC_HPD3_INT_CONTROL, tmp); 4592 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4593 WREG32(DC_HPD4_INT_CONTROL, tmp); 4594 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4595 WREG32(DC_HPD5_INT_CONTROL, tmp); 4596 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4597 WREG32(DC_HPD6_INT_CONTROL, tmp); 4598 4599 } 4600 4601 int evergreen_irq_set(struct radeon_device *rdev) 4602 { 4603 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; 4604 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; 4605 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 4606 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 4607 u32 grbm_int_cntl = 0; 4608 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0; 4609 u32 dma_cntl, dma_cntl1 = 0; 4610 u32 thermal_int = 0; 4611 4612 if (!rdev->irq.installed) { 4613 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 4614 return -EINVAL; 4615 } 4616 /* don't enable anything if the ih is disabled */ 4617 if (!rdev->ih.enabled) { 4618 r600_disable_interrupts(rdev); 4619 /* force the active interrupt state to all disabled */ 4620 evergreen_disable_interrupt_state(rdev); 4621 return 0; 4622 } 4623 4624 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 4625 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 4626 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 4627 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 4628 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 4629 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 4630 if (rdev->family == CHIP_ARUBA) 4631 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) & 4632 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 4633 else 4634 thermal_int = RREG32(CG_THERMAL_INT) & 4635 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 4636 4637 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4638 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4639 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4640 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4641 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4642 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4643 4644 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE; 4645 4646 if (rdev->family >= CHIP_CAYMAN) { 4647 /* enable CP interrupts on all rings */ 4648 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 4649 DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); 4650 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 4651 } 4652 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 4653 DRM_DEBUG("evergreen_irq_set: sw int cp1\n"); 4654 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE; 4655 } 4656 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 4657 DRM_DEBUG("evergreen_irq_set: sw int cp2\n"); 4658 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE; 4659 } 4660 } else { 4661 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 4662 DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); 4663 cp_int_cntl |= RB_INT_ENABLE; 4664 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 4665 } 4666 } 4667 4668 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 4669 DRM_DEBUG("r600_irq_set: sw int dma\n"); 4670 dma_cntl |= TRAP_ENABLE; 4671 } 4672 4673 if (rdev->family >= CHIP_CAYMAN) { 4674 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; 4675 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 4676 DRM_DEBUG("r600_irq_set: sw int dma1\n"); 4677 dma_cntl1 |= TRAP_ENABLE; 4678 } 4679 } 4680 4681 if (rdev->irq.dpm_thermal) { 4682 DRM_DEBUG("dpm thermal\n"); 4683 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; 4684 } 4685 4686 if (rdev->irq.crtc_vblank_int[0] || 4687 atomic_read(&rdev->irq.pflip[0])) { 4688 DRM_DEBUG("evergreen_irq_set: vblank 0\n"); 4689 crtc1 |= VBLANK_INT_MASK; 4690 } 4691 if (rdev->irq.crtc_vblank_int[1] || 4692 atomic_read(&rdev->irq.pflip[1])) { 4693 DRM_DEBUG("evergreen_irq_set: vblank 1\n"); 4694 crtc2 |= VBLANK_INT_MASK; 4695 } 4696 if (rdev->irq.crtc_vblank_int[2] || 4697 atomic_read(&rdev->irq.pflip[2])) { 4698 DRM_DEBUG("evergreen_irq_set: vblank 2\n"); 4699 crtc3 |= VBLANK_INT_MASK; 4700 } 4701 if (rdev->irq.crtc_vblank_int[3] || 4702 atomic_read(&rdev->irq.pflip[3])) { 4703 DRM_DEBUG("evergreen_irq_set: vblank 3\n"); 4704 crtc4 |= VBLANK_INT_MASK; 4705 } 4706 if (rdev->irq.crtc_vblank_int[4] || 4707 atomic_read(&rdev->irq.pflip[4])) { 4708 DRM_DEBUG("evergreen_irq_set: vblank 4\n"); 4709 crtc5 |= VBLANK_INT_MASK; 4710 } 4711 if (rdev->irq.crtc_vblank_int[5] || 4712 atomic_read(&rdev->irq.pflip[5])) { 4713 DRM_DEBUG("evergreen_irq_set: vblank 5\n"); 4714 crtc6 |= VBLANK_INT_MASK; 4715 } 4716 if (rdev->irq.hpd[0]) { 4717 DRM_DEBUG("evergreen_irq_set: hpd 1\n"); 4718 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 4719 } 4720 if (rdev->irq.hpd[1]) { 4721 DRM_DEBUG("evergreen_irq_set: hpd 2\n"); 4722 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 4723 } 4724 if (rdev->irq.hpd[2]) { 4725 DRM_DEBUG("evergreen_irq_set: hpd 3\n"); 4726 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 4727 } 4728 if (rdev->irq.hpd[3]) { 4729 DRM_DEBUG("evergreen_irq_set: hpd 4\n"); 4730 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 4731 } 4732 if (rdev->irq.hpd[4]) { 4733 DRM_DEBUG("evergreen_irq_set: hpd 5\n"); 4734 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 4735 } 4736 if (rdev->irq.hpd[5]) { 4737 DRM_DEBUG("evergreen_irq_set: hpd 6\n"); 4738 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 4739 } 4740 if (rdev->irq.afmt[0]) { 4741 DRM_DEBUG("evergreen_irq_set: hdmi 0\n"); 4742 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4743 } 4744 if (rdev->irq.afmt[1]) { 4745 DRM_DEBUG("evergreen_irq_set: hdmi 1\n"); 4746 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4747 } 4748 if (rdev->irq.afmt[2]) { 4749 DRM_DEBUG("evergreen_irq_set: hdmi 2\n"); 4750 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4751 } 4752 if (rdev->irq.afmt[3]) { 4753 DRM_DEBUG("evergreen_irq_set: hdmi 3\n"); 4754 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4755 } 4756 if (rdev->irq.afmt[4]) { 4757 DRM_DEBUG("evergreen_irq_set: hdmi 4\n"); 4758 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4759 } 4760 if (rdev->irq.afmt[5]) { 4761 DRM_DEBUG("evergreen_irq_set: hdmi 5\n"); 4762 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4763 } 4764 4765 if (rdev->family >= CHIP_CAYMAN) { 4766 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl); 4767 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1); 4768 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2); 4769 } else 4770 WREG32(CP_INT_CNTL, cp_int_cntl); 4771 4772 WREG32(DMA_CNTL, dma_cntl); 4773 4774 if (rdev->family >= CHIP_CAYMAN) 4775 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1); 4776 4777 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 4778 4779 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 4780 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 4781 if (rdev->num_crtc >= 4) { 4782 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 4783 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 4784 } 4785 if (rdev->num_crtc >= 6) { 4786 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 4787 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 4788 } 4789 4790 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 4791 GRPH_PFLIP_INT_MASK); 4792 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 4793 GRPH_PFLIP_INT_MASK); 4794 if (rdev->num_crtc >= 4) { 4795 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 4796 GRPH_PFLIP_INT_MASK); 4797 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 4798 GRPH_PFLIP_INT_MASK); 4799 } 4800 if (rdev->num_crtc >= 6) { 4801 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 4802 GRPH_PFLIP_INT_MASK); 4803 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 4804 GRPH_PFLIP_INT_MASK); 4805 } 4806 4807 WREG32(DC_HPD1_INT_CONTROL, hpd1); 4808 WREG32(DC_HPD2_INT_CONTROL, hpd2); 4809 WREG32(DC_HPD3_INT_CONTROL, hpd3); 4810 WREG32(DC_HPD4_INT_CONTROL, hpd4); 4811 WREG32(DC_HPD5_INT_CONTROL, hpd5); 4812 WREG32(DC_HPD6_INT_CONTROL, hpd6); 4813 if (rdev->family == CHIP_ARUBA) 4814 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int); 4815 else 4816 WREG32(CG_THERMAL_INT, thermal_int); 4817 4818 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1); 4819 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2); 4820 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3); 4821 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4); 4822 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5); 4823 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6); 4824 4825 /* posting read */ 4826 RREG32(SRBM_STATUS); 4827 4828 return 0; 4829 } 4830 4831 static void evergreen_irq_ack(struct radeon_device *rdev) 4832 { 4833 u32 tmp; 4834 4835 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS); 4836 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 4837 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 4838 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 4839 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 4840 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 4841 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 4842 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 4843 if (rdev->num_crtc >= 4) { 4844 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 4845 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 4846 } 4847 if (rdev->num_crtc >= 6) { 4848 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 4849 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 4850 } 4851 4852 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 4853 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 4854 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 4855 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 4856 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 4857 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 4858 4859 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 4860 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4861 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 4862 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4863 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) 4864 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 4865 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) 4866 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 4867 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 4868 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 4869 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) 4870 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 4871 4872 if (rdev->num_crtc >= 4) { 4873 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 4874 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4875 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 4876 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4877 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 4878 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 4879 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 4880 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 4881 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 4882 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 4883 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 4884 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 4885 } 4886 4887 if (rdev->num_crtc >= 6) { 4888 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 4889 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4890 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 4891 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4892 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 4893 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 4894 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 4895 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 4896 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 4897 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 4898 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 4899 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 4900 } 4901 4902 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 4903 tmp = RREG32(DC_HPD1_INT_CONTROL); 4904 tmp |= DC_HPDx_INT_ACK; 4905 WREG32(DC_HPD1_INT_CONTROL, tmp); 4906 } 4907 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 4908 tmp = RREG32(DC_HPD2_INT_CONTROL); 4909 tmp |= DC_HPDx_INT_ACK; 4910 WREG32(DC_HPD2_INT_CONTROL, tmp); 4911 } 4912 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 4913 tmp = RREG32(DC_HPD3_INT_CONTROL); 4914 tmp |= DC_HPDx_INT_ACK; 4915 WREG32(DC_HPD3_INT_CONTROL, tmp); 4916 } 4917 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 4918 tmp = RREG32(DC_HPD4_INT_CONTROL); 4919 tmp |= DC_HPDx_INT_ACK; 4920 WREG32(DC_HPD4_INT_CONTROL, tmp); 4921 } 4922 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 4923 tmp = RREG32(DC_HPD5_INT_CONTROL); 4924 tmp |= DC_HPDx_INT_ACK; 4925 WREG32(DC_HPD5_INT_CONTROL, tmp); 4926 } 4927 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 4928 tmp = RREG32(DC_HPD5_INT_CONTROL); 4929 tmp |= DC_HPDx_INT_ACK; 4930 WREG32(DC_HPD6_INT_CONTROL, tmp); 4931 } 4932 4933 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) { 4934 tmp = RREG32(DC_HPD1_INT_CONTROL); 4935 tmp |= DC_HPDx_RX_INT_ACK; 4936 WREG32(DC_HPD1_INT_CONTROL, tmp); 4937 } 4938 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) { 4939 tmp = RREG32(DC_HPD2_INT_CONTROL); 4940 tmp |= DC_HPDx_RX_INT_ACK; 4941 WREG32(DC_HPD2_INT_CONTROL, tmp); 4942 } 4943 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) { 4944 tmp = RREG32(DC_HPD3_INT_CONTROL); 4945 tmp |= DC_HPDx_RX_INT_ACK; 4946 WREG32(DC_HPD3_INT_CONTROL, tmp); 4947 } 4948 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) { 4949 tmp = RREG32(DC_HPD4_INT_CONTROL); 4950 tmp |= DC_HPDx_RX_INT_ACK; 4951 WREG32(DC_HPD4_INT_CONTROL, tmp); 4952 } 4953 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) { 4954 tmp = RREG32(DC_HPD5_INT_CONTROL); 4955 tmp |= DC_HPDx_RX_INT_ACK; 4956 WREG32(DC_HPD5_INT_CONTROL, tmp); 4957 } 4958 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) { 4959 tmp = RREG32(DC_HPD5_INT_CONTROL); 4960 tmp |= DC_HPDx_RX_INT_ACK; 4961 WREG32(DC_HPD6_INT_CONTROL, tmp); 4962 } 4963 4964 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) { 4965 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET); 4966 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4967 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp); 4968 } 4969 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) { 4970 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET); 4971 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4972 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp); 4973 } 4974 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) { 4975 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET); 4976 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4977 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp); 4978 } 4979 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) { 4980 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET); 4981 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4982 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp); 4983 } 4984 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) { 4985 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET); 4986 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4987 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp); 4988 } 4989 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) { 4990 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET); 4991 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4992 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp); 4993 } 4994 } 4995 4996 static void evergreen_irq_disable(struct radeon_device *rdev) 4997 { 4998 r600_disable_interrupts(rdev); 4999 /* Wait and acknowledge irq */ 5000 mdelay(1); 5001 evergreen_irq_ack(rdev); 5002 evergreen_disable_interrupt_state(rdev); 5003 } 5004 5005 void evergreen_irq_suspend(struct radeon_device *rdev) 5006 { 5007 evergreen_irq_disable(rdev); 5008 r600_rlc_stop(rdev); 5009 } 5010 5011 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev) 5012 { 5013 u32 wptr, tmp; 5014 5015 if (rdev->wb.enabled) 5016 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 5017 else 5018 wptr = RREG32(IH_RB_WPTR); 5019 5020 if (wptr & RB_OVERFLOW) { 5021 wptr &= ~RB_OVERFLOW; 5022 /* When a ring buffer overflow happen start parsing interrupt 5023 * from the last not overwritten vector (wptr + 16). Hopefully 5024 * this should allow us to catchup. 5025 */ 5026 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", 5027 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); 5028 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 5029 tmp = RREG32(IH_RB_CNTL); 5030 tmp |= IH_WPTR_OVERFLOW_CLEAR; 5031 WREG32(IH_RB_CNTL, tmp); 5032 } 5033 return (wptr & rdev->ih.ptr_mask); 5034 } 5035 5036 irqreturn_t evergreen_irq_process(struct radeon_device *rdev) 5037 { 5038 u32 wptr; 5039 u32 rptr; 5040 u32 src_id, src_data; 5041 u32 ring_index; 5042 bool queue_hotplug = false; 5043 bool queue_hdmi = false; 5044 bool queue_dp = false; 5045 bool queue_thermal = false; 5046 u32 status, addr; 5047 5048 if (!rdev->ih.enabled || rdev->shutdown) 5049 return IRQ_NONE; 5050 5051 wptr = evergreen_get_ih_wptr(rdev); 5052 5053 restart_ih: 5054 /* is somebody else already processing irqs? */ 5055 if (atomic_xchg(&rdev->ih.lock, 1)) 5056 return IRQ_NONE; 5057 5058 rptr = rdev->ih.rptr; 5059 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 5060 5061 /* Order reading of wptr vs. reading of IH ring data */ 5062 rmb(); 5063 5064 /* display interrupts */ 5065 evergreen_irq_ack(rdev); 5066 5067 while (rptr != wptr) { 5068 /* wptr/rptr are in bytes! */ 5069 ring_index = rptr / 4; 5070 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 5071 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 5072 5073 switch (src_id) { 5074 case 1: /* D1 vblank/vline */ 5075 switch (src_data) { 5076 case 0: /* D1 vblank */ 5077 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)) 5078 DRM_DEBUG("IH: D1 vblank - IH event w/o asserted irq bit?\n"); 5079 5080 if (rdev->irq.crtc_vblank_int[0]) { 5081 drm_handle_vblank(rdev->ddev, 0); 5082 rdev->pm.vblank_sync = true; 5083 wake_up(&rdev->irq.vblank_queue); 5084 } 5085 if (atomic_read(&rdev->irq.pflip[0])) 5086 radeon_crtc_handle_vblank(rdev, 0); 5087 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 5088 DRM_DEBUG("IH: D1 vblank\n"); 5089 5090 break; 5091 case 1: /* D1 vline */ 5092 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)) 5093 DRM_DEBUG("IH: D1 vline - IH event w/o asserted irq bit?\n"); 5094 5095 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; 5096 DRM_DEBUG("IH: D1 vline\n"); 5097 5098 break; 5099 default: 5100 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5101 break; 5102 } 5103 break; 5104 case 2: /* D2 vblank/vline */ 5105 switch (src_data) { 5106 case 0: /* D2 vblank */ 5107 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)) 5108 DRM_DEBUG("IH: D2 vblank - IH event w/o asserted irq bit?\n"); 5109 5110 if (rdev->irq.crtc_vblank_int[1]) { 5111 drm_handle_vblank(rdev->ddev, 1); 5112 rdev->pm.vblank_sync = true; 5113 wake_up(&rdev->irq.vblank_queue); 5114 } 5115 if (atomic_read(&rdev->irq.pflip[1])) 5116 radeon_crtc_handle_vblank(rdev, 1); 5117 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 5118 DRM_DEBUG("IH: D2 vblank\n"); 5119 5120 break; 5121 case 1: /* D2 vline */ 5122 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)) 5123 DRM_DEBUG("IH: D2 vline - IH event w/o asserted irq bit?\n"); 5124 5125 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 5126 DRM_DEBUG("IH: D2 vline\n"); 5127 5128 break; 5129 default: 5130 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5131 break; 5132 } 5133 break; 5134 case 3: /* D3 vblank/vline */ 5135 switch (src_data) { 5136 case 0: /* D3 vblank */ 5137 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)) 5138 DRM_DEBUG("IH: D3 vblank - IH event w/o asserted irq bit?\n"); 5139 5140 if (rdev->irq.crtc_vblank_int[2]) { 5141 drm_handle_vblank(rdev->ddev, 2); 5142 rdev->pm.vblank_sync = true; 5143 wake_up(&rdev->irq.vblank_queue); 5144 } 5145 if (atomic_read(&rdev->irq.pflip[2])) 5146 radeon_crtc_handle_vblank(rdev, 2); 5147 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 5148 DRM_DEBUG("IH: D3 vblank\n"); 5149 5150 break; 5151 case 1: /* D3 vline */ 5152 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)) 5153 DRM_DEBUG("IH: D3 vline - IH event w/o asserted irq bit?\n"); 5154 5155 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 5156 DRM_DEBUG("IH: D3 vline\n"); 5157 5158 break; 5159 default: 5160 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5161 break; 5162 } 5163 break; 5164 case 4: /* D4 vblank/vline */ 5165 switch (src_data) { 5166 case 0: /* D4 vblank */ 5167 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)) 5168 DRM_DEBUG("IH: D4 vblank - IH event w/o asserted irq bit?\n"); 5169 5170 if (rdev->irq.crtc_vblank_int[3]) { 5171 drm_handle_vblank(rdev->ddev, 3); 5172 rdev->pm.vblank_sync = true; 5173 wake_up(&rdev->irq.vblank_queue); 5174 } 5175 if (atomic_read(&rdev->irq.pflip[3])) 5176 radeon_crtc_handle_vblank(rdev, 3); 5177 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 5178 DRM_DEBUG("IH: D4 vblank\n"); 5179 5180 break; 5181 case 1: /* D4 vline */ 5182 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)) 5183 DRM_DEBUG("IH: D4 vline - IH event w/o asserted irq bit?\n"); 5184 5185 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 5186 DRM_DEBUG("IH: D4 vline\n"); 5187 5188 break; 5189 default: 5190 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5191 break; 5192 } 5193 break; 5194 case 5: /* D5 vblank/vline */ 5195 switch (src_data) { 5196 case 0: /* D5 vblank */ 5197 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)) 5198 DRM_DEBUG("IH: D5 vblank - IH event w/o asserted irq bit?\n"); 5199 5200 if (rdev->irq.crtc_vblank_int[4]) { 5201 drm_handle_vblank(rdev->ddev, 4); 5202 rdev->pm.vblank_sync = true; 5203 wake_up(&rdev->irq.vblank_queue); 5204 } 5205 if (atomic_read(&rdev->irq.pflip[4])) 5206 radeon_crtc_handle_vblank(rdev, 4); 5207 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 5208 DRM_DEBUG("IH: D5 vblank\n"); 5209 5210 break; 5211 case 1: /* D5 vline */ 5212 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)) 5213 DRM_DEBUG("IH: D5 vline - IH event w/o asserted irq bit?\n"); 5214 5215 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 5216 DRM_DEBUG("IH: D5 vline\n"); 5217 5218 break; 5219 default: 5220 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5221 break; 5222 } 5223 break; 5224 case 6: /* D6 vblank/vline */ 5225 switch (src_data) { 5226 case 0: /* D6 vblank */ 5227 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)) 5228 DRM_DEBUG("IH: D6 vblank - IH event w/o asserted irq bit?\n"); 5229 5230 if (rdev->irq.crtc_vblank_int[5]) { 5231 drm_handle_vblank(rdev->ddev, 5); 5232 rdev->pm.vblank_sync = true; 5233 wake_up(&rdev->irq.vblank_queue); 5234 } 5235 if (atomic_read(&rdev->irq.pflip[5])) 5236 radeon_crtc_handle_vblank(rdev, 5); 5237 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 5238 DRM_DEBUG("IH: D6 vblank\n"); 5239 5240 break; 5241 case 1: /* D6 vline */ 5242 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)) 5243 DRM_DEBUG("IH: D6 vline - IH event w/o asserted irq bit?\n"); 5244 5245 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 5246 DRM_DEBUG("IH: D6 vline\n"); 5247 5248 break; 5249 default: 5250 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5251 break; 5252 } 5253 break; 5254 case 8: /* D1 page flip */ 5255 case 10: /* D2 page flip */ 5256 case 12: /* D3 page flip */ 5257 case 14: /* D4 page flip */ 5258 case 16: /* D5 page flip */ 5259 case 18: /* D6 page flip */ 5260 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); 5261 if (radeon_use_pflipirq > 0) 5262 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); 5263 break; 5264 case 42: /* HPD hotplug */ 5265 switch (src_data) { 5266 case 0: 5267 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT)) 5268 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5269 5270 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; 5271 queue_hotplug = true; 5272 DRM_DEBUG("IH: HPD1\n"); 5273 break; 5274 case 1: 5275 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT)) 5276 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5277 5278 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; 5279 queue_hotplug = true; 5280 DRM_DEBUG("IH: HPD2\n"); 5281 break; 5282 case 2: 5283 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT)) 5284 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5285 5286 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 5287 queue_hotplug = true; 5288 DRM_DEBUG("IH: HPD3\n"); 5289 break; 5290 case 3: 5291 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT)) 5292 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5293 5294 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 5295 queue_hotplug = true; 5296 DRM_DEBUG("IH: HPD4\n"); 5297 break; 5298 case 4: 5299 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT)) 5300 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5301 5302 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 5303 queue_hotplug = true; 5304 DRM_DEBUG("IH: HPD5\n"); 5305 break; 5306 case 5: 5307 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT)) 5308 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5309 5310 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 5311 queue_hotplug = true; 5312 DRM_DEBUG("IH: HPD6\n"); 5313 break; 5314 case 6: 5315 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT)) 5316 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5317 5318 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT; 5319 queue_dp = true; 5320 DRM_DEBUG("IH: HPD_RX 1\n"); 5321 break; 5322 case 7: 5323 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT)) 5324 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5325 5326 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT; 5327 queue_dp = true; 5328 DRM_DEBUG("IH: HPD_RX 2\n"); 5329 break; 5330 case 8: 5331 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT)) 5332 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5333 5334 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT; 5335 queue_dp = true; 5336 DRM_DEBUG("IH: HPD_RX 3\n"); 5337 break; 5338 case 9: 5339 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT)) 5340 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5341 5342 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT; 5343 queue_dp = true; 5344 DRM_DEBUG("IH: HPD_RX 4\n"); 5345 break; 5346 case 10: 5347 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT)) 5348 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5349 5350 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT; 5351 queue_dp = true; 5352 DRM_DEBUG("IH: HPD_RX 5\n"); 5353 break; 5354 case 11: 5355 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT)) 5356 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5357 5358 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT; 5359 queue_dp = true; 5360 DRM_DEBUG("IH: HPD_RX 6\n"); 5361 break; 5362 default: 5363 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5364 break; 5365 } 5366 break; 5367 case 44: /* hdmi */ 5368 switch (src_data) { 5369 case 0: 5370 if (!(rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG)) 5371 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5372 5373 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG; 5374 queue_hdmi = true; 5375 DRM_DEBUG("IH: HDMI0\n"); 5376 break; 5377 case 1: 5378 if (!(rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG)) 5379 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5380 5381 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG; 5382 queue_hdmi = true; 5383 DRM_DEBUG("IH: HDMI1\n"); 5384 break; 5385 case 2: 5386 if (!(rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG)) 5387 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5388 5389 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG; 5390 queue_hdmi = true; 5391 DRM_DEBUG("IH: HDMI2\n"); 5392 break; 5393 case 3: 5394 if (!(rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG)) 5395 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5396 5397 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG; 5398 queue_hdmi = true; 5399 DRM_DEBUG("IH: HDMI3\n"); 5400 break; 5401 case 4: 5402 if (!(rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG)) 5403 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5404 5405 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG; 5406 queue_hdmi = true; 5407 DRM_DEBUG("IH: HDMI4\n"); 5408 break; 5409 case 5: 5410 if (!(rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG)) 5411 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5412 5413 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG; 5414 queue_hdmi = true; 5415 DRM_DEBUG("IH: HDMI5\n"); 5416 break; 5417 default: 5418 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); 5419 break; 5420 } 5421 case 96: 5422 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR)); 5423 WREG32(SRBM_INT_ACK, 0x1); 5424 break; 5425 case 124: /* UVD */ 5426 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 5427 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 5428 break; 5429 case 146: 5430 case 147: 5431 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 5432 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 5433 /* reset addr and status */ 5434 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 5435 if (addr == 0x0 && status == 0x0) 5436 break; 5437 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 5438 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 5439 addr); 5440 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 5441 status); 5442 cayman_vm_decode_fault(rdev, status, addr); 5443 break; 5444 case 176: /* CP_INT in ring buffer */ 5445 case 177: /* CP_INT in IB1 */ 5446 case 178: /* CP_INT in IB2 */ 5447 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data); 5448 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 5449 break; 5450 case 181: /* CP EOP event */ 5451 DRM_DEBUG("IH: CP EOP\n"); 5452 if (rdev->family >= CHIP_CAYMAN) { 5453 switch (src_data) { 5454 case 0: 5455 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 5456 break; 5457 case 1: 5458 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 5459 break; 5460 case 2: 5461 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 5462 break; 5463 } 5464 } else 5465 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 5466 break; 5467 case 224: /* DMA trap event */ 5468 DRM_DEBUG("IH: DMA trap\n"); 5469 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 5470 break; 5471 case 230: /* thermal low to high */ 5472 DRM_DEBUG("IH: thermal low to high\n"); 5473 rdev->pm.dpm.thermal.high_to_low = false; 5474 queue_thermal = true; 5475 break; 5476 case 231: /* thermal high to low */ 5477 DRM_DEBUG("IH: thermal high to low\n"); 5478 rdev->pm.dpm.thermal.high_to_low = true; 5479 queue_thermal = true; 5480 break; 5481 case 233: /* GUI IDLE */ 5482 DRM_DEBUG("IH: GUI idle\n"); 5483 break; 5484 case 244: /* DMA trap event */ 5485 if (rdev->family >= CHIP_CAYMAN) { 5486 DRM_DEBUG("IH: DMA1 trap\n"); 5487 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 5488 } 5489 break; 5490 default: 5491 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5492 break; 5493 } 5494 5495 /* wptr/rptr are in bytes! */ 5496 rptr += 16; 5497 rptr &= rdev->ih.ptr_mask; 5498 WREG32(IH_RB_RPTR, rptr); 5499 } 5500 if (queue_dp) 5501 schedule_work(&rdev->dp_work); 5502 if (queue_hotplug) 5503 taskqueue_enqueue(rdev->tq, &rdev->hotplug_work); 5504 if (queue_hdmi) 5505 taskqueue_enqueue(rdev->tq, &rdev->audio_work); 5506 if (queue_thermal && rdev->pm.dpm_enabled) 5507 taskqueue_enqueue(rdev->tq, &rdev->pm.dpm.thermal.work); 5508 rdev->ih.rptr = rptr; 5509 atomic_set(&rdev->ih.lock, 0); 5510 5511 /* make sure wptr hasn't changed while processing */ 5512 wptr = evergreen_get_ih_wptr(rdev); 5513 if (wptr != rptr) 5514 goto restart_ih; 5515 5516 return IRQ_HANDLED; 5517 } 5518 5519 static void evergreen_uvd_init(struct radeon_device *rdev) 5520 { 5521 int r; 5522 5523 if (!rdev->has_uvd) 5524 return; 5525 5526 r = radeon_uvd_init(rdev); 5527 if (r) { 5528 dev_err(rdev->dev, "failed UVD (%d) init.\n", r); 5529 /* 5530 * At this point rdev->uvd.vcpu_bo is NULL which trickles down 5531 * to early fails uvd_v2_2_resume() and thus nothing happens 5532 * there. So it is pointless to try to go through that code 5533 * hence why we disable uvd here. 5534 */ 5535 rdev->has_uvd = 0; 5536 return; 5537 } 5538 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL; 5539 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096); 5540 } 5541 5542 static void evergreen_uvd_start(struct radeon_device *rdev) 5543 { 5544 int r; 5545 5546 if (!rdev->has_uvd) 5547 return; 5548 5549 r = uvd_v2_2_resume(rdev); 5550 if (r) { 5551 dev_err(rdev->dev, "failed UVD resume (%d).\n", r); 5552 goto error; 5553 } 5554 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX); 5555 if (r) { 5556 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r); 5557 goto error; 5558 } 5559 return; 5560 5561 error: 5562 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 5563 } 5564 5565 static void evergreen_uvd_resume(struct radeon_device *rdev) 5566 { 5567 struct radeon_ring *ring; 5568 int r; 5569 5570 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size) 5571 return; 5572 5573 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 5574 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, RADEON_CP_PACKET2); 5575 if (r) { 5576 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r); 5577 return; 5578 } 5579 r = uvd_v1_0_init(rdev); 5580 if (r) { 5581 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r); 5582 return; 5583 } 5584 } 5585 5586 static int evergreen_startup(struct radeon_device *rdev) 5587 { 5588 struct radeon_ring *ring; 5589 int r; 5590 5591 /* enable pcie gen2 link */ 5592 evergreen_pcie_gen2_enable(rdev); 5593 /* enable aspm */ 5594 evergreen_program_aspm(rdev); 5595 5596 /* scratch needs to be initialized before MC */ 5597 r = r600_vram_scratch_init(rdev); 5598 if (r) 5599 return r; 5600 5601 evergreen_mc_program(rdev); 5602 5603 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) { 5604 r = ni_mc_load_microcode(rdev); 5605 if (r) { 5606 DRM_ERROR("Failed to load MC firmware!\n"); 5607 return r; 5608 } 5609 } 5610 5611 if (rdev->flags & RADEON_IS_AGP) { 5612 evergreen_agp_enable(rdev); 5613 } else { 5614 r = evergreen_pcie_gart_enable(rdev); 5615 if (r) 5616 return r; 5617 } 5618 evergreen_gpu_init(rdev); 5619 5620 /* allocate rlc buffers */ 5621 if (rdev->flags & RADEON_IS_IGP) { 5622 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list; 5623 rdev->rlc.reg_list_size = 5624 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list); 5625 rdev->rlc.cs_data = evergreen_cs_data; 5626 r = sumo_rlc_init(rdev); 5627 if (r) { 5628 DRM_ERROR("Failed to init rlc BOs!\n"); 5629 return r; 5630 } 5631 } 5632 5633 /* allocate wb buffer */ 5634 r = radeon_wb_init(rdev); 5635 if (r) 5636 return r; 5637 5638 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 5639 if (r) { 5640 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 5641 return r; 5642 } 5643 5644 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 5645 if (r) { 5646 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 5647 return r; 5648 } 5649 5650 evergreen_uvd_start(rdev); 5651 5652 /* Enable IRQ */ 5653 if (!rdev->irq.installed) { 5654 r = radeon_irq_kms_init(rdev); 5655 if (r) 5656 return r; 5657 } 5658 5659 r = r600_irq_init(rdev); 5660 if (r) { 5661 DRM_ERROR("radeon: IH init failed (%d).\n", r); 5662 radeon_irq_kms_fini(rdev); 5663 return r; 5664 } 5665 evergreen_irq_set(rdev); 5666 5667 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 5668 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 5669 RADEON_CP_PACKET2); 5670 if (r) 5671 return r; 5672 5673 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 5674 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 5675 DMA_PACKET(DMA_PACKET_NOP, 0, 0)); 5676 if (r) 5677 return r; 5678 5679 r = evergreen_cp_load_microcode(rdev); 5680 if (r) 5681 return r; 5682 r = evergreen_cp_resume(rdev); 5683 if (r) 5684 return r; 5685 r = r600_dma_resume(rdev); 5686 if (r) 5687 return r; 5688 5689 evergreen_uvd_resume(rdev); 5690 5691 r = radeon_ib_pool_init(rdev); 5692 if (r) { 5693 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 5694 return r; 5695 } 5696 5697 r = radeon_audio_init(rdev); 5698 if (r) { 5699 DRM_ERROR("radeon: audio init failed\n"); 5700 return r; 5701 } 5702 5703 return 0; 5704 } 5705 5706 int evergreen_resume(struct radeon_device *rdev) 5707 { 5708 int r; 5709 5710 /* reset the asic, the gfx blocks are often in a bad state 5711 * after the driver is unloaded or after a resume 5712 */ 5713 if (radeon_asic_reset(rdev)) 5714 dev_warn(rdev->dev, "GPU reset failed !\n"); 5715 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw, 5716 * posting will perform necessary task to bring back GPU into good 5717 * shape. 5718 */ 5719 /* post card */ 5720 atom_asic_init(rdev->mode_info.atom_context); 5721 5722 /* init golden registers */ 5723 evergreen_init_golden_registers(rdev); 5724 5725 if (rdev->pm.pm_method == PM_METHOD_DPM) 5726 radeon_pm_resume(rdev); 5727 5728 rdev->accel_working = true; 5729 r = evergreen_startup(rdev); 5730 if (r) { 5731 DRM_ERROR("evergreen startup failed on resume\n"); 5732 rdev->accel_working = false; 5733 return r; 5734 } 5735 5736 return r; 5737 5738 } 5739 5740 int evergreen_suspend(struct radeon_device *rdev) 5741 { 5742 radeon_pm_suspend(rdev); 5743 radeon_audio_fini(rdev); 5744 if (rdev->has_uvd) { 5745 uvd_v1_0_fini(rdev); 5746 radeon_uvd_suspend(rdev); 5747 } 5748 r700_cp_stop(rdev); 5749 r600_dma_stop(rdev); 5750 evergreen_irq_suspend(rdev); 5751 radeon_wb_disable(rdev); 5752 evergreen_pcie_gart_disable(rdev); 5753 5754 return 0; 5755 } 5756 5757 /* Plan is to move initialization in that function and use 5758 * helper function so that radeon_device_init pretty much 5759 * do nothing more than calling asic specific function. This 5760 * should also allow to remove a bunch of callback function 5761 * like vram_info. 5762 */ 5763 int evergreen_init(struct radeon_device *rdev) 5764 { 5765 int r; 5766 5767 /* Read BIOS */ 5768 if (!radeon_get_bios(rdev)) { 5769 if (ASIC_IS_AVIVO(rdev)) 5770 return -EINVAL; 5771 } 5772 /* Must be an ATOMBIOS */ 5773 if (!rdev->is_atom_bios) { 5774 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n"); 5775 return -EINVAL; 5776 } 5777 r = radeon_atombios_init(rdev); 5778 if (r) 5779 return r; 5780 /* reset the asic, the gfx blocks are often in a bad state 5781 * after the driver is unloaded or after a resume 5782 */ 5783 if (radeon_asic_reset(rdev)) 5784 dev_warn(rdev->dev, "GPU reset failed !\n"); 5785 /* Post card if necessary */ 5786 if (!radeon_card_posted(rdev)) { 5787 if (!rdev->bios) { 5788 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 5789 return -EINVAL; 5790 } 5791 DRM_INFO("GPU not posted. posting now...\n"); 5792 atom_asic_init(rdev->mode_info.atom_context); 5793 } 5794 /* init golden registers */ 5795 evergreen_init_golden_registers(rdev); 5796 /* Initialize scratch registers */ 5797 r600_scratch_init(rdev); 5798 /* Initialize surface registers */ 5799 radeon_surface_init(rdev); 5800 /* Initialize clocks */ 5801 radeon_get_clock_info(rdev->ddev); 5802 /* Fence driver */ 5803 r = radeon_fence_driver_init(rdev); 5804 if (r) 5805 return r; 5806 /* initialize AGP */ 5807 if (rdev->flags & RADEON_IS_AGP) { 5808 r = radeon_agp_init(rdev); 5809 if (r) 5810 radeon_agp_disable(rdev); 5811 } 5812 /* initialize memory controller */ 5813 r = evergreen_mc_init(rdev); 5814 if (r) 5815 return r; 5816 /* Memory manager */ 5817 r = radeon_bo_init(rdev); 5818 if (r) 5819 return r; 5820 5821 if (ASIC_IS_DCE5(rdev)) { 5822 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { 5823 r = ni_init_microcode(rdev); 5824 if (r) { 5825 DRM_ERROR("Failed to load firmware!\n"); 5826 return r; 5827 } 5828 } 5829 } else { 5830 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { 5831 r = r600_init_microcode(rdev); 5832 if (r) { 5833 DRM_ERROR("Failed to load firmware!\n"); 5834 return r; 5835 } 5836 } 5837 } 5838 5839 /* Initialize power management */ 5840 radeon_pm_init(rdev); 5841 5842 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL; 5843 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024); 5844 5845 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL; 5846 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024); 5847 5848 evergreen_uvd_init(rdev); 5849 5850 rdev->ih.ring_obj = NULL; 5851 r600_ih_ring_init(rdev, 64 * 1024); 5852 5853 r = r600_pcie_gart_init(rdev); 5854 if (r) 5855 return r; 5856 5857 rdev->accel_working = true; 5858 r = evergreen_startup(rdev); 5859 if (r) { 5860 dev_err(rdev->dev, "disabling GPU acceleration\n"); 5861 r700_cp_fini(rdev); 5862 r600_dma_fini(rdev); 5863 r600_irq_fini(rdev); 5864 if (rdev->flags & RADEON_IS_IGP) 5865 sumo_rlc_fini(rdev); 5866 radeon_wb_fini(rdev); 5867 radeon_ib_pool_fini(rdev); 5868 radeon_irq_kms_fini(rdev); 5869 evergreen_pcie_gart_fini(rdev); 5870 rdev->accel_working = false; 5871 } 5872 5873 /* Don't start up if the MC ucode is missing on BTC parts. 5874 * The default clocks and voltages before the MC ucode 5875 * is loaded are not suffient for advanced operations. 5876 */ 5877 if (ASIC_IS_DCE5(rdev)) { 5878 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 5879 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 5880 return -EINVAL; 5881 } 5882 } 5883 5884 return 0; 5885 } 5886 5887 void evergreen_fini(struct radeon_device *rdev) 5888 { 5889 radeon_pm_fini(rdev); 5890 radeon_audio_fini(rdev); 5891 r700_cp_fini(rdev); 5892 r600_dma_fini(rdev); 5893 r600_irq_fini(rdev); 5894 if (rdev->flags & RADEON_IS_IGP) 5895 sumo_rlc_fini(rdev); 5896 radeon_wb_fini(rdev); 5897 radeon_ib_pool_fini(rdev); 5898 radeon_irq_kms_fini(rdev); 5899 uvd_v1_0_fini(rdev); 5900 radeon_uvd_fini(rdev); 5901 evergreen_pcie_gart_fini(rdev); 5902 r600_vram_scratch_fini(rdev); 5903 radeon_gem_fini(rdev); 5904 radeon_fence_driver_fini(rdev); 5905 radeon_agp_fini(rdev); 5906 radeon_bo_fini(rdev); 5907 radeon_atombios_fini(rdev); 5908 if (ASIC_IS_DCE5(rdev)) 5909 ni_fini_microcode(rdev); 5910 else 5911 r600_fini_microcode(rdev); 5912 kfree(rdev->bios); 5913 rdev->bios = NULL; 5914 } 5915 5916 void evergreen_pcie_gen2_enable(struct radeon_device *rdev) 5917 { 5918 u32 link_width_cntl, speed_cntl, mask; 5919 int ret; 5920 5921 if (radeon_pcie_gen2 == 0) 5922 return; 5923 5924 if (rdev->flags & RADEON_IS_IGP) 5925 return; 5926 5927 if (!(rdev->flags & RADEON_IS_PCIE)) 5928 return; 5929 5930 /* x2 cards have a special sequence */ 5931 if (ASIC_IS_X2(rdev)) 5932 return; 5933 5934 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); 5935 if (ret != 0) 5936 return; 5937 5938 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80))) 5939 return; 5940 5941 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5942 if (speed_cntl & LC_CURRENT_DATA_RATE) { 5943 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 5944 return; 5945 } 5946 5947 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 5948 5949 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) || 5950 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) { 5951 5952 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5953 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 5954 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 5955 5956 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5957 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN; 5958 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5959 5960 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5961 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT; 5962 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5963 5964 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5965 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT; 5966 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5967 5968 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5969 speed_cntl |= LC_GEN2_EN_STRAP; 5970 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5971 5972 } else { 5973 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5974 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */ 5975 if (1) 5976 link_width_cntl |= LC_UPCONFIGURE_DIS; 5977 else 5978 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 5979 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 5980 } 5981 } 5982 5983 void evergreen_program_aspm(struct radeon_device *rdev) 5984 { 5985 u32 data, orig; 5986 u32 pcie_lc_cntl, pcie_lc_cntl_old; 5987 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false; 5988 /* fusion_platform = true 5989 * if the system is a fusion system 5990 * (APU or DGPU in a fusion system). 5991 * todo: check if the system is a fusion platform. 5992 */ 5993 bool fusion_platform = false; 5994 5995 if (radeon_aspm == 0) 5996 return; 5997 5998 if (!(rdev->flags & RADEON_IS_PCIE)) 5999 return; 6000 6001 switch (rdev->family) { 6002 case CHIP_CYPRESS: 6003 case CHIP_HEMLOCK: 6004 case CHIP_JUNIPER: 6005 case CHIP_REDWOOD: 6006 case CHIP_CEDAR: 6007 case CHIP_SUMO: 6008 case CHIP_SUMO2: 6009 case CHIP_PALM: 6010 case CHIP_ARUBA: 6011 disable_l0s = true; 6012 break; 6013 default: 6014 disable_l0s = false; 6015 break; 6016 } 6017 6018 if (rdev->flags & RADEON_IS_IGP) 6019 fusion_platform = true; /* XXX also dGPUs in a fusion system */ 6020 6021 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING); 6022 if (fusion_platform) 6023 data &= ~MULTI_PIF; 6024 else 6025 data |= MULTI_PIF; 6026 if (data != orig) 6027 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data); 6028 6029 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING); 6030 if (fusion_platform) 6031 data &= ~MULTI_PIF; 6032 else 6033 data |= MULTI_PIF; 6034 if (data != orig) 6035 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data); 6036 6037 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL); 6038 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 6039 if (!disable_l0s) { 6040 if (rdev->family >= CHIP_BARTS) 6041 pcie_lc_cntl |= LC_L0S_INACTIVITY(7); 6042 else 6043 pcie_lc_cntl |= LC_L0S_INACTIVITY(3); 6044 } 6045 6046 if (!disable_l1) { 6047 if (rdev->family >= CHIP_BARTS) 6048 pcie_lc_cntl |= LC_L1_INACTIVITY(7); 6049 else 6050 pcie_lc_cntl |= LC_L1_INACTIVITY(8); 6051 6052 if (!disable_plloff_in_l1) { 6053 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 6054 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 6055 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 6056 if (data != orig) 6057 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 6058 6059 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 6060 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 6061 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 6062 if (data != orig) 6063 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 6064 6065 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 6066 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 6067 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 6068 if (data != orig) 6069 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 6070 6071 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 6072 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 6073 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 6074 if (data != orig) 6075 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 6076 6077 if (rdev->family >= CHIP_BARTS) { 6078 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 6079 data &= ~PLL_RAMP_UP_TIME_0_MASK; 6080 data |= PLL_RAMP_UP_TIME_0(4); 6081 if (data != orig) 6082 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 6083 6084 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 6085 data &= ~PLL_RAMP_UP_TIME_1_MASK; 6086 data |= PLL_RAMP_UP_TIME_1(4); 6087 if (data != orig) 6088 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 6089 6090 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 6091 data &= ~PLL_RAMP_UP_TIME_0_MASK; 6092 data |= PLL_RAMP_UP_TIME_0(4); 6093 if (data != orig) 6094 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 6095 6096 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 6097 data &= ~PLL_RAMP_UP_TIME_1_MASK; 6098 data |= PLL_RAMP_UP_TIME_1(4); 6099 if (data != orig) 6100 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 6101 } 6102 6103 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 6104 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 6105 data |= LC_DYN_LANES_PWR_STATE(3); 6106 if (data != orig) 6107 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 6108 6109 if (rdev->family >= CHIP_BARTS) { 6110 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL); 6111 data &= ~LS2_EXIT_TIME_MASK; 6112 data |= LS2_EXIT_TIME(1); 6113 if (data != orig) 6114 WREG32_PIF_PHY0(PB0_PIF_CNTL, data); 6115 6116 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL); 6117 data &= ~LS2_EXIT_TIME_MASK; 6118 data |= LS2_EXIT_TIME(1); 6119 if (data != orig) 6120 WREG32_PIF_PHY1(PB1_PIF_CNTL, data); 6121 } 6122 } 6123 } 6124 6125 /* evergreen parts only */ 6126 if (rdev->family < CHIP_BARTS) 6127 pcie_lc_cntl |= LC_PMI_TO_L1_DIS; 6128 6129 if (pcie_lc_cntl != pcie_lc_cntl_old) 6130 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl); 6131 } 6132