1 /* 2 * Copyright 2010 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <drm/drmP.h> 26 #include "radeon.h" 27 #include "radeon_asic.h" 28 #include <uapi_drm/radeon_drm.h> 29 #include "evergreend.h" 30 #include "atom.h" 31 #include "avivod.h" 32 #include "evergreen_reg.h" 33 #include "evergreen_blit_shaders.h" 34 #include "radeon_ucode.h" 35 36 static const u32 crtc_offsets[6] = 37 { 38 EVERGREEN_CRTC0_REGISTER_OFFSET, 39 EVERGREEN_CRTC1_REGISTER_OFFSET, 40 EVERGREEN_CRTC2_REGISTER_OFFSET, 41 EVERGREEN_CRTC3_REGISTER_OFFSET, 42 EVERGREEN_CRTC4_REGISTER_OFFSET, 43 EVERGREEN_CRTC5_REGISTER_OFFSET 44 }; 45 46 #include "clearstate_evergreen.h" 47 48 static const u32 sumo_rlc_save_restore_register_list[] = 49 { 50 0x98fc, 51 0x9830, 52 0x9834, 53 0x9838, 54 0x9870, 55 0x9874, 56 0x8a14, 57 0x8b24, 58 0x8bcc, 59 0x8b10, 60 0x8d00, 61 0x8d04, 62 0x8c00, 63 0x8c04, 64 0x8c08, 65 0x8c0c, 66 0x8d8c, 67 0x8c20, 68 0x8c24, 69 0x8c28, 70 0x8c18, 71 0x8c1c, 72 0x8cf0, 73 0x8e2c, 74 0x8e38, 75 0x8c30, 76 0x9508, 77 0x9688, 78 0x9608, 79 0x960c, 80 0x9610, 81 0x9614, 82 0x88c4, 83 0x88d4, 84 0xa008, 85 0x900c, 86 0x9100, 87 0x913c, 88 0x98f8, 89 0x98f4, 90 0x9b7c, 91 0x3f8c, 92 0x8950, 93 0x8954, 94 0x8a18, 95 0x8b28, 96 0x9144, 97 0x9148, 98 0x914c, 99 0x3f90, 100 0x3f94, 101 0x915c, 102 0x9160, 103 0x9178, 104 0x917c, 105 0x9180, 106 0x918c, 107 0x9190, 108 0x9194, 109 0x9198, 110 0x919c, 111 0x91a8, 112 0x91ac, 113 0x91b0, 114 0x91b4, 115 0x91b8, 116 0x91c4, 117 0x91c8, 118 0x91cc, 119 0x91d0, 120 0x91d4, 121 0x91e0, 122 0x91e4, 123 0x91ec, 124 0x91f0, 125 0x91f4, 126 0x9200, 127 0x9204, 128 0x929c, 129 0x9150, 130 0x802c, 131 }; 132 133 static void evergreen_gpu_init(struct radeon_device *rdev); 134 135 static const u32 evergreen_golden_registers[] = 136 { 137 0x3f90, 0xffff0000, 0xff000000, 138 0x9148, 0xffff0000, 0xff000000, 139 0x3f94, 0xffff0000, 0xff000000, 140 0x914c, 0xffff0000, 0xff000000, 141 0x9b7c, 0xffffffff, 0x00000000, 142 0x8a14, 0xffffffff, 0x00000007, 143 0x8b10, 0xffffffff, 0x00000000, 144 0x960c, 0xffffffff, 0x54763210, 145 0x88c4, 0xffffffff, 0x000000c2, 146 0x88d4, 0xffffffff, 0x00000010, 147 0x8974, 0xffffffff, 0x00000000, 148 0xc78, 0x00000080, 0x00000080, 149 0x5eb4, 0xffffffff, 0x00000002, 150 0x5e78, 0xffffffff, 0x001000f0, 151 0x6104, 0x01000300, 0x00000000, 152 0x5bc0, 0x00300000, 0x00000000, 153 0x7030, 0xffffffff, 0x00000011, 154 0x7c30, 0xffffffff, 0x00000011, 155 0x10830, 0xffffffff, 0x00000011, 156 0x11430, 0xffffffff, 0x00000011, 157 0x12030, 0xffffffff, 0x00000011, 158 0x12c30, 0xffffffff, 0x00000011, 159 0xd02c, 0xffffffff, 0x08421000, 160 0x240c, 0xffffffff, 0x00000380, 161 0x8b24, 0xffffffff, 0x00ff0fff, 162 0x28a4c, 0x06000000, 0x06000000, 163 0x10c, 0x00000001, 0x00000001, 164 0x8d00, 0xffffffff, 0x100e4848, 165 0x8d04, 0xffffffff, 0x00164745, 166 0x8c00, 0xffffffff, 0xe4000003, 167 0x8c04, 0xffffffff, 0x40600060, 168 0x8c08, 0xffffffff, 0x001c001c, 169 0x8cf0, 0xffffffff, 0x08e00620, 170 0x8c20, 0xffffffff, 0x00800080, 171 0x8c24, 0xffffffff, 0x00800080, 172 0x8c18, 0xffffffff, 0x20202078, 173 0x8c1c, 0xffffffff, 0x00001010, 174 0x28350, 0xffffffff, 0x00000000, 175 0xa008, 0xffffffff, 0x00010000, 176 0x5c4, 0xffffffff, 0x00000001, 177 0x9508, 0xffffffff, 0x00000002, 178 0x913c, 0x0000000f, 0x0000000a 179 }; 180 181 static const u32 evergreen_golden_registers2[] = 182 { 183 0x2f4c, 0xffffffff, 0x00000000, 184 0x54f4, 0xffffffff, 0x00000000, 185 0x54f0, 0xffffffff, 0x00000000, 186 0x5498, 0xffffffff, 0x00000000, 187 0x549c, 0xffffffff, 0x00000000, 188 0x5494, 0xffffffff, 0x00000000, 189 0x53cc, 0xffffffff, 0x00000000, 190 0x53c8, 0xffffffff, 0x00000000, 191 0x53c4, 0xffffffff, 0x00000000, 192 0x53c0, 0xffffffff, 0x00000000, 193 0x53bc, 0xffffffff, 0x00000000, 194 0x53b8, 0xffffffff, 0x00000000, 195 0x53b4, 0xffffffff, 0x00000000, 196 0x53b0, 0xffffffff, 0x00000000 197 }; 198 199 static const u32 cypress_mgcg_init[] = 200 { 201 0x802c, 0xffffffff, 0xc0000000, 202 0x5448, 0xffffffff, 0x00000100, 203 0x55e4, 0xffffffff, 0x00000100, 204 0x160c, 0xffffffff, 0x00000100, 205 0x5644, 0xffffffff, 0x00000100, 206 0xc164, 0xffffffff, 0x00000100, 207 0x8a18, 0xffffffff, 0x00000100, 208 0x897c, 0xffffffff, 0x06000100, 209 0x8b28, 0xffffffff, 0x00000100, 210 0x9144, 0xffffffff, 0x00000100, 211 0x9a60, 0xffffffff, 0x00000100, 212 0x9868, 0xffffffff, 0x00000100, 213 0x8d58, 0xffffffff, 0x00000100, 214 0x9510, 0xffffffff, 0x00000100, 215 0x949c, 0xffffffff, 0x00000100, 216 0x9654, 0xffffffff, 0x00000100, 217 0x9030, 0xffffffff, 0x00000100, 218 0x9034, 0xffffffff, 0x00000100, 219 0x9038, 0xffffffff, 0x00000100, 220 0x903c, 0xffffffff, 0x00000100, 221 0x9040, 0xffffffff, 0x00000100, 222 0xa200, 0xffffffff, 0x00000100, 223 0xa204, 0xffffffff, 0x00000100, 224 0xa208, 0xffffffff, 0x00000100, 225 0xa20c, 0xffffffff, 0x00000100, 226 0x971c, 0xffffffff, 0x00000100, 227 0x977c, 0xffffffff, 0x00000100, 228 0x3f80, 0xffffffff, 0x00000100, 229 0xa210, 0xffffffff, 0x00000100, 230 0xa214, 0xffffffff, 0x00000100, 231 0x4d8, 0xffffffff, 0x00000100, 232 0x9784, 0xffffffff, 0x00000100, 233 0x9698, 0xffffffff, 0x00000100, 234 0x4d4, 0xffffffff, 0x00000200, 235 0x30cc, 0xffffffff, 0x00000100, 236 0xd0c0, 0xffffffff, 0xff000100, 237 0x802c, 0xffffffff, 0x40000000, 238 0x915c, 0xffffffff, 0x00010000, 239 0x9160, 0xffffffff, 0x00030002, 240 0x9178, 0xffffffff, 0x00070000, 241 0x917c, 0xffffffff, 0x00030002, 242 0x9180, 0xffffffff, 0x00050004, 243 0x918c, 0xffffffff, 0x00010006, 244 0x9190, 0xffffffff, 0x00090008, 245 0x9194, 0xffffffff, 0x00070000, 246 0x9198, 0xffffffff, 0x00030002, 247 0x919c, 0xffffffff, 0x00050004, 248 0x91a8, 0xffffffff, 0x00010006, 249 0x91ac, 0xffffffff, 0x00090008, 250 0x91b0, 0xffffffff, 0x00070000, 251 0x91b4, 0xffffffff, 0x00030002, 252 0x91b8, 0xffffffff, 0x00050004, 253 0x91c4, 0xffffffff, 0x00010006, 254 0x91c8, 0xffffffff, 0x00090008, 255 0x91cc, 0xffffffff, 0x00070000, 256 0x91d0, 0xffffffff, 0x00030002, 257 0x91d4, 0xffffffff, 0x00050004, 258 0x91e0, 0xffffffff, 0x00010006, 259 0x91e4, 0xffffffff, 0x00090008, 260 0x91e8, 0xffffffff, 0x00000000, 261 0x91ec, 0xffffffff, 0x00070000, 262 0x91f0, 0xffffffff, 0x00030002, 263 0x91f4, 0xffffffff, 0x00050004, 264 0x9200, 0xffffffff, 0x00010006, 265 0x9204, 0xffffffff, 0x00090008, 266 0x9208, 0xffffffff, 0x00070000, 267 0x920c, 0xffffffff, 0x00030002, 268 0x9210, 0xffffffff, 0x00050004, 269 0x921c, 0xffffffff, 0x00010006, 270 0x9220, 0xffffffff, 0x00090008, 271 0x9224, 0xffffffff, 0x00070000, 272 0x9228, 0xffffffff, 0x00030002, 273 0x922c, 0xffffffff, 0x00050004, 274 0x9238, 0xffffffff, 0x00010006, 275 0x923c, 0xffffffff, 0x00090008, 276 0x9240, 0xffffffff, 0x00070000, 277 0x9244, 0xffffffff, 0x00030002, 278 0x9248, 0xffffffff, 0x00050004, 279 0x9254, 0xffffffff, 0x00010006, 280 0x9258, 0xffffffff, 0x00090008, 281 0x925c, 0xffffffff, 0x00070000, 282 0x9260, 0xffffffff, 0x00030002, 283 0x9264, 0xffffffff, 0x00050004, 284 0x9270, 0xffffffff, 0x00010006, 285 0x9274, 0xffffffff, 0x00090008, 286 0x9278, 0xffffffff, 0x00070000, 287 0x927c, 0xffffffff, 0x00030002, 288 0x9280, 0xffffffff, 0x00050004, 289 0x928c, 0xffffffff, 0x00010006, 290 0x9290, 0xffffffff, 0x00090008, 291 0x9294, 0xffffffff, 0x00000000, 292 0x929c, 0xffffffff, 0x00000001, 293 0x802c, 0xffffffff, 0x40010000, 294 0x915c, 0xffffffff, 0x00010000, 295 0x9160, 0xffffffff, 0x00030002, 296 0x9178, 0xffffffff, 0x00070000, 297 0x917c, 0xffffffff, 0x00030002, 298 0x9180, 0xffffffff, 0x00050004, 299 0x918c, 0xffffffff, 0x00010006, 300 0x9190, 0xffffffff, 0x00090008, 301 0x9194, 0xffffffff, 0x00070000, 302 0x9198, 0xffffffff, 0x00030002, 303 0x919c, 0xffffffff, 0x00050004, 304 0x91a8, 0xffffffff, 0x00010006, 305 0x91ac, 0xffffffff, 0x00090008, 306 0x91b0, 0xffffffff, 0x00070000, 307 0x91b4, 0xffffffff, 0x00030002, 308 0x91b8, 0xffffffff, 0x00050004, 309 0x91c4, 0xffffffff, 0x00010006, 310 0x91c8, 0xffffffff, 0x00090008, 311 0x91cc, 0xffffffff, 0x00070000, 312 0x91d0, 0xffffffff, 0x00030002, 313 0x91d4, 0xffffffff, 0x00050004, 314 0x91e0, 0xffffffff, 0x00010006, 315 0x91e4, 0xffffffff, 0x00090008, 316 0x91e8, 0xffffffff, 0x00000000, 317 0x91ec, 0xffffffff, 0x00070000, 318 0x91f0, 0xffffffff, 0x00030002, 319 0x91f4, 0xffffffff, 0x00050004, 320 0x9200, 0xffffffff, 0x00010006, 321 0x9204, 0xffffffff, 0x00090008, 322 0x9208, 0xffffffff, 0x00070000, 323 0x920c, 0xffffffff, 0x00030002, 324 0x9210, 0xffffffff, 0x00050004, 325 0x921c, 0xffffffff, 0x00010006, 326 0x9220, 0xffffffff, 0x00090008, 327 0x9224, 0xffffffff, 0x00070000, 328 0x9228, 0xffffffff, 0x00030002, 329 0x922c, 0xffffffff, 0x00050004, 330 0x9238, 0xffffffff, 0x00010006, 331 0x923c, 0xffffffff, 0x00090008, 332 0x9240, 0xffffffff, 0x00070000, 333 0x9244, 0xffffffff, 0x00030002, 334 0x9248, 0xffffffff, 0x00050004, 335 0x9254, 0xffffffff, 0x00010006, 336 0x9258, 0xffffffff, 0x00090008, 337 0x925c, 0xffffffff, 0x00070000, 338 0x9260, 0xffffffff, 0x00030002, 339 0x9264, 0xffffffff, 0x00050004, 340 0x9270, 0xffffffff, 0x00010006, 341 0x9274, 0xffffffff, 0x00090008, 342 0x9278, 0xffffffff, 0x00070000, 343 0x927c, 0xffffffff, 0x00030002, 344 0x9280, 0xffffffff, 0x00050004, 345 0x928c, 0xffffffff, 0x00010006, 346 0x9290, 0xffffffff, 0x00090008, 347 0x9294, 0xffffffff, 0x00000000, 348 0x929c, 0xffffffff, 0x00000001, 349 0x802c, 0xffffffff, 0xc0000000 350 }; 351 352 static const u32 redwood_mgcg_init[] = 353 { 354 0x802c, 0xffffffff, 0xc0000000, 355 0x5448, 0xffffffff, 0x00000100, 356 0x55e4, 0xffffffff, 0x00000100, 357 0x160c, 0xffffffff, 0x00000100, 358 0x5644, 0xffffffff, 0x00000100, 359 0xc164, 0xffffffff, 0x00000100, 360 0x8a18, 0xffffffff, 0x00000100, 361 0x897c, 0xffffffff, 0x06000100, 362 0x8b28, 0xffffffff, 0x00000100, 363 0x9144, 0xffffffff, 0x00000100, 364 0x9a60, 0xffffffff, 0x00000100, 365 0x9868, 0xffffffff, 0x00000100, 366 0x8d58, 0xffffffff, 0x00000100, 367 0x9510, 0xffffffff, 0x00000100, 368 0x949c, 0xffffffff, 0x00000100, 369 0x9654, 0xffffffff, 0x00000100, 370 0x9030, 0xffffffff, 0x00000100, 371 0x9034, 0xffffffff, 0x00000100, 372 0x9038, 0xffffffff, 0x00000100, 373 0x903c, 0xffffffff, 0x00000100, 374 0x9040, 0xffffffff, 0x00000100, 375 0xa200, 0xffffffff, 0x00000100, 376 0xa204, 0xffffffff, 0x00000100, 377 0xa208, 0xffffffff, 0x00000100, 378 0xa20c, 0xffffffff, 0x00000100, 379 0x971c, 0xffffffff, 0x00000100, 380 0x977c, 0xffffffff, 0x00000100, 381 0x3f80, 0xffffffff, 0x00000100, 382 0xa210, 0xffffffff, 0x00000100, 383 0xa214, 0xffffffff, 0x00000100, 384 0x4d8, 0xffffffff, 0x00000100, 385 0x9784, 0xffffffff, 0x00000100, 386 0x9698, 0xffffffff, 0x00000100, 387 0x4d4, 0xffffffff, 0x00000200, 388 0x30cc, 0xffffffff, 0x00000100, 389 0xd0c0, 0xffffffff, 0xff000100, 390 0x802c, 0xffffffff, 0x40000000, 391 0x915c, 0xffffffff, 0x00010000, 392 0x9160, 0xffffffff, 0x00030002, 393 0x9178, 0xffffffff, 0x00070000, 394 0x917c, 0xffffffff, 0x00030002, 395 0x9180, 0xffffffff, 0x00050004, 396 0x918c, 0xffffffff, 0x00010006, 397 0x9190, 0xffffffff, 0x00090008, 398 0x9194, 0xffffffff, 0x00070000, 399 0x9198, 0xffffffff, 0x00030002, 400 0x919c, 0xffffffff, 0x00050004, 401 0x91a8, 0xffffffff, 0x00010006, 402 0x91ac, 0xffffffff, 0x00090008, 403 0x91b0, 0xffffffff, 0x00070000, 404 0x91b4, 0xffffffff, 0x00030002, 405 0x91b8, 0xffffffff, 0x00050004, 406 0x91c4, 0xffffffff, 0x00010006, 407 0x91c8, 0xffffffff, 0x00090008, 408 0x91cc, 0xffffffff, 0x00070000, 409 0x91d0, 0xffffffff, 0x00030002, 410 0x91d4, 0xffffffff, 0x00050004, 411 0x91e0, 0xffffffff, 0x00010006, 412 0x91e4, 0xffffffff, 0x00090008, 413 0x91e8, 0xffffffff, 0x00000000, 414 0x91ec, 0xffffffff, 0x00070000, 415 0x91f0, 0xffffffff, 0x00030002, 416 0x91f4, 0xffffffff, 0x00050004, 417 0x9200, 0xffffffff, 0x00010006, 418 0x9204, 0xffffffff, 0x00090008, 419 0x9294, 0xffffffff, 0x00000000, 420 0x929c, 0xffffffff, 0x00000001, 421 0x802c, 0xffffffff, 0xc0000000 422 }; 423 424 static const u32 cedar_golden_registers[] = 425 { 426 0x3f90, 0xffff0000, 0xff000000, 427 0x9148, 0xffff0000, 0xff000000, 428 0x3f94, 0xffff0000, 0xff000000, 429 0x914c, 0xffff0000, 0xff000000, 430 0x9b7c, 0xffffffff, 0x00000000, 431 0x8a14, 0xffffffff, 0x00000007, 432 0x8b10, 0xffffffff, 0x00000000, 433 0x960c, 0xffffffff, 0x54763210, 434 0x88c4, 0xffffffff, 0x000000c2, 435 0x88d4, 0xffffffff, 0x00000000, 436 0x8974, 0xffffffff, 0x00000000, 437 0xc78, 0x00000080, 0x00000080, 438 0x5eb4, 0xffffffff, 0x00000002, 439 0x5e78, 0xffffffff, 0x001000f0, 440 0x6104, 0x01000300, 0x00000000, 441 0x5bc0, 0x00300000, 0x00000000, 442 0x7030, 0xffffffff, 0x00000011, 443 0x7c30, 0xffffffff, 0x00000011, 444 0x10830, 0xffffffff, 0x00000011, 445 0x11430, 0xffffffff, 0x00000011, 446 0xd02c, 0xffffffff, 0x08421000, 447 0x240c, 0xffffffff, 0x00000380, 448 0x8b24, 0xffffffff, 0x00ff0fff, 449 0x28a4c, 0x06000000, 0x06000000, 450 0x10c, 0x00000001, 0x00000001, 451 0x8d00, 0xffffffff, 0x100e4848, 452 0x8d04, 0xffffffff, 0x00164745, 453 0x8c00, 0xffffffff, 0xe4000003, 454 0x8c04, 0xffffffff, 0x40600060, 455 0x8c08, 0xffffffff, 0x001c001c, 456 0x8cf0, 0xffffffff, 0x08e00410, 457 0x8c20, 0xffffffff, 0x00800080, 458 0x8c24, 0xffffffff, 0x00800080, 459 0x8c18, 0xffffffff, 0x20202078, 460 0x8c1c, 0xffffffff, 0x00001010, 461 0x28350, 0xffffffff, 0x00000000, 462 0xa008, 0xffffffff, 0x00010000, 463 0x5c4, 0xffffffff, 0x00000001, 464 0x9508, 0xffffffff, 0x00000002 465 }; 466 467 static const u32 cedar_mgcg_init[] = 468 { 469 0x802c, 0xffffffff, 0xc0000000, 470 0x5448, 0xffffffff, 0x00000100, 471 0x55e4, 0xffffffff, 0x00000100, 472 0x160c, 0xffffffff, 0x00000100, 473 0x5644, 0xffffffff, 0x00000100, 474 0xc164, 0xffffffff, 0x00000100, 475 0x8a18, 0xffffffff, 0x00000100, 476 0x897c, 0xffffffff, 0x06000100, 477 0x8b28, 0xffffffff, 0x00000100, 478 0x9144, 0xffffffff, 0x00000100, 479 0x9a60, 0xffffffff, 0x00000100, 480 0x9868, 0xffffffff, 0x00000100, 481 0x8d58, 0xffffffff, 0x00000100, 482 0x9510, 0xffffffff, 0x00000100, 483 0x949c, 0xffffffff, 0x00000100, 484 0x9654, 0xffffffff, 0x00000100, 485 0x9030, 0xffffffff, 0x00000100, 486 0x9034, 0xffffffff, 0x00000100, 487 0x9038, 0xffffffff, 0x00000100, 488 0x903c, 0xffffffff, 0x00000100, 489 0x9040, 0xffffffff, 0x00000100, 490 0xa200, 0xffffffff, 0x00000100, 491 0xa204, 0xffffffff, 0x00000100, 492 0xa208, 0xffffffff, 0x00000100, 493 0xa20c, 0xffffffff, 0x00000100, 494 0x971c, 0xffffffff, 0x00000100, 495 0x977c, 0xffffffff, 0x00000100, 496 0x3f80, 0xffffffff, 0x00000100, 497 0xa210, 0xffffffff, 0x00000100, 498 0xa214, 0xffffffff, 0x00000100, 499 0x4d8, 0xffffffff, 0x00000100, 500 0x9784, 0xffffffff, 0x00000100, 501 0x9698, 0xffffffff, 0x00000100, 502 0x4d4, 0xffffffff, 0x00000200, 503 0x30cc, 0xffffffff, 0x00000100, 504 0xd0c0, 0xffffffff, 0xff000100, 505 0x802c, 0xffffffff, 0x40000000, 506 0x915c, 0xffffffff, 0x00010000, 507 0x9178, 0xffffffff, 0x00050000, 508 0x917c, 0xffffffff, 0x00030002, 509 0x918c, 0xffffffff, 0x00010004, 510 0x9190, 0xffffffff, 0x00070006, 511 0x9194, 0xffffffff, 0x00050000, 512 0x9198, 0xffffffff, 0x00030002, 513 0x91a8, 0xffffffff, 0x00010004, 514 0x91ac, 0xffffffff, 0x00070006, 515 0x91e8, 0xffffffff, 0x00000000, 516 0x9294, 0xffffffff, 0x00000000, 517 0x929c, 0xffffffff, 0x00000001, 518 0x802c, 0xffffffff, 0xc0000000 519 }; 520 521 static const u32 juniper_mgcg_init[] = 522 { 523 0x802c, 0xffffffff, 0xc0000000, 524 0x5448, 0xffffffff, 0x00000100, 525 0x55e4, 0xffffffff, 0x00000100, 526 0x160c, 0xffffffff, 0x00000100, 527 0x5644, 0xffffffff, 0x00000100, 528 0xc164, 0xffffffff, 0x00000100, 529 0x8a18, 0xffffffff, 0x00000100, 530 0x897c, 0xffffffff, 0x06000100, 531 0x8b28, 0xffffffff, 0x00000100, 532 0x9144, 0xffffffff, 0x00000100, 533 0x9a60, 0xffffffff, 0x00000100, 534 0x9868, 0xffffffff, 0x00000100, 535 0x8d58, 0xffffffff, 0x00000100, 536 0x9510, 0xffffffff, 0x00000100, 537 0x949c, 0xffffffff, 0x00000100, 538 0x9654, 0xffffffff, 0x00000100, 539 0x9030, 0xffffffff, 0x00000100, 540 0x9034, 0xffffffff, 0x00000100, 541 0x9038, 0xffffffff, 0x00000100, 542 0x903c, 0xffffffff, 0x00000100, 543 0x9040, 0xffffffff, 0x00000100, 544 0xa200, 0xffffffff, 0x00000100, 545 0xa204, 0xffffffff, 0x00000100, 546 0xa208, 0xffffffff, 0x00000100, 547 0xa20c, 0xffffffff, 0x00000100, 548 0x971c, 0xffffffff, 0x00000100, 549 0xd0c0, 0xffffffff, 0xff000100, 550 0x802c, 0xffffffff, 0x40000000, 551 0x915c, 0xffffffff, 0x00010000, 552 0x9160, 0xffffffff, 0x00030002, 553 0x9178, 0xffffffff, 0x00070000, 554 0x917c, 0xffffffff, 0x00030002, 555 0x9180, 0xffffffff, 0x00050004, 556 0x918c, 0xffffffff, 0x00010006, 557 0x9190, 0xffffffff, 0x00090008, 558 0x9194, 0xffffffff, 0x00070000, 559 0x9198, 0xffffffff, 0x00030002, 560 0x919c, 0xffffffff, 0x00050004, 561 0x91a8, 0xffffffff, 0x00010006, 562 0x91ac, 0xffffffff, 0x00090008, 563 0x91b0, 0xffffffff, 0x00070000, 564 0x91b4, 0xffffffff, 0x00030002, 565 0x91b8, 0xffffffff, 0x00050004, 566 0x91c4, 0xffffffff, 0x00010006, 567 0x91c8, 0xffffffff, 0x00090008, 568 0x91cc, 0xffffffff, 0x00070000, 569 0x91d0, 0xffffffff, 0x00030002, 570 0x91d4, 0xffffffff, 0x00050004, 571 0x91e0, 0xffffffff, 0x00010006, 572 0x91e4, 0xffffffff, 0x00090008, 573 0x91e8, 0xffffffff, 0x00000000, 574 0x91ec, 0xffffffff, 0x00070000, 575 0x91f0, 0xffffffff, 0x00030002, 576 0x91f4, 0xffffffff, 0x00050004, 577 0x9200, 0xffffffff, 0x00010006, 578 0x9204, 0xffffffff, 0x00090008, 579 0x9208, 0xffffffff, 0x00070000, 580 0x920c, 0xffffffff, 0x00030002, 581 0x9210, 0xffffffff, 0x00050004, 582 0x921c, 0xffffffff, 0x00010006, 583 0x9220, 0xffffffff, 0x00090008, 584 0x9224, 0xffffffff, 0x00070000, 585 0x9228, 0xffffffff, 0x00030002, 586 0x922c, 0xffffffff, 0x00050004, 587 0x9238, 0xffffffff, 0x00010006, 588 0x923c, 0xffffffff, 0x00090008, 589 0x9240, 0xffffffff, 0x00070000, 590 0x9244, 0xffffffff, 0x00030002, 591 0x9248, 0xffffffff, 0x00050004, 592 0x9254, 0xffffffff, 0x00010006, 593 0x9258, 0xffffffff, 0x00090008, 594 0x925c, 0xffffffff, 0x00070000, 595 0x9260, 0xffffffff, 0x00030002, 596 0x9264, 0xffffffff, 0x00050004, 597 0x9270, 0xffffffff, 0x00010006, 598 0x9274, 0xffffffff, 0x00090008, 599 0x9278, 0xffffffff, 0x00070000, 600 0x927c, 0xffffffff, 0x00030002, 601 0x9280, 0xffffffff, 0x00050004, 602 0x928c, 0xffffffff, 0x00010006, 603 0x9290, 0xffffffff, 0x00090008, 604 0x9294, 0xffffffff, 0x00000000, 605 0x929c, 0xffffffff, 0x00000001, 606 0x802c, 0xffffffff, 0xc0000000, 607 0x977c, 0xffffffff, 0x00000100, 608 0x3f80, 0xffffffff, 0x00000100, 609 0xa210, 0xffffffff, 0x00000100, 610 0xa214, 0xffffffff, 0x00000100, 611 0x4d8, 0xffffffff, 0x00000100, 612 0x9784, 0xffffffff, 0x00000100, 613 0x9698, 0xffffffff, 0x00000100, 614 0x4d4, 0xffffffff, 0x00000200, 615 0x30cc, 0xffffffff, 0x00000100, 616 0x802c, 0xffffffff, 0xc0000000 617 }; 618 619 static const u32 supersumo_golden_registers[] = 620 { 621 0x5eb4, 0xffffffff, 0x00000002, 622 0x5c4, 0xffffffff, 0x00000001, 623 0x7030, 0xffffffff, 0x00000011, 624 0x7c30, 0xffffffff, 0x00000011, 625 0x6104, 0x01000300, 0x00000000, 626 0x5bc0, 0x00300000, 0x00000000, 627 0x8c04, 0xffffffff, 0x40600060, 628 0x8c08, 0xffffffff, 0x001c001c, 629 0x8c20, 0xffffffff, 0x00800080, 630 0x8c24, 0xffffffff, 0x00800080, 631 0x8c18, 0xffffffff, 0x20202078, 632 0x8c1c, 0xffffffff, 0x00001010, 633 0x918c, 0xffffffff, 0x00010006, 634 0x91a8, 0xffffffff, 0x00010006, 635 0x91c4, 0xffffffff, 0x00010006, 636 0x91e0, 0xffffffff, 0x00010006, 637 0x9200, 0xffffffff, 0x00010006, 638 0x9150, 0xffffffff, 0x6e944040, 639 0x917c, 0xffffffff, 0x00030002, 640 0x9180, 0xffffffff, 0x00050004, 641 0x9198, 0xffffffff, 0x00030002, 642 0x919c, 0xffffffff, 0x00050004, 643 0x91b4, 0xffffffff, 0x00030002, 644 0x91b8, 0xffffffff, 0x00050004, 645 0x91d0, 0xffffffff, 0x00030002, 646 0x91d4, 0xffffffff, 0x00050004, 647 0x91f0, 0xffffffff, 0x00030002, 648 0x91f4, 0xffffffff, 0x00050004, 649 0x915c, 0xffffffff, 0x00010000, 650 0x9160, 0xffffffff, 0x00030002, 651 0x3f90, 0xffff0000, 0xff000000, 652 0x9178, 0xffffffff, 0x00070000, 653 0x9194, 0xffffffff, 0x00070000, 654 0x91b0, 0xffffffff, 0x00070000, 655 0x91cc, 0xffffffff, 0x00070000, 656 0x91ec, 0xffffffff, 0x00070000, 657 0x9148, 0xffff0000, 0xff000000, 658 0x9190, 0xffffffff, 0x00090008, 659 0x91ac, 0xffffffff, 0x00090008, 660 0x91c8, 0xffffffff, 0x00090008, 661 0x91e4, 0xffffffff, 0x00090008, 662 0x9204, 0xffffffff, 0x00090008, 663 0x3f94, 0xffff0000, 0xff000000, 664 0x914c, 0xffff0000, 0xff000000, 665 0x929c, 0xffffffff, 0x00000001, 666 0x8a18, 0xffffffff, 0x00000100, 667 0x8b28, 0xffffffff, 0x00000100, 668 0x9144, 0xffffffff, 0x00000100, 669 0x5644, 0xffffffff, 0x00000100, 670 0x9b7c, 0xffffffff, 0x00000000, 671 0x8030, 0xffffffff, 0x0000100a, 672 0x8a14, 0xffffffff, 0x00000007, 673 0x8b24, 0xffffffff, 0x00ff0fff, 674 0x8b10, 0xffffffff, 0x00000000, 675 0x28a4c, 0x06000000, 0x06000000, 676 0x4d8, 0xffffffff, 0x00000100, 677 0x913c, 0xffff000f, 0x0100000a, 678 0x960c, 0xffffffff, 0x54763210, 679 0x88c4, 0xffffffff, 0x000000c2, 680 0x88d4, 0xffffffff, 0x00000010, 681 0x8974, 0xffffffff, 0x00000000, 682 0xc78, 0x00000080, 0x00000080, 683 0x5e78, 0xffffffff, 0x001000f0, 684 0xd02c, 0xffffffff, 0x08421000, 685 0xa008, 0xffffffff, 0x00010000, 686 0x8d00, 0xffffffff, 0x100e4848, 687 0x8d04, 0xffffffff, 0x00164745, 688 0x8c00, 0xffffffff, 0xe4000003, 689 0x8cf0, 0x1fffffff, 0x08e00620, 690 0x28350, 0xffffffff, 0x00000000, 691 0x9508, 0xffffffff, 0x00000002 692 }; 693 694 static const u32 sumo_golden_registers[] = 695 { 696 0x900c, 0x00ffffff, 0x0017071f, 697 0x8c18, 0xffffffff, 0x10101060, 698 0x8c1c, 0xffffffff, 0x00001010, 699 0x8c30, 0x0000000f, 0x00000005, 700 0x9688, 0x0000000f, 0x00000007 701 }; 702 703 static const u32 wrestler_golden_registers[] = 704 { 705 0x5eb4, 0xffffffff, 0x00000002, 706 0x5c4, 0xffffffff, 0x00000001, 707 0x7030, 0xffffffff, 0x00000011, 708 0x7c30, 0xffffffff, 0x00000011, 709 0x6104, 0x01000300, 0x00000000, 710 0x5bc0, 0x00300000, 0x00000000, 711 0x918c, 0xffffffff, 0x00010006, 712 0x91a8, 0xffffffff, 0x00010006, 713 0x9150, 0xffffffff, 0x6e944040, 714 0x917c, 0xffffffff, 0x00030002, 715 0x9198, 0xffffffff, 0x00030002, 716 0x915c, 0xffffffff, 0x00010000, 717 0x3f90, 0xffff0000, 0xff000000, 718 0x9178, 0xffffffff, 0x00070000, 719 0x9194, 0xffffffff, 0x00070000, 720 0x9148, 0xffff0000, 0xff000000, 721 0x9190, 0xffffffff, 0x00090008, 722 0x91ac, 0xffffffff, 0x00090008, 723 0x3f94, 0xffff0000, 0xff000000, 724 0x914c, 0xffff0000, 0xff000000, 725 0x929c, 0xffffffff, 0x00000001, 726 0x8a18, 0xffffffff, 0x00000100, 727 0x8b28, 0xffffffff, 0x00000100, 728 0x9144, 0xffffffff, 0x00000100, 729 0x9b7c, 0xffffffff, 0x00000000, 730 0x8030, 0xffffffff, 0x0000100a, 731 0x8a14, 0xffffffff, 0x00000001, 732 0x8b24, 0xffffffff, 0x00ff0fff, 733 0x8b10, 0xffffffff, 0x00000000, 734 0x28a4c, 0x06000000, 0x06000000, 735 0x4d8, 0xffffffff, 0x00000100, 736 0x913c, 0xffff000f, 0x0100000a, 737 0x960c, 0xffffffff, 0x54763210, 738 0x88c4, 0xffffffff, 0x000000c2, 739 0x88d4, 0xffffffff, 0x00000010, 740 0x8974, 0xffffffff, 0x00000000, 741 0xc78, 0x00000080, 0x00000080, 742 0x5e78, 0xffffffff, 0x001000f0, 743 0xd02c, 0xffffffff, 0x08421000, 744 0xa008, 0xffffffff, 0x00010000, 745 0x8d00, 0xffffffff, 0x100e4848, 746 0x8d04, 0xffffffff, 0x00164745, 747 0x8c00, 0xffffffff, 0xe4000003, 748 0x8cf0, 0x1fffffff, 0x08e00410, 749 0x28350, 0xffffffff, 0x00000000, 750 0x9508, 0xffffffff, 0x00000002, 751 0x900c, 0xffffffff, 0x0017071f, 752 0x8c18, 0xffffffff, 0x10101060, 753 0x8c1c, 0xffffffff, 0x00001010 754 }; 755 756 static const u32 barts_golden_registers[] = 757 { 758 0x5eb4, 0xffffffff, 0x00000002, 759 0x5e78, 0x8f311ff1, 0x001000f0, 760 0x3f90, 0xffff0000, 0xff000000, 761 0x9148, 0xffff0000, 0xff000000, 762 0x3f94, 0xffff0000, 0xff000000, 763 0x914c, 0xffff0000, 0xff000000, 764 0xc78, 0x00000080, 0x00000080, 765 0xbd4, 0x70073777, 0x00010001, 766 0xd02c, 0xbfffff1f, 0x08421000, 767 0xd0b8, 0x03773777, 0x02011003, 768 0x5bc0, 0x00200000, 0x50100000, 769 0x98f8, 0x33773777, 0x02011003, 770 0x98fc, 0xffffffff, 0x76543210, 771 0x7030, 0x31000311, 0x00000011, 772 0x2f48, 0x00000007, 0x02011003, 773 0x6b28, 0x00000010, 0x00000012, 774 0x7728, 0x00000010, 0x00000012, 775 0x10328, 0x00000010, 0x00000012, 776 0x10f28, 0x00000010, 0x00000012, 777 0x11b28, 0x00000010, 0x00000012, 778 0x12728, 0x00000010, 0x00000012, 779 0x240c, 0x000007ff, 0x00000380, 780 0x8a14, 0xf000001f, 0x00000007, 781 0x8b24, 0x3fff3fff, 0x00ff0fff, 782 0x8b10, 0x0000ff0f, 0x00000000, 783 0x28a4c, 0x07ffffff, 0x06000000, 784 0x10c, 0x00000001, 0x00010003, 785 0xa02c, 0xffffffff, 0x0000009b, 786 0x913c, 0x0000000f, 0x0100000a, 787 0x8d00, 0xffff7f7f, 0x100e4848, 788 0x8d04, 0x00ffffff, 0x00164745, 789 0x8c00, 0xfffc0003, 0xe4000003, 790 0x8c04, 0xf8ff00ff, 0x40600060, 791 0x8c08, 0x00ff00ff, 0x001c001c, 792 0x8cf0, 0x1fff1fff, 0x08e00620, 793 0x8c20, 0x0fff0fff, 0x00800080, 794 0x8c24, 0x0fff0fff, 0x00800080, 795 0x8c18, 0xffffffff, 0x20202078, 796 0x8c1c, 0x0000ffff, 0x00001010, 797 0x28350, 0x00000f01, 0x00000000, 798 0x9508, 0x3700001f, 0x00000002, 799 0x960c, 0xffffffff, 0x54763210, 800 0x88c4, 0x001f3ae3, 0x000000c2, 801 0x88d4, 0x0000001f, 0x00000010, 802 0x8974, 0xffffffff, 0x00000000 803 }; 804 805 static const u32 turks_golden_registers[] = 806 { 807 0x5eb4, 0xffffffff, 0x00000002, 808 0x5e78, 0x8f311ff1, 0x001000f0, 809 0x8c8, 0x00003000, 0x00001070, 810 0x8cc, 0x000fffff, 0x00040035, 811 0x3f90, 0xffff0000, 0xfff00000, 812 0x9148, 0xffff0000, 0xfff00000, 813 0x3f94, 0xffff0000, 0xfff00000, 814 0x914c, 0xffff0000, 0xfff00000, 815 0xc78, 0x00000080, 0x00000080, 816 0xbd4, 0x00073007, 0x00010002, 817 0xd02c, 0xbfffff1f, 0x08421000, 818 0xd0b8, 0x03773777, 0x02010002, 819 0x5bc0, 0x00200000, 0x50100000, 820 0x98f8, 0x33773777, 0x00010002, 821 0x98fc, 0xffffffff, 0x33221100, 822 0x7030, 0x31000311, 0x00000011, 823 0x2f48, 0x33773777, 0x00010002, 824 0x6b28, 0x00000010, 0x00000012, 825 0x7728, 0x00000010, 0x00000012, 826 0x10328, 0x00000010, 0x00000012, 827 0x10f28, 0x00000010, 0x00000012, 828 0x11b28, 0x00000010, 0x00000012, 829 0x12728, 0x00000010, 0x00000012, 830 0x240c, 0x000007ff, 0x00000380, 831 0x8a14, 0xf000001f, 0x00000007, 832 0x8b24, 0x3fff3fff, 0x00ff0fff, 833 0x8b10, 0x0000ff0f, 0x00000000, 834 0x28a4c, 0x07ffffff, 0x06000000, 835 0x10c, 0x00000001, 0x00010003, 836 0xa02c, 0xffffffff, 0x0000009b, 837 0x913c, 0x0000000f, 0x0100000a, 838 0x8d00, 0xffff7f7f, 0x100e4848, 839 0x8d04, 0x00ffffff, 0x00164745, 840 0x8c00, 0xfffc0003, 0xe4000003, 841 0x8c04, 0xf8ff00ff, 0x40600060, 842 0x8c08, 0x00ff00ff, 0x001c001c, 843 0x8cf0, 0x1fff1fff, 0x08e00410, 844 0x8c20, 0x0fff0fff, 0x00800080, 845 0x8c24, 0x0fff0fff, 0x00800080, 846 0x8c18, 0xffffffff, 0x20202078, 847 0x8c1c, 0x0000ffff, 0x00001010, 848 0x28350, 0x00000f01, 0x00000000, 849 0x9508, 0x3700001f, 0x00000002, 850 0x960c, 0xffffffff, 0x54763210, 851 0x88c4, 0x001f3ae3, 0x000000c2, 852 0x88d4, 0x0000001f, 0x00000010, 853 0x8974, 0xffffffff, 0x00000000 854 }; 855 856 static const u32 caicos_golden_registers[] = 857 { 858 0x5eb4, 0xffffffff, 0x00000002, 859 0x5e78, 0x8f311ff1, 0x001000f0, 860 0x8c8, 0x00003420, 0x00001450, 861 0x8cc, 0x000fffff, 0x00040035, 862 0x3f90, 0xffff0000, 0xfffc0000, 863 0x9148, 0xffff0000, 0xfffc0000, 864 0x3f94, 0xffff0000, 0xfffc0000, 865 0x914c, 0xffff0000, 0xfffc0000, 866 0xc78, 0x00000080, 0x00000080, 867 0xbd4, 0x00073007, 0x00010001, 868 0xd02c, 0xbfffff1f, 0x08421000, 869 0xd0b8, 0x03773777, 0x02010001, 870 0x5bc0, 0x00200000, 0x50100000, 871 0x98f8, 0x33773777, 0x02010001, 872 0x98fc, 0xffffffff, 0x33221100, 873 0x7030, 0x31000311, 0x00000011, 874 0x2f48, 0x33773777, 0x02010001, 875 0x6b28, 0x00000010, 0x00000012, 876 0x7728, 0x00000010, 0x00000012, 877 0x10328, 0x00000010, 0x00000012, 878 0x10f28, 0x00000010, 0x00000012, 879 0x11b28, 0x00000010, 0x00000012, 880 0x12728, 0x00000010, 0x00000012, 881 0x240c, 0x000007ff, 0x00000380, 882 0x8a14, 0xf000001f, 0x00000001, 883 0x8b24, 0x3fff3fff, 0x00ff0fff, 884 0x8b10, 0x0000ff0f, 0x00000000, 885 0x28a4c, 0x07ffffff, 0x06000000, 886 0x10c, 0x00000001, 0x00010003, 887 0xa02c, 0xffffffff, 0x0000009b, 888 0x913c, 0x0000000f, 0x0100000a, 889 0x8d00, 0xffff7f7f, 0x100e4848, 890 0x8d04, 0x00ffffff, 0x00164745, 891 0x8c00, 0xfffc0003, 0xe4000003, 892 0x8c04, 0xf8ff00ff, 0x40600060, 893 0x8c08, 0x00ff00ff, 0x001c001c, 894 0x8cf0, 0x1fff1fff, 0x08e00410, 895 0x8c20, 0x0fff0fff, 0x00800080, 896 0x8c24, 0x0fff0fff, 0x00800080, 897 0x8c18, 0xffffffff, 0x20202078, 898 0x8c1c, 0x0000ffff, 0x00001010, 899 0x28350, 0x00000f01, 0x00000000, 900 0x9508, 0x3700001f, 0x00000002, 901 0x960c, 0xffffffff, 0x54763210, 902 0x88c4, 0x001f3ae3, 0x000000c2, 903 0x88d4, 0x0000001f, 0x00000010, 904 0x8974, 0xffffffff, 0x00000000 905 }; 906 907 static void evergreen_init_golden_registers(struct radeon_device *rdev) 908 { 909 switch (rdev->family) { 910 case CHIP_CYPRESS: 911 case CHIP_HEMLOCK: 912 radeon_program_register_sequence(rdev, 913 evergreen_golden_registers, 914 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 915 radeon_program_register_sequence(rdev, 916 evergreen_golden_registers2, 917 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 918 radeon_program_register_sequence(rdev, 919 cypress_mgcg_init, 920 (const u32)ARRAY_SIZE(cypress_mgcg_init)); 921 break; 922 case CHIP_JUNIPER: 923 radeon_program_register_sequence(rdev, 924 evergreen_golden_registers, 925 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 926 radeon_program_register_sequence(rdev, 927 evergreen_golden_registers2, 928 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 929 radeon_program_register_sequence(rdev, 930 juniper_mgcg_init, 931 (const u32)ARRAY_SIZE(juniper_mgcg_init)); 932 break; 933 case CHIP_REDWOOD: 934 radeon_program_register_sequence(rdev, 935 evergreen_golden_registers, 936 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 937 radeon_program_register_sequence(rdev, 938 evergreen_golden_registers2, 939 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 940 radeon_program_register_sequence(rdev, 941 redwood_mgcg_init, 942 (const u32)ARRAY_SIZE(redwood_mgcg_init)); 943 break; 944 case CHIP_CEDAR: 945 radeon_program_register_sequence(rdev, 946 cedar_golden_registers, 947 (const u32)ARRAY_SIZE(cedar_golden_registers)); 948 radeon_program_register_sequence(rdev, 949 evergreen_golden_registers2, 950 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 951 radeon_program_register_sequence(rdev, 952 cedar_mgcg_init, 953 (const u32)ARRAY_SIZE(cedar_mgcg_init)); 954 break; 955 case CHIP_PALM: 956 radeon_program_register_sequence(rdev, 957 wrestler_golden_registers, 958 (const u32)ARRAY_SIZE(wrestler_golden_registers)); 959 break; 960 case CHIP_SUMO: 961 radeon_program_register_sequence(rdev, 962 supersumo_golden_registers, 963 (const u32)ARRAY_SIZE(supersumo_golden_registers)); 964 break; 965 case CHIP_SUMO2: 966 radeon_program_register_sequence(rdev, 967 supersumo_golden_registers, 968 (const u32)ARRAY_SIZE(supersumo_golden_registers)); 969 radeon_program_register_sequence(rdev, 970 sumo_golden_registers, 971 (const u32)ARRAY_SIZE(sumo_golden_registers)); 972 break; 973 case CHIP_BARTS: 974 radeon_program_register_sequence(rdev, 975 barts_golden_registers, 976 (const u32)ARRAY_SIZE(barts_golden_registers)); 977 break; 978 case CHIP_TURKS: 979 radeon_program_register_sequence(rdev, 980 turks_golden_registers, 981 (const u32)ARRAY_SIZE(turks_golden_registers)); 982 break; 983 case CHIP_CAICOS: 984 radeon_program_register_sequence(rdev, 985 caicos_golden_registers, 986 (const u32)ARRAY_SIZE(caicos_golden_registers)); 987 break; 988 default: 989 break; 990 } 991 } 992 993 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw, 994 unsigned *bankh, unsigned *mtaspect, 995 unsigned *tile_split) 996 { 997 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK; 998 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK; 999 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK; 1000 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK; 1001 switch (*bankw) { 1002 default: 1003 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break; 1004 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break; 1005 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break; 1006 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break; 1007 } 1008 switch (*bankh) { 1009 default: 1010 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break; 1011 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break; 1012 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break; 1013 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break; 1014 } 1015 switch (*mtaspect) { 1016 default: 1017 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break; 1018 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break; 1019 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break; 1020 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break; 1021 } 1022 } 1023 1024 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock, 1025 u32 cntl_reg, u32 status_reg) 1026 { 1027 int r, i; 1028 struct atom_clock_dividers dividers; 1029 1030 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, 1031 clock, false, ÷rs); 1032 if (r) 1033 return r; 1034 1035 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK)); 1036 1037 for (i = 0; i < 100; i++) { 1038 if (RREG32(status_reg) & DCLK_STATUS) 1039 break; 1040 mdelay(10); 1041 } 1042 if (i == 100) 1043 return -ETIMEDOUT; 1044 1045 return 0; 1046 } 1047 1048 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 1049 { 1050 int r = 0; 1051 u32 cg_scratch = RREG32(CG_SCRATCH1); 1052 1053 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); 1054 if (r) 1055 goto done; 1056 cg_scratch &= 0xffff0000; 1057 cg_scratch |= vclk / 100; /* Mhz */ 1058 1059 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); 1060 if (r) 1061 goto done; 1062 cg_scratch &= 0x0000ffff; 1063 cg_scratch |= (dclk / 100) << 16; /* Mhz */ 1064 1065 done: 1066 WREG32(CG_SCRATCH1, cg_scratch); 1067 1068 return r; 1069 } 1070 1071 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 1072 { 1073 /* start off with something large */ 1074 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0; 1075 int r; 1076 1077 /* bypass vclk and dclk with bclk */ 1078 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1079 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1), 1080 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 1081 1082 /* put PLL in bypass mode */ 1083 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK); 1084 1085 if (!vclk || !dclk) { 1086 /* keep the Bypass mode, put PLL to sleep */ 1087 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 1088 return 0; 1089 } 1090 1091 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000, 1092 16384, 0x03FFFFFF, 0, 128, 5, 1093 &fb_div, &vclk_div, &dclk_div); 1094 if (r) 1095 return r; 1096 1097 /* set VCO_MODE to 1 */ 1098 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK); 1099 1100 /* toggle UPLL_SLEEP to 1 then back to 0 */ 1101 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 1102 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK); 1103 1104 /* deassert UPLL_RESET */ 1105 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 1106 1107 mdelay(1); 1108 1109 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 1110 if (r) 1111 return r; 1112 1113 /* assert UPLL_RESET again */ 1114 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK); 1115 1116 /* disable spread spectrum. */ 1117 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK); 1118 1119 /* set feedback divider */ 1120 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK); 1121 1122 /* set ref divider to 0 */ 1123 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK); 1124 1125 if (fb_div < 307200) 1126 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9); 1127 else 1128 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9); 1129 1130 /* set PDIV_A and PDIV_B */ 1131 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1132 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div), 1133 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK)); 1134 1135 /* give the PLL some time to settle */ 1136 mdelay(15); 1137 1138 /* deassert PLL_RESET */ 1139 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 1140 1141 mdelay(15); 1142 1143 /* switch from bypass mode to normal mode */ 1144 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK); 1145 1146 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 1147 if (r) 1148 return r; 1149 1150 /* switch VCLK and DCLK selection */ 1151 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1152 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2), 1153 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 1154 1155 mdelay(100); 1156 1157 return 0; 1158 } 1159 1160 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev) 1161 { 1162 int readrq; 1163 u16 v; 1164 1165 readrq = pcie_get_readrq(rdev->pdev); 1166 v = ffs(readrq) - 8; 1167 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it 1168 * to avoid hangs or perfomance issues 1169 */ 1170 if ((v == 0) || (v == 6) || (v == 7)) 1171 pcie_set_readrq(rdev->pdev, 512); 1172 } 1173 1174 void dce4_program_fmt(struct drm_encoder *encoder) 1175 { 1176 struct drm_device *dev = encoder->dev; 1177 struct radeon_device *rdev = dev->dev_private; 1178 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); 1179 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); 1180 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); 1181 int bpc = 0; 1182 u32 tmp = 0; 1183 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; 1184 1185 if (connector) { 1186 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1187 bpc = radeon_get_monitor_bpc(connector); 1188 dither = radeon_connector->dither; 1189 } 1190 1191 /* LVDS/eDP FMT is set up by atom */ 1192 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) 1193 return; 1194 1195 /* not needed for analog */ 1196 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || 1197 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) 1198 return; 1199 1200 if (bpc == 0) 1201 return; 1202 1203 switch (bpc) { 1204 case 6: 1205 if (dither == RADEON_FMT_DITHER_ENABLE) 1206 /* XXX sort out optimal dither settings */ 1207 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 1208 FMT_SPATIAL_DITHER_EN); 1209 else 1210 tmp |= FMT_TRUNCATE_EN; 1211 break; 1212 case 8: 1213 if (dither == RADEON_FMT_DITHER_ENABLE) 1214 /* XXX sort out optimal dither settings */ 1215 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 1216 FMT_RGB_RANDOM_ENABLE | 1217 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH); 1218 else 1219 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH); 1220 break; 1221 case 10: 1222 default: 1223 /* not needed */ 1224 break; 1225 } 1226 1227 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); 1228 } 1229 1230 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc) 1231 { 1232 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK) 1233 return true; 1234 else 1235 return false; 1236 } 1237 1238 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc) 1239 { 1240 u32 pos1, pos2; 1241 1242 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]); 1243 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]); 1244 1245 if (pos1 != pos2) 1246 return true; 1247 else 1248 return false; 1249 } 1250 1251 /** 1252 * dce4_wait_for_vblank - vblank wait asic callback. 1253 * 1254 * @rdev: radeon_device pointer 1255 * @crtc: crtc to wait for vblank on 1256 * 1257 * Wait for vblank on the requested crtc (evergreen+). 1258 */ 1259 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc) 1260 { 1261 unsigned i = 0; 1262 1263 if (crtc >= rdev->num_crtc) 1264 return; 1265 1266 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN)) 1267 return; 1268 1269 /* depending on when we hit vblank, we may be close to active; if so, 1270 * wait for another frame. 1271 */ 1272 while (dce4_is_in_vblank(rdev, crtc)) { 1273 if (i++ % 100 == 0) { 1274 if (!dce4_is_counter_moving(rdev, crtc)) 1275 break; 1276 } 1277 } 1278 1279 while (!dce4_is_in_vblank(rdev, crtc)) { 1280 if (i++ % 100 == 0) { 1281 if (!dce4_is_counter_moving(rdev, crtc)) 1282 break; 1283 } 1284 } 1285 } 1286 1287 /** 1288 * evergreen_page_flip - pageflip callback. 1289 * 1290 * @rdev: radeon_device pointer 1291 * @crtc_id: crtc to cleanup pageflip on 1292 * @crtc_base: new address of the crtc (GPU MC address) 1293 * 1294 * Does the actual pageflip (evergreen+). 1295 * During vblank we take the crtc lock and wait for the update_pending 1296 * bit to go high, when it does, we release the lock, and allow the 1297 * double buffered update to take place. 1298 * Returns the current update pending status. 1299 */ 1300 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) 1301 { 1302 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; 1303 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset); 1304 int i; 1305 1306 /* Lock the graphics update lock */ 1307 tmp |= EVERGREEN_GRPH_UPDATE_LOCK; 1308 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); 1309 1310 /* update the scanout addresses */ 1311 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, 1312 upper_32_bits(crtc_base)); 1313 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, 1314 (u32)crtc_base); 1315 1316 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, 1317 upper_32_bits(crtc_base)); 1318 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, 1319 (u32)crtc_base); 1320 1321 /* Wait for update_pending to go high. */ 1322 for (i = 0; i < rdev->usec_timeout; i++) { 1323 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) 1324 break; 1325 udelay(1); 1326 } 1327 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n"); 1328 1329 /* Unlock the lock, so double-buffering can take place inside vblank */ 1330 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK; 1331 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); 1332 } 1333 1334 /** 1335 * evergreen_page_flip_pending - check if page flip is still pending 1336 * 1337 * @rdev: radeon_device pointer 1338 * @crtc_id: crtc to check 1339 * 1340 * Returns the current update pending status. 1341 */ 1342 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id) 1343 { 1344 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; 1345 1346 /* Return current update_pending status: */ 1347 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & 1348 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING); 1349 } 1350 1351 /* get temperature in millidegrees */ 1352 int evergreen_get_temp(struct radeon_device *rdev) 1353 { 1354 u32 temp, toffset; 1355 int actual_temp = 0; 1356 1357 if (rdev->family == CHIP_JUNIPER) { 1358 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >> 1359 TOFFSET_SHIFT; 1360 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >> 1361 TS0_ADC_DOUT_SHIFT; 1362 1363 if (toffset & 0x100) 1364 actual_temp = temp / 2 - (0x200 - toffset); 1365 else 1366 actual_temp = temp / 2 + toffset; 1367 1368 actual_temp = actual_temp * 1000; 1369 1370 } else { 1371 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >> 1372 ASIC_T_SHIFT; 1373 1374 if (temp & 0x400) 1375 actual_temp = -256; 1376 else if (temp & 0x200) 1377 actual_temp = 255; 1378 else if (temp & 0x100) { 1379 actual_temp = temp & 0x1ff; 1380 actual_temp |= ~0x1ff; 1381 } else 1382 actual_temp = temp & 0xff; 1383 1384 actual_temp = (actual_temp * 1000) / 2; 1385 } 1386 1387 return actual_temp; 1388 } 1389 1390 int sumo_get_temp(struct radeon_device *rdev) 1391 { 1392 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff; 1393 int actual_temp = temp - 49; 1394 1395 return actual_temp * 1000; 1396 } 1397 1398 /** 1399 * sumo_pm_init_profile - Initialize power profiles callback. 1400 * 1401 * @rdev: radeon_device pointer 1402 * 1403 * Initialize the power states used in profile mode 1404 * (sumo, trinity, SI). 1405 * Used for profile mode only. 1406 */ 1407 void sumo_pm_init_profile(struct radeon_device *rdev) 1408 { 1409 int idx; 1410 1411 /* default */ 1412 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index; 1413 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index; 1414 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0; 1415 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0; 1416 1417 /* low,mid sh/mh */ 1418 if (rdev->flags & RADEON_IS_MOBILITY) 1419 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0); 1420 else 1421 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1422 1423 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx; 1424 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx; 1425 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0; 1426 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0; 1427 1428 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx; 1429 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx; 1430 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0; 1431 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0; 1432 1433 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx; 1434 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx; 1435 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0; 1436 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0; 1437 1438 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx; 1439 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx; 1440 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0; 1441 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0; 1442 1443 /* high sh/mh */ 1444 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1445 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx; 1446 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx; 1447 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0; 1448 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 1449 rdev->pm.power_state[idx].num_clock_modes - 1; 1450 1451 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx; 1452 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx; 1453 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0; 1454 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 1455 rdev->pm.power_state[idx].num_clock_modes - 1; 1456 } 1457 1458 /** 1459 * btc_pm_init_profile - Initialize power profiles callback. 1460 * 1461 * @rdev: radeon_device pointer 1462 * 1463 * Initialize the power states used in profile mode 1464 * (BTC, cayman). 1465 * Used for profile mode only. 1466 */ 1467 void btc_pm_init_profile(struct radeon_device *rdev) 1468 { 1469 int idx; 1470 1471 /* default */ 1472 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index; 1473 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index; 1474 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0; 1475 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2; 1476 /* starting with BTC, there is one state that is used for both 1477 * MH and SH. Difference is that we always use the high clock index for 1478 * mclk. 1479 */ 1480 if (rdev->flags & RADEON_IS_MOBILITY) 1481 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0); 1482 else 1483 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1484 /* low sh */ 1485 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx; 1486 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx; 1487 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0; 1488 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0; 1489 /* mid sh */ 1490 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx; 1491 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx; 1492 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0; 1493 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1; 1494 /* high sh */ 1495 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx; 1496 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx; 1497 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0; 1498 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2; 1499 /* low mh */ 1500 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx; 1501 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx; 1502 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0; 1503 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0; 1504 /* mid mh */ 1505 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx; 1506 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx; 1507 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0; 1508 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1; 1509 /* high mh */ 1510 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx; 1511 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx; 1512 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0; 1513 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2; 1514 } 1515 1516 /** 1517 * evergreen_pm_misc - set additional pm hw parameters callback. 1518 * 1519 * @rdev: radeon_device pointer 1520 * 1521 * Set non-clock parameters associated with a power state 1522 * (voltage, etc.) (evergreen+). 1523 */ 1524 void evergreen_pm_misc(struct radeon_device *rdev) 1525 { 1526 int req_ps_idx = rdev->pm.requested_power_state_index; 1527 int req_cm_idx = rdev->pm.requested_clock_mode_index; 1528 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx]; 1529 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage; 1530 1531 if (voltage->type == VOLTAGE_SW) { 1532 /* 0xff0x are flags rather then an actual voltage */ 1533 if ((voltage->voltage & 0xff00) == 0xff00) 1534 return; 1535 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) { 1536 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC); 1537 rdev->pm.current_vddc = voltage->voltage; 1538 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage); 1539 } 1540 1541 /* starting with BTC, there is one state that is used for both 1542 * MH and SH. Difference is that we always use the high clock index for 1543 * mclk and vddci. 1544 */ 1545 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) && 1546 (rdev->family >= CHIP_BARTS) && 1547 rdev->pm.active_crtc_count && 1548 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) || 1549 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX))) 1550 voltage = &rdev->pm.power_state[req_ps_idx]. 1551 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage; 1552 1553 /* 0xff0x are flags rather then an actual voltage */ 1554 if ((voltage->vddci & 0xff00) == 0xff00) 1555 return; 1556 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) { 1557 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI); 1558 rdev->pm.current_vddci = voltage->vddci; 1559 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci); 1560 } 1561 } 1562 } 1563 1564 /** 1565 * evergreen_pm_prepare - pre-power state change callback. 1566 * 1567 * @rdev: radeon_device pointer 1568 * 1569 * Prepare for a power state change (evergreen+). 1570 */ 1571 void evergreen_pm_prepare(struct radeon_device *rdev) 1572 { 1573 struct drm_device *ddev = rdev->ddev; 1574 struct drm_crtc *crtc; 1575 struct radeon_crtc *radeon_crtc; 1576 u32 tmp; 1577 1578 /* disable any active CRTCs */ 1579 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 1580 radeon_crtc = to_radeon_crtc(crtc); 1581 if (radeon_crtc->enabled) { 1582 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 1583 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 1584 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 1585 } 1586 } 1587 } 1588 1589 /** 1590 * evergreen_pm_finish - post-power state change callback. 1591 * 1592 * @rdev: radeon_device pointer 1593 * 1594 * Clean up after a power state change (evergreen+). 1595 */ 1596 void evergreen_pm_finish(struct radeon_device *rdev) 1597 { 1598 struct drm_device *ddev = rdev->ddev; 1599 struct drm_crtc *crtc; 1600 struct radeon_crtc *radeon_crtc; 1601 u32 tmp; 1602 1603 /* enable any active CRTCs */ 1604 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 1605 radeon_crtc = to_radeon_crtc(crtc); 1606 if (radeon_crtc->enabled) { 1607 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 1608 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 1609 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 1610 } 1611 } 1612 } 1613 1614 /** 1615 * evergreen_hpd_sense - hpd sense callback. 1616 * 1617 * @rdev: radeon_device pointer 1618 * @hpd: hpd (hotplug detect) pin 1619 * 1620 * Checks if a digital monitor is connected (evergreen+). 1621 * Returns true if connected, false if not connected. 1622 */ 1623 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd) 1624 { 1625 bool connected = false; 1626 1627 switch (hpd) { 1628 case RADEON_HPD_1: 1629 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE) 1630 connected = true; 1631 break; 1632 case RADEON_HPD_2: 1633 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE) 1634 connected = true; 1635 break; 1636 case RADEON_HPD_3: 1637 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE) 1638 connected = true; 1639 break; 1640 case RADEON_HPD_4: 1641 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE) 1642 connected = true; 1643 break; 1644 case RADEON_HPD_5: 1645 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE) 1646 connected = true; 1647 break; 1648 case RADEON_HPD_6: 1649 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE) 1650 connected = true; 1651 break; 1652 default: 1653 break; 1654 } 1655 1656 return connected; 1657 } 1658 1659 /** 1660 * evergreen_hpd_set_polarity - hpd set polarity callback. 1661 * 1662 * @rdev: radeon_device pointer 1663 * @hpd: hpd (hotplug detect) pin 1664 * 1665 * Set the polarity of the hpd pin (evergreen+). 1666 */ 1667 void evergreen_hpd_set_polarity(struct radeon_device *rdev, 1668 enum radeon_hpd_id hpd) 1669 { 1670 u32 tmp; 1671 bool connected = evergreen_hpd_sense(rdev, hpd); 1672 1673 switch (hpd) { 1674 case RADEON_HPD_1: 1675 tmp = RREG32(DC_HPD1_INT_CONTROL); 1676 if (connected) 1677 tmp &= ~DC_HPDx_INT_POLARITY; 1678 else 1679 tmp |= DC_HPDx_INT_POLARITY; 1680 WREG32(DC_HPD1_INT_CONTROL, tmp); 1681 break; 1682 case RADEON_HPD_2: 1683 tmp = RREG32(DC_HPD2_INT_CONTROL); 1684 if (connected) 1685 tmp &= ~DC_HPDx_INT_POLARITY; 1686 else 1687 tmp |= DC_HPDx_INT_POLARITY; 1688 WREG32(DC_HPD2_INT_CONTROL, tmp); 1689 break; 1690 case RADEON_HPD_3: 1691 tmp = RREG32(DC_HPD3_INT_CONTROL); 1692 if (connected) 1693 tmp &= ~DC_HPDx_INT_POLARITY; 1694 else 1695 tmp |= DC_HPDx_INT_POLARITY; 1696 WREG32(DC_HPD3_INT_CONTROL, tmp); 1697 break; 1698 case RADEON_HPD_4: 1699 tmp = RREG32(DC_HPD4_INT_CONTROL); 1700 if (connected) 1701 tmp &= ~DC_HPDx_INT_POLARITY; 1702 else 1703 tmp |= DC_HPDx_INT_POLARITY; 1704 WREG32(DC_HPD4_INT_CONTROL, tmp); 1705 break; 1706 case RADEON_HPD_5: 1707 tmp = RREG32(DC_HPD5_INT_CONTROL); 1708 if (connected) 1709 tmp &= ~DC_HPDx_INT_POLARITY; 1710 else 1711 tmp |= DC_HPDx_INT_POLARITY; 1712 WREG32(DC_HPD5_INT_CONTROL, tmp); 1713 break; 1714 case RADEON_HPD_6: 1715 tmp = RREG32(DC_HPD6_INT_CONTROL); 1716 if (connected) 1717 tmp &= ~DC_HPDx_INT_POLARITY; 1718 else 1719 tmp |= DC_HPDx_INT_POLARITY; 1720 WREG32(DC_HPD6_INT_CONTROL, tmp); 1721 break; 1722 default: 1723 break; 1724 } 1725 } 1726 1727 /** 1728 * evergreen_hpd_init - hpd setup callback. 1729 * 1730 * @rdev: radeon_device pointer 1731 * 1732 * Setup the hpd pins used by the card (evergreen+). 1733 * Enable the pin, set the polarity, and enable the hpd interrupts. 1734 */ 1735 void evergreen_hpd_init(struct radeon_device *rdev) 1736 { 1737 struct drm_device *dev = rdev->ddev; 1738 struct drm_connector *connector; 1739 unsigned enabled = 0; 1740 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | 1741 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN; 1742 1743 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 1744 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1745 1746 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP || 1747 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) { 1748 /* don't try to enable hpd on eDP or LVDS avoid breaking the 1749 * aux dp channel on imac and help (but not completely fix) 1750 * https://bugzilla.redhat.com/show_bug.cgi?id=726143 1751 * also avoid interrupt storms during dpms. 1752 */ 1753 continue; 1754 } 1755 switch (radeon_connector->hpd.hpd) { 1756 case RADEON_HPD_1: 1757 WREG32(DC_HPD1_CONTROL, tmp); 1758 break; 1759 case RADEON_HPD_2: 1760 WREG32(DC_HPD2_CONTROL, tmp); 1761 break; 1762 case RADEON_HPD_3: 1763 WREG32(DC_HPD3_CONTROL, tmp); 1764 break; 1765 case RADEON_HPD_4: 1766 WREG32(DC_HPD4_CONTROL, tmp); 1767 break; 1768 case RADEON_HPD_5: 1769 WREG32(DC_HPD5_CONTROL, tmp); 1770 break; 1771 case RADEON_HPD_6: 1772 WREG32(DC_HPD6_CONTROL, tmp); 1773 break; 1774 default: 1775 break; 1776 } 1777 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd); 1778 enabled |= 1 << radeon_connector->hpd.hpd; 1779 } 1780 radeon_irq_kms_enable_hpd(rdev, enabled); 1781 } 1782 1783 /** 1784 * evergreen_hpd_fini - hpd tear down callback. 1785 * 1786 * @rdev: radeon_device pointer 1787 * 1788 * Tear down the hpd pins used by the card (evergreen+). 1789 * Disable the hpd interrupts. 1790 */ 1791 void evergreen_hpd_fini(struct radeon_device *rdev) 1792 { 1793 struct drm_device *dev = rdev->ddev; 1794 struct drm_connector *connector; 1795 unsigned disabled = 0; 1796 1797 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 1798 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1799 switch (radeon_connector->hpd.hpd) { 1800 case RADEON_HPD_1: 1801 WREG32(DC_HPD1_CONTROL, 0); 1802 break; 1803 case RADEON_HPD_2: 1804 WREG32(DC_HPD2_CONTROL, 0); 1805 break; 1806 case RADEON_HPD_3: 1807 WREG32(DC_HPD3_CONTROL, 0); 1808 break; 1809 case RADEON_HPD_4: 1810 WREG32(DC_HPD4_CONTROL, 0); 1811 break; 1812 case RADEON_HPD_5: 1813 WREG32(DC_HPD5_CONTROL, 0); 1814 break; 1815 case RADEON_HPD_6: 1816 WREG32(DC_HPD6_CONTROL, 0); 1817 break; 1818 default: 1819 break; 1820 } 1821 disabled |= 1 << radeon_connector->hpd.hpd; 1822 } 1823 radeon_irq_kms_disable_hpd(rdev, disabled); 1824 } 1825 1826 /* watermark setup */ 1827 1828 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev, 1829 struct radeon_crtc *radeon_crtc, 1830 struct drm_display_mode *mode, 1831 struct drm_display_mode *other_mode) 1832 { 1833 u32 tmp, buffer_alloc, i; 1834 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 1835 /* 1836 * Line Buffer Setup 1837 * There are 3 line buffers, each one shared by 2 display controllers. 1838 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between 1839 * the display controllers. The paritioning is done via one of four 1840 * preset allocations specified in bits 2:0: 1841 * first display controller 1842 * 0 - first half of lb (3840 * 2) 1843 * 1 - first 3/4 of lb (5760 * 2) 1844 * 2 - whole lb (7680 * 2), other crtc must be disabled 1845 * 3 - first 1/4 of lb (1920 * 2) 1846 * second display controller 1847 * 4 - second half of lb (3840 * 2) 1848 * 5 - second 3/4 of lb (5760 * 2) 1849 * 6 - whole lb (7680 * 2), other crtc must be disabled 1850 * 7 - last 1/4 of lb (1920 * 2) 1851 */ 1852 /* this can get tricky if we have two large displays on a paired group 1853 * of crtcs. Ideally for multiple large displays we'd assign them to 1854 * non-linked crtcs for maximum line buffer allocation. 1855 */ 1856 if (radeon_crtc->base.enabled && mode) { 1857 if (other_mode) { 1858 tmp = 0; /* 1/2 */ 1859 buffer_alloc = 1; 1860 } else { 1861 tmp = 2; /* whole */ 1862 buffer_alloc = 2; 1863 } 1864 } else { 1865 tmp = 0; 1866 buffer_alloc = 0; 1867 } 1868 1869 /* second controller of the pair uses second half of the lb */ 1870 if (radeon_crtc->crtc_id % 2) 1871 tmp += 4; 1872 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp); 1873 1874 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) { 1875 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 1876 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 1877 for (i = 0; i < rdev->usec_timeout; i++) { 1878 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 1879 DMIF_BUFFERS_ALLOCATED_COMPLETED) 1880 break; 1881 udelay(1); 1882 } 1883 } 1884 1885 if (radeon_crtc->base.enabled && mode) { 1886 switch (tmp) { 1887 case 0: 1888 case 4: 1889 default: 1890 if (ASIC_IS_DCE5(rdev)) 1891 return 4096 * 2; 1892 else 1893 return 3840 * 2; 1894 case 1: 1895 case 5: 1896 if (ASIC_IS_DCE5(rdev)) 1897 return 6144 * 2; 1898 else 1899 return 5760 * 2; 1900 case 2: 1901 case 6: 1902 if (ASIC_IS_DCE5(rdev)) 1903 return 8192 * 2; 1904 else 1905 return 7680 * 2; 1906 case 3: 1907 case 7: 1908 if (ASIC_IS_DCE5(rdev)) 1909 return 2048 * 2; 1910 else 1911 return 1920 * 2; 1912 } 1913 } 1914 1915 /* controller not enabled, so no lb used */ 1916 return 0; 1917 } 1918 1919 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev) 1920 { 1921 u32 tmp = RREG32(MC_SHARED_CHMAP); 1922 1923 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 1924 case 0: 1925 default: 1926 return 1; 1927 case 1: 1928 return 2; 1929 case 2: 1930 return 4; 1931 case 3: 1932 return 8; 1933 } 1934 } 1935 1936 struct evergreen_wm_params { 1937 u32 dram_channels; /* number of dram channels */ 1938 u32 yclk; /* bandwidth per dram data pin in kHz */ 1939 u32 sclk; /* engine clock in kHz */ 1940 u32 disp_clk; /* display clock in kHz */ 1941 u32 src_width; /* viewport width */ 1942 u32 active_time; /* active display time in ns */ 1943 u32 blank_time; /* blank time in ns */ 1944 bool interlaced; /* mode is interlaced */ 1945 fixed20_12 vsc; /* vertical scale ratio */ 1946 u32 num_heads; /* number of active crtcs */ 1947 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 1948 u32 lb_size; /* line buffer allocated to pipe */ 1949 u32 vtaps; /* vertical scaler taps */ 1950 }; 1951 1952 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm) 1953 { 1954 /* Calculate DRAM Bandwidth and the part allocated to display. */ 1955 fixed20_12 dram_efficiency; /* 0.7 */ 1956 fixed20_12 yclk, dram_channels, bandwidth; 1957 fixed20_12 a; 1958 1959 a.full = dfixed_const(1000); 1960 yclk.full = dfixed_const(wm->yclk); 1961 yclk.full = dfixed_div(yclk, a); 1962 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1963 a.full = dfixed_const(10); 1964 dram_efficiency.full = dfixed_const(7); 1965 dram_efficiency.full = dfixed_div(dram_efficiency, a); 1966 bandwidth.full = dfixed_mul(dram_channels, yclk); 1967 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 1968 1969 return dfixed_trunc(bandwidth); 1970 } 1971 1972 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 1973 { 1974 /* Calculate DRAM Bandwidth and the part allocated to display. */ 1975 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 1976 fixed20_12 yclk, dram_channels, bandwidth; 1977 fixed20_12 a; 1978 1979 a.full = dfixed_const(1000); 1980 yclk.full = dfixed_const(wm->yclk); 1981 yclk.full = dfixed_div(yclk, a); 1982 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1983 a.full = dfixed_const(10); 1984 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 1985 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 1986 bandwidth.full = dfixed_mul(dram_channels, yclk); 1987 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 1988 1989 return dfixed_trunc(bandwidth); 1990 } 1991 1992 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm) 1993 { 1994 /* Calculate the display Data return Bandwidth */ 1995 fixed20_12 return_efficiency; /* 0.8 */ 1996 fixed20_12 sclk, bandwidth; 1997 fixed20_12 a; 1998 1999 a.full = dfixed_const(1000); 2000 sclk.full = dfixed_const(wm->sclk); 2001 sclk.full = dfixed_div(sclk, a); 2002 a.full = dfixed_const(10); 2003 return_efficiency.full = dfixed_const(8); 2004 return_efficiency.full = dfixed_div(return_efficiency, a); 2005 a.full = dfixed_const(32); 2006 bandwidth.full = dfixed_mul(a, sclk); 2007 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 2008 2009 return dfixed_trunc(bandwidth); 2010 } 2011 2012 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm) 2013 { 2014 /* Calculate the DMIF Request Bandwidth */ 2015 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 2016 fixed20_12 disp_clk, bandwidth; 2017 fixed20_12 a; 2018 2019 a.full = dfixed_const(1000); 2020 disp_clk.full = dfixed_const(wm->disp_clk); 2021 disp_clk.full = dfixed_div(disp_clk, a); 2022 a.full = dfixed_const(10); 2023 disp_clk_request_efficiency.full = dfixed_const(8); 2024 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 2025 a.full = dfixed_const(32); 2026 bandwidth.full = dfixed_mul(a, disp_clk); 2027 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency); 2028 2029 return dfixed_trunc(bandwidth); 2030 } 2031 2032 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm) 2033 { 2034 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 2035 u32 dram_bandwidth = evergreen_dram_bandwidth(wm); 2036 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm); 2037 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm); 2038 2039 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 2040 } 2041 2042 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm) 2043 { 2044 /* Calculate the display mode Average Bandwidth 2045 * DisplayMode should contain the source and destination dimensions, 2046 * timing, etc. 2047 */ 2048 fixed20_12 bpp; 2049 fixed20_12 line_time; 2050 fixed20_12 src_width; 2051 fixed20_12 bandwidth; 2052 fixed20_12 a; 2053 2054 a.full = dfixed_const(1000); 2055 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 2056 line_time.full = dfixed_div(line_time, a); 2057 bpp.full = dfixed_const(wm->bytes_per_pixel); 2058 src_width.full = dfixed_const(wm->src_width); 2059 bandwidth.full = dfixed_mul(src_width, bpp); 2060 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 2061 bandwidth.full = dfixed_div(bandwidth, line_time); 2062 2063 return dfixed_trunc(bandwidth); 2064 } 2065 2066 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm) 2067 { 2068 /* First calcualte the latency in ns */ 2069 u32 mc_latency = 2000; /* 2000 ns. */ 2070 u32 available_bandwidth = evergreen_available_bandwidth(wm); 2071 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 2072 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 2073 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 2074 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 2075 (wm->num_heads * cursor_line_pair_return_time); 2076 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 2077 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 2078 fixed20_12 a, b, c; 2079 2080 if (wm->num_heads == 0) 2081 return 0; 2082 2083 a.full = dfixed_const(2); 2084 b.full = dfixed_const(1); 2085 if ((wm->vsc.full > a.full) || 2086 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 2087 (wm->vtaps >= 5) || 2088 ((wm->vsc.full >= a.full) && wm->interlaced)) 2089 max_src_lines_per_dst_line = 4; 2090 else 2091 max_src_lines_per_dst_line = 2; 2092 2093 a.full = dfixed_const(available_bandwidth); 2094 b.full = dfixed_const(wm->num_heads); 2095 a.full = dfixed_div(a, b); 2096 2097 b.full = dfixed_const(1000); 2098 c.full = dfixed_const(wm->disp_clk); 2099 b.full = dfixed_div(c, b); 2100 c.full = dfixed_const(wm->bytes_per_pixel); 2101 b.full = dfixed_mul(b, c); 2102 2103 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b)); 2104 2105 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 2106 b.full = dfixed_const(1000); 2107 c.full = dfixed_const(lb_fill_bw); 2108 b.full = dfixed_div(c, b); 2109 a.full = dfixed_div(a, b); 2110 line_fill_time = dfixed_trunc(a); 2111 2112 if (line_fill_time < wm->active_time) 2113 return latency; 2114 else 2115 return latency + (line_fill_time - wm->active_time); 2116 2117 } 2118 2119 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 2120 { 2121 if (evergreen_average_bandwidth(wm) <= 2122 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads)) 2123 return true; 2124 else 2125 return false; 2126 }; 2127 2128 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm) 2129 { 2130 if (evergreen_average_bandwidth(wm) <= 2131 (evergreen_available_bandwidth(wm) / wm->num_heads)) 2132 return true; 2133 else 2134 return false; 2135 }; 2136 2137 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm) 2138 { 2139 u32 lb_partitions = wm->lb_size / wm->src_width; 2140 u32 line_time = wm->active_time + wm->blank_time; 2141 u32 latency_tolerant_lines; 2142 u32 latency_hiding; 2143 fixed20_12 a; 2144 2145 a.full = dfixed_const(1); 2146 if (wm->vsc.full > a.full) 2147 latency_tolerant_lines = 1; 2148 else { 2149 if (lb_partitions <= (wm->vtaps + 1)) 2150 latency_tolerant_lines = 1; 2151 else 2152 latency_tolerant_lines = 2; 2153 } 2154 2155 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 2156 2157 if (evergreen_latency_watermark(wm) <= latency_hiding) 2158 return true; 2159 else 2160 return false; 2161 } 2162 2163 static void evergreen_program_watermarks(struct radeon_device *rdev, 2164 struct radeon_crtc *radeon_crtc, 2165 u32 lb_size, u32 num_heads) 2166 { 2167 struct drm_display_mode *mode = &radeon_crtc->base.mode; 2168 struct evergreen_wm_params wm_low, wm_high; 2169 u32 dram_channels; 2170 u32 pixel_period; 2171 u32 line_time = 0; 2172 u32 latency_watermark_a = 0, latency_watermark_b = 0; 2173 u32 priority_a_mark = 0, priority_b_mark = 0; 2174 u32 priority_a_cnt = PRIORITY_OFF; 2175 u32 priority_b_cnt = PRIORITY_OFF; 2176 u32 pipe_offset = radeon_crtc->crtc_id * 16; 2177 u32 tmp, arb_control3; 2178 fixed20_12 a, b, c; 2179 2180 if (radeon_crtc->base.enabled && num_heads && mode) { 2181 pixel_period = 1000000 / (u32)mode->clock; 2182 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 2183 priority_a_cnt = 0; 2184 priority_b_cnt = 0; 2185 dram_channels = evergreen_get_number_of_dram_channels(rdev); 2186 2187 /* watermark for high clocks */ 2188 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2189 wm_high.yclk = 2190 radeon_dpm_get_mclk(rdev, false) * 10; 2191 wm_high.sclk = 2192 radeon_dpm_get_sclk(rdev, false) * 10; 2193 } else { 2194 wm_high.yclk = rdev->pm.current_mclk * 10; 2195 wm_high.sclk = rdev->pm.current_sclk * 10; 2196 } 2197 2198 wm_high.disp_clk = mode->clock; 2199 wm_high.src_width = mode->crtc_hdisplay; 2200 wm_high.active_time = mode->crtc_hdisplay * pixel_period; 2201 wm_high.blank_time = line_time - wm_high.active_time; 2202 wm_high.interlaced = false; 2203 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2204 wm_high.interlaced = true; 2205 wm_high.vsc = radeon_crtc->vsc; 2206 wm_high.vtaps = 1; 2207 if (radeon_crtc->rmx_type != RMX_OFF) 2208 wm_high.vtaps = 2; 2209 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2210 wm_high.lb_size = lb_size; 2211 wm_high.dram_channels = dram_channels; 2212 wm_high.num_heads = num_heads; 2213 2214 /* watermark for low clocks */ 2215 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2216 wm_low.yclk = 2217 radeon_dpm_get_mclk(rdev, true) * 10; 2218 wm_low.sclk = 2219 radeon_dpm_get_sclk(rdev, true) * 10; 2220 } else { 2221 wm_low.yclk = rdev->pm.current_mclk * 10; 2222 wm_low.sclk = rdev->pm.current_sclk * 10; 2223 } 2224 2225 wm_low.disp_clk = mode->clock; 2226 wm_low.src_width = mode->crtc_hdisplay; 2227 wm_low.active_time = mode->crtc_hdisplay * pixel_period; 2228 wm_low.blank_time = line_time - wm_low.active_time; 2229 wm_low.interlaced = false; 2230 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2231 wm_low.interlaced = true; 2232 wm_low.vsc = radeon_crtc->vsc; 2233 wm_low.vtaps = 1; 2234 if (radeon_crtc->rmx_type != RMX_OFF) 2235 wm_low.vtaps = 2; 2236 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2237 wm_low.lb_size = lb_size; 2238 wm_low.dram_channels = dram_channels; 2239 wm_low.num_heads = num_heads; 2240 2241 /* set for high clocks */ 2242 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535); 2243 /* set for low clocks */ 2244 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535); 2245 2246 /* possibly force display priority to high */ 2247 /* should really do this at mode validation time... */ 2248 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 2249 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) || 2250 !evergreen_check_latency_hiding(&wm_high) || 2251 (rdev->disp_priority == 2)) { 2252 DRM_DEBUG_KMS("force priority a to high\n"); 2253 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2254 } 2255 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 2256 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) || 2257 !evergreen_check_latency_hiding(&wm_low) || 2258 (rdev->disp_priority == 2)) { 2259 DRM_DEBUG_KMS("force priority b to high\n"); 2260 priority_b_cnt |= PRIORITY_ALWAYS_ON; 2261 } 2262 2263 a.full = dfixed_const(1000); 2264 b.full = dfixed_const(mode->clock); 2265 b.full = dfixed_div(b, a); 2266 c.full = dfixed_const(latency_watermark_a); 2267 c.full = dfixed_mul(c, b); 2268 c.full = dfixed_mul(c, radeon_crtc->hsc); 2269 c.full = dfixed_div(c, a); 2270 a.full = dfixed_const(16); 2271 c.full = dfixed_div(c, a); 2272 priority_a_mark = dfixed_trunc(c); 2273 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK; 2274 2275 a.full = dfixed_const(1000); 2276 b.full = dfixed_const(mode->clock); 2277 b.full = dfixed_div(b, a); 2278 c.full = dfixed_const(latency_watermark_b); 2279 c.full = dfixed_mul(c, b); 2280 c.full = dfixed_mul(c, radeon_crtc->hsc); 2281 c.full = dfixed_div(c, a); 2282 a.full = dfixed_const(16); 2283 c.full = dfixed_div(c, a); 2284 priority_b_mark = dfixed_trunc(c); 2285 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; 2286 } 2287 2288 /* select wm A */ 2289 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 2290 tmp = arb_control3; 2291 tmp &= ~LATENCY_WATERMARK_MASK(3); 2292 tmp |= LATENCY_WATERMARK_MASK(1); 2293 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 2294 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 2295 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 2296 LATENCY_HIGH_WATERMARK(line_time))); 2297 /* select wm B */ 2298 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 2299 tmp &= ~LATENCY_WATERMARK_MASK(3); 2300 tmp |= LATENCY_WATERMARK_MASK(2); 2301 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 2302 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 2303 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 2304 LATENCY_HIGH_WATERMARK(line_time))); 2305 /* restore original selection */ 2306 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3); 2307 2308 /* write the priority marks */ 2309 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); 2310 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); 2311 2312 /* save values for DPM */ 2313 radeon_crtc->line_time = line_time; 2314 radeon_crtc->wm_high = latency_watermark_a; 2315 radeon_crtc->wm_low = latency_watermark_b; 2316 } 2317 2318 /** 2319 * evergreen_bandwidth_update - update display watermarks callback. 2320 * 2321 * @rdev: radeon_device pointer 2322 * 2323 * Update the display watermarks based on the requested mode(s) 2324 * (evergreen+). 2325 */ 2326 void evergreen_bandwidth_update(struct radeon_device *rdev) 2327 { 2328 struct drm_display_mode *mode0 = NULL; 2329 struct drm_display_mode *mode1 = NULL; 2330 u32 num_heads = 0, lb_size; 2331 int i; 2332 2333 radeon_update_display_priority(rdev); 2334 2335 for (i = 0; i < rdev->num_crtc; i++) { 2336 if (rdev->mode_info.crtcs[i]->base.enabled) 2337 num_heads++; 2338 } 2339 for (i = 0; i < rdev->num_crtc; i += 2) { 2340 mode0 = &rdev->mode_info.crtcs[i]->base.mode; 2341 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode; 2342 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1); 2343 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 2344 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0); 2345 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads); 2346 } 2347 } 2348 2349 /** 2350 * evergreen_mc_wait_for_idle - wait for MC idle callback. 2351 * 2352 * @rdev: radeon_device pointer 2353 * 2354 * Wait for the MC (memory controller) to be idle. 2355 * (evergreen+). 2356 * Returns 0 if the MC is idle, -1 if not. 2357 */ 2358 int evergreen_mc_wait_for_idle(struct radeon_device *rdev) 2359 { 2360 unsigned i; 2361 u32 tmp; 2362 2363 for (i = 0; i < rdev->usec_timeout; i++) { 2364 /* read MC_STATUS */ 2365 tmp = RREG32(SRBM_STATUS) & 0x1F00; 2366 if (!tmp) 2367 return 0; 2368 udelay(1); 2369 } 2370 return -1; 2371 } 2372 2373 /* 2374 * GART 2375 */ 2376 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev) 2377 { 2378 unsigned i; 2379 u32 tmp; 2380 2381 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 2382 2383 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1)); 2384 for (i = 0; i < rdev->usec_timeout; i++) { 2385 /* read MC_STATUS */ 2386 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE); 2387 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT; 2388 if (tmp == 2) { 2389 printk(KERN_WARNING "[drm] r600 flush TLB failed\n"); 2390 return; 2391 } 2392 if (tmp) { 2393 return; 2394 } 2395 udelay(1); 2396 } 2397 } 2398 2399 static int evergreen_pcie_gart_enable(struct radeon_device *rdev) 2400 { 2401 u32 tmp; 2402 int r; 2403 2404 if (rdev->gart.robj == NULL) { 2405 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 2406 return -EINVAL; 2407 } 2408 r = radeon_gart_table_vram_pin(rdev); 2409 if (r) 2410 return r; 2411 /* Setup L2 cache */ 2412 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 2413 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 2414 EFFECTIVE_L2_QUEUE_SIZE(7)); 2415 WREG32(VM_L2_CNTL2, 0); 2416 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2417 /* Setup TLB control */ 2418 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 2419 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 2420 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 2421 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2422 if (rdev->flags & RADEON_IS_IGP) { 2423 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp); 2424 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp); 2425 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp); 2426 } else { 2427 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2428 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2429 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2430 if ((rdev->family == CHIP_JUNIPER) || 2431 (rdev->family == CHIP_CYPRESS) || 2432 (rdev->family == CHIP_HEMLOCK) || 2433 (rdev->family == CHIP_BARTS)) 2434 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp); 2435 } 2436 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2437 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2438 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2439 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2440 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 2441 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 2442 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 2443 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 2444 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT); 2445 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 2446 (u32)(rdev->dummy_page.addr >> 12)); 2447 WREG32(VM_CONTEXT1_CNTL, 0); 2448 2449 evergreen_pcie_gart_tlb_flush(rdev); 2450 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 2451 (unsigned)(rdev->mc.gtt_size >> 20), 2452 (unsigned long long)rdev->gart.table_addr); 2453 rdev->gart.ready = true; 2454 return 0; 2455 } 2456 2457 static void evergreen_pcie_gart_disable(struct radeon_device *rdev) 2458 { 2459 u32 tmp; 2460 2461 /* Disable all tables */ 2462 WREG32(VM_CONTEXT0_CNTL, 0); 2463 WREG32(VM_CONTEXT1_CNTL, 0); 2464 2465 /* Setup L2 cache */ 2466 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING | 2467 EFFECTIVE_L2_QUEUE_SIZE(7)); 2468 WREG32(VM_L2_CNTL2, 0); 2469 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2470 /* Setup TLB control */ 2471 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2472 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2473 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2474 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2475 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2476 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2477 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2478 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2479 radeon_gart_table_vram_unpin(rdev); 2480 } 2481 2482 static void evergreen_pcie_gart_fini(struct radeon_device *rdev) 2483 { 2484 evergreen_pcie_gart_disable(rdev); 2485 radeon_gart_table_vram_free(rdev); 2486 radeon_gart_fini(rdev); 2487 } 2488 2489 2490 static void evergreen_agp_enable(struct radeon_device *rdev) 2491 { 2492 u32 tmp; 2493 2494 /* Setup L2 cache */ 2495 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 2496 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 2497 EFFECTIVE_L2_QUEUE_SIZE(7)); 2498 WREG32(VM_L2_CNTL2, 0); 2499 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2500 /* Setup TLB control */ 2501 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 2502 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 2503 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 2504 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2505 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2506 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2507 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2508 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2509 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2510 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2511 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2512 WREG32(VM_CONTEXT0_CNTL, 0); 2513 WREG32(VM_CONTEXT1_CNTL, 0); 2514 } 2515 2516 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save) 2517 { 2518 u32 crtc_enabled, tmp, frame_count, blackout; 2519 int i, j; 2520 2521 if (!ASIC_IS_NODCE(rdev)) { 2522 save->vga_render_control = RREG32(VGA_RENDER_CONTROL); 2523 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL); 2524 2525 /* disable VGA render */ 2526 WREG32(VGA_RENDER_CONTROL, 0); 2527 } 2528 /* blank the display controllers */ 2529 for (i = 0; i < rdev->num_crtc; i++) { 2530 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN; 2531 if (crtc_enabled) { 2532 save->crtc_enabled[i] = true; 2533 if (ASIC_IS_DCE6(rdev)) { 2534 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); 2535 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) { 2536 radeon_wait_for_vblank(rdev, i); 2537 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2538 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN; 2539 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); 2540 } 2541 } else { 2542 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2543 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) { 2544 radeon_wait_for_vblank(rdev, i); 2545 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2546 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 2547 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2548 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2549 } 2550 } 2551 /* wait for the next frame */ 2552 frame_count = radeon_get_vblank_counter(rdev, i); 2553 for (j = 0; j < rdev->usec_timeout; j++) { 2554 if (radeon_get_vblank_counter(rdev, i) != frame_count) 2555 break; 2556 udelay(1); 2557 } 2558 2559 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */ 2560 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2561 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2562 tmp &= ~EVERGREEN_CRTC_MASTER_EN; 2563 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2564 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2565 save->crtc_enabled[i] = false; 2566 /* ***** */ 2567 } else { 2568 save->crtc_enabled[i] = false; 2569 } 2570 } 2571 2572 radeon_mc_wait_for_idle(rdev); 2573 2574 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 2575 if ((blackout & BLACKOUT_MODE_MASK) != 1) { 2576 /* Block CPU access */ 2577 WREG32(BIF_FB_EN, 0); 2578 /* blackout the MC */ 2579 blackout &= ~BLACKOUT_MODE_MASK; 2580 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 2581 } 2582 /* wait for the MC to settle */ 2583 udelay(100); 2584 2585 /* lock double buffered regs */ 2586 for (i = 0; i < rdev->num_crtc; i++) { 2587 if (save->crtc_enabled[i]) { 2588 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2589 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) { 2590 tmp |= EVERGREEN_GRPH_UPDATE_LOCK; 2591 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp); 2592 } 2593 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]); 2594 if (!(tmp & 1)) { 2595 tmp |= 1; 2596 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp); 2597 } 2598 } 2599 } 2600 } 2601 2602 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save) 2603 { 2604 u32 tmp, frame_count; 2605 int i, j; 2606 2607 /* update crtc base addresses */ 2608 for (i = 0; i < rdev->num_crtc; i++) { 2609 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], 2610 upper_32_bits(rdev->mc.vram_start)); 2611 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], 2612 upper_32_bits(rdev->mc.vram_start)); 2613 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i], 2614 (u32)rdev->mc.vram_start); 2615 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i], 2616 (u32)rdev->mc.vram_start); 2617 } 2618 2619 if (!ASIC_IS_NODCE(rdev)) { 2620 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start)); 2621 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start); 2622 } 2623 2624 /* unlock regs and wait for update */ 2625 for (i = 0; i < rdev->num_crtc; i++) { 2626 if (save->crtc_enabled[i]) { 2627 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]); 2628 if ((tmp & 0x7) != 3) { 2629 tmp &= ~0x7; 2630 tmp |= 0x3; 2631 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp); 2632 } 2633 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2634 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) { 2635 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK; 2636 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp); 2637 } 2638 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]); 2639 if (tmp & 1) { 2640 tmp &= ~1; 2641 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp); 2642 } 2643 for (j = 0; j < rdev->usec_timeout; j++) { 2644 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2645 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0) 2646 break; 2647 udelay(1); 2648 } 2649 } 2650 } 2651 2652 /* unblackout the MC */ 2653 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL); 2654 tmp &= ~BLACKOUT_MODE_MASK; 2655 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp); 2656 /* allow CPU access */ 2657 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 2658 2659 for (i = 0; i < rdev->num_crtc; i++) { 2660 if (save->crtc_enabled[i]) { 2661 if (ASIC_IS_DCE6(rdev)) { 2662 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); 2663 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN; 2664 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2665 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); 2666 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2667 } else { 2668 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2669 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 2670 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2671 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2672 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2673 } 2674 /* wait for the next frame */ 2675 frame_count = radeon_get_vblank_counter(rdev, i); 2676 for (j = 0; j < rdev->usec_timeout; j++) { 2677 if (radeon_get_vblank_counter(rdev, i) != frame_count) 2678 break; 2679 udelay(1); 2680 } 2681 } 2682 } 2683 if (!ASIC_IS_NODCE(rdev)) { 2684 /* Unlock vga access */ 2685 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control); 2686 mdelay(1); 2687 WREG32(VGA_RENDER_CONTROL, save->vga_render_control); 2688 } 2689 } 2690 2691 void evergreen_mc_program(struct radeon_device *rdev) 2692 { 2693 struct evergreen_mc_save save; 2694 u32 tmp; 2695 int i, j; 2696 2697 /* Initialize HDP */ 2698 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 2699 WREG32((0x2c14 + j), 0x00000000); 2700 WREG32((0x2c18 + j), 0x00000000); 2701 WREG32((0x2c1c + j), 0x00000000); 2702 WREG32((0x2c20 + j), 0x00000000); 2703 WREG32((0x2c24 + j), 0x00000000); 2704 } 2705 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 2706 2707 evergreen_mc_stop(rdev, &save); 2708 if (evergreen_mc_wait_for_idle(rdev)) { 2709 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2710 } 2711 /* Lockout access through VGA aperture*/ 2712 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 2713 /* Update configuration */ 2714 if (rdev->flags & RADEON_IS_AGP) { 2715 if (rdev->mc.vram_start < rdev->mc.gtt_start) { 2716 /* VRAM before AGP */ 2717 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2718 rdev->mc.vram_start >> 12); 2719 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2720 rdev->mc.gtt_end >> 12); 2721 } else { 2722 /* VRAM after AGP */ 2723 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2724 rdev->mc.gtt_start >> 12); 2725 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2726 rdev->mc.vram_end >> 12); 2727 } 2728 } else { 2729 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2730 rdev->mc.vram_start >> 12); 2731 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2732 rdev->mc.vram_end >> 12); 2733 } 2734 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12); 2735 /* llano/ontario only */ 2736 if ((rdev->family == CHIP_PALM) || 2737 (rdev->family == CHIP_SUMO) || 2738 (rdev->family == CHIP_SUMO2)) { 2739 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF; 2740 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24; 2741 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20; 2742 WREG32(MC_FUS_VM_FB_OFFSET, tmp); 2743 } 2744 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 2745 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 2746 WREG32(MC_VM_FB_LOCATION, tmp); 2747 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 2748 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 2749 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 2750 if (rdev->flags & RADEON_IS_AGP) { 2751 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16); 2752 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16); 2753 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22); 2754 } else { 2755 WREG32(MC_VM_AGP_BASE, 0); 2756 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 2757 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 2758 } 2759 if (evergreen_mc_wait_for_idle(rdev)) { 2760 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2761 } 2762 evergreen_mc_resume(rdev, &save); 2763 /* we need to own VRAM, so turn off the VGA renderer here 2764 * to stop it overwriting our objects */ 2765 rv515_vga_render_disable(rdev); 2766 } 2767 2768 /* 2769 * CP. 2770 */ 2771 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 2772 { 2773 struct radeon_ring *ring = &rdev->ring[ib->ring]; 2774 u32 next_rptr; 2775 2776 /* set to DX10/11 mode */ 2777 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0)); 2778 radeon_ring_write(ring, 1); 2779 2780 if (ring->rptr_save_reg) { 2781 next_rptr = ring->wptr + 3 + 4; 2782 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 2783 radeon_ring_write(ring, ((ring->rptr_save_reg - 2784 PACKET3_SET_CONFIG_REG_START) >> 2)); 2785 radeon_ring_write(ring, next_rptr); 2786 } else if (rdev->wb.enabled) { 2787 next_rptr = ring->wptr + 5 + 4; 2788 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3)); 2789 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 2790 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18)); 2791 radeon_ring_write(ring, next_rptr); 2792 radeon_ring_write(ring, 0); 2793 } 2794 2795 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 2796 radeon_ring_write(ring, 2797 #ifdef __BIG_ENDIAN 2798 (2 << 0) | 2799 #endif 2800 (ib->gpu_addr & 0xFFFFFFFC)); 2801 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF); 2802 radeon_ring_write(ring, ib->length_dw); 2803 } 2804 2805 2806 static int evergreen_cp_load_microcode(struct radeon_device *rdev) 2807 { 2808 const __be32 *fw_data; 2809 int i; 2810 2811 if (!rdev->me_fw || !rdev->pfp_fw) 2812 return -EINVAL; 2813 2814 r700_cp_stop(rdev); 2815 WREG32(CP_RB_CNTL, 2816 #ifdef __BIG_ENDIAN 2817 BUF_SWAP_32BIT | 2818 #endif 2819 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3)); 2820 2821 fw_data = (const __be32 *)rdev->pfp_fw->data; 2822 WREG32(CP_PFP_UCODE_ADDR, 0); 2823 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++) 2824 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 2825 WREG32(CP_PFP_UCODE_ADDR, 0); 2826 2827 fw_data = (const __be32 *)rdev->me_fw->data; 2828 WREG32(CP_ME_RAM_WADDR, 0); 2829 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++) 2830 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 2831 2832 WREG32(CP_PFP_UCODE_ADDR, 0); 2833 WREG32(CP_ME_RAM_WADDR, 0); 2834 WREG32(CP_ME_RAM_RADDR, 0); 2835 return 0; 2836 } 2837 2838 static int evergreen_cp_start(struct radeon_device *rdev) 2839 { 2840 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 2841 int r, i; 2842 uint32_t cp_me; 2843 2844 r = radeon_ring_lock(rdev, ring, 7); 2845 if (r) { 2846 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 2847 return r; 2848 } 2849 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5)); 2850 radeon_ring_write(ring, 0x1); 2851 radeon_ring_write(ring, 0x0); 2852 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1); 2853 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 2854 radeon_ring_write(ring, 0); 2855 radeon_ring_write(ring, 0); 2856 radeon_ring_unlock_commit(rdev, ring, false); 2857 2858 cp_me = 0xff; 2859 WREG32(CP_ME_CNTL, cp_me); 2860 2861 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19); 2862 if (r) { 2863 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 2864 return r; 2865 } 2866 2867 /* setup clear context state */ 2868 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 2869 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 2870 2871 for (i = 0; i < evergreen_default_size; i++) 2872 radeon_ring_write(ring, evergreen_default_state[i]); 2873 2874 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 2875 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 2876 2877 /* set clear context state */ 2878 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 2879 radeon_ring_write(ring, 0); 2880 2881 /* SQ_VTX_BASE_VTX_LOC */ 2882 radeon_ring_write(ring, 0xc0026f00); 2883 radeon_ring_write(ring, 0x00000000); 2884 radeon_ring_write(ring, 0x00000000); 2885 radeon_ring_write(ring, 0x00000000); 2886 2887 /* Clear consts */ 2888 radeon_ring_write(ring, 0xc0036f00); 2889 radeon_ring_write(ring, 0x00000bc4); 2890 radeon_ring_write(ring, 0xffffffff); 2891 radeon_ring_write(ring, 0xffffffff); 2892 radeon_ring_write(ring, 0xffffffff); 2893 2894 radeon_ring_write(ring, 0xc0026900); 2895 radeon_ring_write(ring, 0x00000316); 2896 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 2897 radeon_ring_write(ring, 0x00000010); /* */ 2898 2899 radeon_ring_unlock_commit(rdev, ring, false); 2900 2901 return 0; 2902 } 2903 2904 static int evergreen_cp_resume(struct radeon_device *rdev) 2905 { 2906 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 2907 u32 tmp; 2908 u32 rb_bufsz; 2909 int r; 2910 2911 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */ 2912 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP | 2913 SOFT_RESET_PA | 2914 SOFT_RESET_SH | 2915 SOFT_RESET_VGT | 2916 SOFT_RESET_SPI | 2917 SOFT_RESET_SX)); 2918 RREG32(GRBM_SOFT_RESET); 2919 mdelay(15); 2920 WREG32(GRBM_SOFT_RESET, 0); 2921 RREG32(GRBM_SOFT_RESET); 2922 2923 /* Set ring buffer size */ 2924 rb_bufsz = order_base_2(ring->ring_size / 8); 2925 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 2926 #ifdef __BIG_ENDIAN 2927 tmp |= BUF_SWAP_32BIT; 2928 #endif 2929 WREG32(CP_RB_CNTL, tmp); 2930 WREG32(CP_SEM_WAIT_TIMER, 0x0); 2931 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 2932 2933 /* Set the write pointer delay */ 2934 WREG32(CP_RB_WPTR_DELAY, 0); 2935 2936 /* Initialize the ring buffer's read and write pointers */ 2937 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA); 2938 WREG32(CP_RB_RPTR_WR, 0); 2939 ring->wptr = 0; 2940 WREG32(CP_RB_WPTR, ring->wptr); 2941 2942 /* set the wb address whether it's enabled or not */ 2943 WREG32(CP_RB_RPTR_ADDR, 2944 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC)); 2945 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 2946 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 2947 2948 if (rdev->wb.enabled) 2949 WREG32(SCRATCH_UMSK, 0xff); 2950 else { 2951 tmp |= RB_NO_UPDATE; 2952 WREG32(SCRATCH_UMSK, 0); 2953 } 2954 2955 mdelay(1); 2956 WREG32(CP_RB_CNTL, tmp); 2957 2958 WREG32(CP_RB_BASE, ring->gpu_addr >> 8); 2959 WREG32(CP_DEBUG, (1 << 27) | (1 << 28)); 2960 2961 evergreen_cp_start(rdev); 2962 ring->ready = true; 2963 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring); 2964 if (r) { 2965 ring->ready = false; 2966 return r; 2967 } 2968 return 0; 2969 } 2970 2971 /* 2972 * Core functions 2973 */ 2974 static void evergreen_gpu_init(struct radeon_device *rdev) 2975 { 2976 u32 gb_addr_config; 2977 u32 mc_shared_chmap, mc_arb_ramcfg; 2978 u32 sx_debug_1; 2979 u32 smx_dc_ctl0; 2980 u32 sq_config; 2981 u32 sq_lds_resource_mgmt; 2982 u32 sq_gpr_resource_mgmt_1; 2983 u32 sq_gpr_resource_mgmt_2; 2984 u32 sq_gpr_resource_mgmt_3; 2985 u32 sq_thread_resource_mgmt; 2986 u32 sq_thread_resource_mgmt_2; 2987 u32 sq_stack_resource_mgmt_1; 2988 u32 sq_stack_resource_mgmt_2; 2989 u32 sq_stack_resource_mgmt_3; 2990 u32 vgt_cache_invalidation; 2991 u32 hdp_host_path_cntl, tmp; 2992 u32 disabled_rb_mask; 2993 int i, j, num_shader_engines, ps_thread_count; 2994 2995 switch (rdev->family) { 2996 case CHIP_CYPRESS: 2997 case CHIP_HEMLOCK: 2998 rdev->config.evergreen.num_ses = 2; 2999 rdev->config.evergreen.max_pipes = 4; 3000 rdev->config.evergreen.max_tile_pipes = 8; 3001 rdev->config.evergreen.max_simds = 10; 3002 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3003 rdev->config.evergreen.max_gprs = 256; 3004 rdev->config.evergreen.max_threads = 248; 3005 rdev->config.evergreen.max_gs_threads = 32; 3006 rdev->config.evergreen.max_stack_entries = 512; 3007 rdev->config.evergreen.sx_num_of_sets = 4; 3008 rdev->config.evergreen.sx_max_export_size = 256; 3009 rdev->config.evergreen.sx_max_export_pos_size = 64; 3010 rdev->config.evergreen.sx_max_export_smx_size = 192; 3011 rdev->config.evergreen.max_hw_contexts = 8; 3012 rdev->config.evergreen.sq_num_cf_insts = 2; 3013 3014 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3015 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3016 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3017 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN; 3018 break; 3019 case CHIP_JUNIPER: 3020 rdev->config.evergreen.num_ses = 1; 3021 rdev->config.evergreen.max_pipes = 4; 3022 rdev->config.evergreen.max_tile_pipes = 4; 3023 rdev->config.evergreen.max_simds = 10; 3024 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3025 rdev->config.evergreen.max_gprs = 256; 3026 rdev->config.evergreen.max_threads = 248; 3027 rdev->config.evergreen.max_gs_threads = 32; 3028 rdev->config.evergreen.max_stack_entries = 512; 3029 rdev->config.evergreen.sx_num_of_sets = 4; 3030 rdev->config.evergreen.sx_max_export_size = 256; 3031 rdev->config.evergreen.sx_max_export_pos_size = 64; 3032 rdev->config.evergreen.sx_max_export_smx_size = 192; 3033 rdev->config.evergreen.max_hw_contexts = 8; 3034 rdev->config.evergreen.sq_num_cf_insts = 2; 3035 3036 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3037 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3038 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3039 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN; 3040 break; 3041 case CHIP_REDWOOD: 3042 rdev->config.evergreen.num_ses = 1; 3043 rdev->config.evergreen.max_pipes = 4; 3044 rdev->config.evergreen.max_tile_pipes = 4; 3045 rdev->config.evergreen.max_simds = 5; 3046 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3047 rdev->config.evergreen.max_gprs = 256; 3048 rdev->config.evergreen.max_threads = 248; 3049 rdev->config.evergreen.max_gs_threads = 32; 3050 rdev->config.evergreen.max_stack_entries = 256; 3051 rdev->config.evergreen.sx_num_of_sets = 4; 3052 rdev->config.evergreen.sx_max_export_size = 256; 3053 rdev->config.evergreen.sx_max_export_pos_size = 64; 3054 rdev->config.evergreen.sx_max_export_smx_size = 192; 3055 rdev->config.evergreen.max_hw_contexts = 8; 3056 rdev->config.evergreen.sq_num_cf_insts = 2; 3057 3058 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3059 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3060 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3061 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN; 3062 break; 3063 case CHIP_CEDAR: 3064 default: 3065 rdev->config.evergreen.num_ses = 1; 3066 rdev->config.evergreen.max_pipes = 2; 3067 rdev->config.evergreen.max_tile_pipes = 2; 3068 rdev->config.evergreen.max_simds = 2; 3069 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3070 rdev->config.evergreen.max_gprs = 256; 3071 rdev->config.evergreen.max_threads = 192; 3072 rdev->config.evergreen.max_gs_threads = 16; 3073 rdev->config.evergreen.max_stack_entries = 256; 3074 rdev->config.evergreen.sx_num_of_sets = 4; 3075 rdev->config.evergreen.sx_max_export_size = 128; 3076 rdev->config.evergreen.sx_max_export_pos_size = 32; 3077 rdev->config.evergreen.sx_max_export_smx_size = 96; 3078 rdev->config.evergreen.max_hw_contexts = 4; 3079 rdev->config.evergreen.sq_num_cf_insts = 1; 3080 3081 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3082 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3083 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3084 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN; 3085 break; 3086 case CHIP_PALM: 3087 rdev->config.evergreen.num_ses = 1; 3088 rdev->config.evergreen.max_pipes = 2; 3089 rdev->config.evergreen.max_tile_pipes = 2; 3090 rdev->config.evergreen.max_simds = 2; 3091 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3092 rdev->config.evergreen.max_gprs = 256; 3093 rdev->config.evergreen.max_threads = 192; 3094 rdev->config.evergreen.max_gs_threads = 16; 3095 rdev->config.evergreen.max_stack_entries = 256; 3096 rdev->config.evergreen.sx_num_of_sets = 4; 3097 rdev->config.evergreen.sx_max_export_size = 128; 3098 rdev->config.evergreen.sx_max_export_pos_size = 32; 3099 rdev->config.evergreen.sx_max_export_smx_size = 96; 3100 rdev->config.evergreen.max_hw_contexts = 4; 3101 rdev->config.evergreen.sq_num_cf_insts = 1; 3102 3103 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3104 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3105 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3106 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN; 3107 break; 3108 case CHIP_SUMO: 3109 rdev->config.evergreen.num_ses = 1; 3110 rdev->config.evergreen.max_pipes = 4; 3111 rdev->config.evergreen.max_tile_pipes = 4; 3112 if (rdev->pdev->device == 0x9648) 3113 rdev->config.evergreen.max_simds = 3; 3114 else if ((rdev->pdev->device == 0x9647) || 3115 (rdev->pdev->device == 0x964a)) 3116 rdev->config.evergreen.max_simds = 4; 3117 else 3118 rdev->config.evergreen.max_simds = 5; 3119 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3120 rdev->config.evergreen.max_gprs = 256; 3121 rdev->config.evergreen.max_threads = 248; 3122 rdev->config.evergreen.max_gs_threads = 32; 3123 rdev->config.evergreen.max_stack_entries = 256; 3124 rdev->config.evergreen.sx_num_of_sets = 4; 3125 rdev->config.evergreen.sx_max_export_size = 256; 3126 rdev->config.evergreen.sx_max_export_pos_size = 64; 3127 rdev->config.evergreen.sx_max_export_smx_size = 192; 3128 rdev->config.evergreen.max_hw_contexts = 8; 3129 rdev->config.evergreen.sq_num_cf_insts = 2; 3130 3131 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3132 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3133 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3134 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN; 3135 break; 3136 case CHIP_SUMO2: 3137 rdev->config.evergreen.num_ses = 1; 3138 rdev->config.evergreen.max_pipes = 4; 3139 rdev->config.evergreen.max_tile_pipes = 4; 3140 rdev->config.evergreen.max_simds = 2; 3141 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3142 rdev->config.evergreen.max_gprs = 256; 3143 rdev->config.evergreen.max_threads = 248; 3144 rdev->config.evergreen.max_gs_threads = 32; 3145 rdev->config.evergreen.max_stack_entries = 512; 3146 rdev->config.evergreen.sx_num_of_sets = 4; 3147 rdev->config.evergreen.sx_max_export_size = 256; 3148 rdev->config.evergreen.sx_max_export_pos_size = 64; 3149 rdev->config.evergreen.sx_max_export_smx_size = 192; 3150 rdev->config.evergreen.max_hw_contexts = 4; 3151 rdev->config.evergreen.sq_num_cf_insts = 2; 3152 3153 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3154 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3155 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3156 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN; 3157 break; 3158 case CHIP_BARTS: 3159 rdev->config.evergreen.num_ses = 2; 3160 rdev->config.evergreen.max_pipes = 4; 3161 rdev->config.evergreen.max_tile_pipes = 8; 3162 rdev->config.evergreen.max_simds = 7; 3163 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3164 rdev->config.evergreen.max_gprs = 256; 3165 rdev->config.evergreen.max_threads = 248; 3166 rdev->config.evergreen.max_gs_threads = 32; 3167 rdev->config.evergreen.max_stack_entries = 512; 3168 rdev->config.evergreen.sx_num_of_sets = 4; 3169 rdev->config.evergreen.sx_max_export_size = 256; 3170 rdev->config.evergreen.sx_max_export_pos_size = 64; 3171 rdev->config.evergreen.sx_max_export_smx_size = 192; 3172 rdev->config.evergreen.max_hw_contexts = 8; 3173 rdev->config.evergreen.sq_num_cf_insts = 2; 3174 3175 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3176 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3177 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3178 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN; 3179 break; 3180 case CHIP_TURKS: 3181 rdev->config.evergreen.num_ses = 1; 3182 rdev->config.evergreen.max_pipes = 4; 3183 rdev->config.evergreen.max_tile_pipes = 4; 3184 rdev->config.evergreen.max_simds = 6; 3185 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3186 rdev->config.evergreen.max_gprs = 256; 3187 rdev->config.evergreen.max_threads = 248; 3188 rdev->config.evergreen.max_gs_threads = 32; 3189 rdev->config.evergreen.max_stack_entries = 256; 3190 rdev->config.evergreen.sx_num_of_sets = 4; 3191 rdev->config.evergreen.sx_max_export_size = 256; 3192 rdev->config.evergreen.sx_max_export_pos_size = 64; 3193 rdev->config.evergreen.sx_max_export_smx_size = 192; 3194 rdev->config.evergreen.max_hw_contexts = 8; 3195 rdev->config.evergreen.sq_num_cf_insts = 2; 3196 3197 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3198 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3199 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3200 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN; 3201 break; 3202 case CHIP_CAICOS: 3203 rdev->config.evergreen.num_ses = 1; 3204 rdev->config.evergreen.max_pipes = 2; 3205 rdev->config.evergreen.max_tile_pipes = 2; 3206 rdev->config.evergreen.max_simds = 2; 3207 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3208 rdev->config.evergreen.max_gprs = 256; 3209 rdev->config.evergreen.max_threads = 192; 3210 rdev->config.evergreen.max_gs_threads = 16; 3211 rdev->config.evergreen.max_stack_entries = 256; 3212 rdev->config.evergreen.sx_num_of_sets = 4; 3213 rdev->config.evergreen.sx_max_export_size = 128; 3214 rdev->config.evergreen.sx_max_export_pos_size = 32; 3215 rdev->config.evergreen.sx_max_export_smx_size = 96; 3216 rdev->config.evergreen.max_hw_contexts = 4; 3217 rdev->config.evergreen.sq_num_cf_insts = 1; 3218 3219 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3220 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3221 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3222 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN; 3223 break; 3224 } 3225 3226 /* Initialize HDP */ 3227 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3228 WREG32((0x2c14 + j), 0x00000000); 3229 WREG32((0x2c18 + j), 0x00000000); 3230 WREG32((0x2c1c + j), 0x00000000); 3231 WREG32((0x2c20 + j), 0x00000000); 3232 WREG32((0x2c24 + j), 0x00000000); 3233 } 3234 3235 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 3236 3237 evergreen_fix_pci_max_read_req_size(rdev); 3238 3239 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 3240 if ((rdev->family == CHIP_PALM) || 3241 (rdev->family == CHIP_SUMO) || 3242 (rdev->family == CHIP_SUMO2)) 3243 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG); 3244 else 3245 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3246 3247 /* setup tiling info dword. gb_addr_config is not adequate since it does 3248 * not have bank info, so create a custom tiling dword. 3249 * bits 3:0 num_pipes 3250 * bits 7:4 num_banks 3251 * bits 11:8 group_size 3252 * bits 15:12 row_size 3253 */ 3254 rdev->config.evergreen.tile_config = 0; 3255 switch (rdev->config.evergreen.max_tile_pipes) { 3256 case 1: 3257 default: 3258 rdev->config.evergreen.tile_config |= (0 << 0); 3259 break; 3260 case 2: 3261 rdev->config.evergreen.tile_config |= (1 << 0); 3262 break; 3263 case 4: 3264 rdev->config.evergreen.tile_config |= (2 << 0); 3265 break; 3266 case 8: 3267 rdev->config.evergreen.tile_config |= (3 << 0); 3268 break; 3269 } 3270 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */ 3271 if (rdev->flags & RADEON_IS_IGP) 3272 rdev->config.evergreen.tile_config |= 1 << 4; 3273 else { 3274 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) { 3275 case 0: /* four banks */ 3276 rdev->config.evergreen.tile_config |= 0 << 4; 3277 break; 3278 case 1: /* eight banks */ 3279 rdev->config.evergreen.tile_config |= 1 << 4; 3280 break; 3281 case 2: /* sixteen banks */ 3282 default: 3283 rdev->config.evergreen.tile_config |= 2 << 4; 3284 break; 3285 } 3286 } 3287 rdev->config.evergreen.tile_config |= 0 << 8; 3288 rdev->config.evergreen.tile_config |= 3289 ((gb_addr_config & 0x30000000) >> 28) << 12; 3290 3291 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1; 3292 3293 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) { 3294 u32 efuse_straps_4; 3295 u32 efuse_straps_3; 3296 3297 efuse_straps_4 = RREG32_RCU(0x204); 3298 efuse_straps_3 = RREG32_RCU(0x203); 3299 tmp = (((efuse_straps_4 & 0xf) << 4) | 3300 ((efuse_straps_3 & 0xf0000000) >> 28)); 3301 } else { 3302 tmp = 0; 3303 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) { 3304 u32 rb_disable_bitmap; 3305 3306 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3307 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3308 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16; 3309 tmp <<= 4; 3310 tmp |= rb_disable_bitmap; 3311 } 3312 } 3313 /* enabled rb are just the one not disabled :) */ 3314 disabled_rb_mask = tmp; 3315 tmp = 0; 3316 for (i = 0; i < rdev->config.evergreen.max_backends; i++) 3317 tmp |= (1 << i); 3318 /* if all the backends are disabled, fix it up here */ 3319 if ((disabled_rb_mask & tmp) == tmp) { 3320 for (i = 0; i < rdev->config.evergreen.max_backends; i++) 3321 disabled_rb_mask &= ~(1 << i); 3322 } 3323 3324 for (i = 0; i < rdev->config.evergreen.num_ses; i++) { 3325 u32 simd_disable_bitmap; 3326 3327 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3328 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3329 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; 3330 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds; 3331 tmp <<= 16; 3332 tmp |= simd_disable_bitmap; 3333 } 3334 rdev->config.evergreen.active_simds = hweight32(~tmp); 3335 3336 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 3337 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 3338 3339 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3340 WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 3341 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3342 WREG32(DMA_TILING_CONFIG, gb_addr_config); 3343 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3344 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3345 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3346 3347 if ((rdev->config.evergreen.max_backends == 1) && 3348 (rdev->flags & RADEON_IS_IGP)) { 3349 if ((disabled_rb_mask & 3) == 1) { 3350 /* RB0 disabled, RB1 enabled */ 3351 tmp = 0x11111111; 3352 } else { 3353 /* RB1 disabled, RB0 enabled */ 3354 tmp = 0x00000000; 3355 } 3356 } else { 3357 tmp = gb_addr_config & NUM_PIPES_MASK; 3358 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends, 3359 EVERGREEN_MAX_BACKENDS, disabled_rb_mask); 3360 } 3361 WREG32(GB_BACKEND_MAP, tmp); 3362 3363 WREG32(CGTS_SYS_TCC_DISABLE, 0); 3364 WREG32(CGTS_TCC_DISABLE, 0); 3365 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0); 3366 WREG32(CGTS_USER_TCC_DISABLE, 0); 3367 3368 /* set HW defaults for 3D engine */ 3369 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | 3370 ROQ_IB2_START(0x2b))); 3371 3372 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30)); 3373 3374 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | 3375 SYNC_GRADIENT | 3376 SYNC_WALKER | 3377 SYNC_ALIGNER)); 3378 3379 sx_debug_1 = RREG32(SX_DEBUG_1); 3380 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS; 3381 WREG32(SX_DEBUG_1, sx_debug_1); 3382 3383 3384 smx_dc_ctl0 = RREG32(SMX_DC_CTL0); 3385 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff); 3386 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets); 3387 WREG32(SMX_DC_CTL0, smx_dc_ctl0); 3388 3389 if (rdev->family <= CHIP_SUMO2) 3390 WREG32(SMX_SAR_CTL0, 0x00010000); 3391 3392 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) | 3393 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) | 3394 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1))); 3395 3396 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) | 3397 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) | 3398 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size))); 3399 3400 WREG32(VGT_NUM_INSTANCES, 1); 3401 WREG32(SPI_CONFIG_CNTL, 0); 3402 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3403 WREG32(CP_PERFMON_CNTL, 0); 3404 3405 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) | 3406 FETCH_FIFO_HIWATER(0x4) | 3407 DONE_FIFO_HIWATER(0xe0) | 3408 ALU_UPDATE_FIFO_HIWATER(0x8))); 3409 3410 sq_config = RREG32(SQ_CONFIG); 3411 sq_config &= ~(PS_PRIO(3) | 3412 VS_PRIO(3) | 3413 GS_PRIO(3) | 3414 ES_PRIO(3)); 3415 sq_config |= (VC_ENABLE | 3416 EXPORT_SRC_C | 3417 PS_PRIO(0) | 3418 VS_PRIO(1) | 3419 GS_PRIO(2) | 3420 ES_PRIO(3)); 3421 3422 switch (rdev->family) { 3423 case CHIP_CEDAR: 3424 case CHIP_PALM: 3425 case CHIP_SUMO: 3426 case CHIP_SUMO2: 3427 case CHIP_CAICOS: 3428 /* no vertex cache */ 3429 sq_config &= ~VC_ENABLE; 3430 break; 3431 default: 3432 break; 3433 } 3434 3435 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT); 3436 3437 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32); 3438 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32); 3439 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4); 3440 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 3441 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 3442 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 3443 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 3444 3445 switch (rdev->family) { 3446 case CHIP_CEDAR: 3447 case CHIP_PALM: 3448 case CHIP_SUMO: 3449 case CHIP_SUMO2: 3450 ps_thread_count = 96; 3451 break; 3452 default: 3453 ps_thread_count = 128; 3454 break; 3455 } 3456 3457 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count); 3458 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3459 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3460 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3461 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3462 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3463 3464 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3465 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3466 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3467 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3468 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3469 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3470 3471 WREG32(SQ_CONFIG, sq_config); 3472 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1); 3473 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2); 3474 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3); 3475 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt); 3476 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2); 3477 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1); 3478 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2); 3479 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3); 3480 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0); 3481 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt); 3482 3483 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3484 FORCE_EOV_MAX_REZ_CNT(255))); 3485 3486 switch (rdev->family) { 3487 case CHIP_CEDAR: 3488 case CHIP_PALM: 3489 case CHIP_SUMO: 3490 case CHIP_SUMO2: 3491 case CHIP_CAICOS: 3492 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY); 3493 break; 3494 default: 3495 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC); 3496 break; 3497 } 3498 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO); 3499 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation); 3500 3501 WREG32(VGT_GS_VERTEX_REUSE, 16); 3502 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0); 3503 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3504 3505 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14); 3506 WREG32(VGT_OUT_DEALLOC_CNTL, 16); 3507 3508 WREG32(CB_PERF_CTR0_SEL_0, 0); 3509 WREG32(CB_PERF_CTR0_SEL_1, 0); 3510 WREG32(CB_PERF_CTR1_SEL_0, 0); 3511 WREG32(CB_PERF_CTR1_SEL_1, 0); 3512 WREG32(CB_PERF_CTR2_SEL_0, 0); 3513 WREG32(CB_PERF_CTR2_SEL_1, 0); 3514 WREG32(CB_PERF_CTR3_SEL_0, 0); 3515 WREG32(CB_PERF_CTR3_SEL_1, 0); 3516 3517 /* clear render buffer base addresses */ 3518 WREG32(CB_COLOR0_BASE, 0); 3519 WREG32(CB_COLOR1_BASE, 0); 3520 WREG32(CB_COLOR2_BASE, 0); 3521 WREG32(CB_COLOR3_BASE, 0); 3522 WREG32(CB_COLOR4_BASE, 0); 3523 WREG32(CB_COLOR5_BASE, 0); 3524 WREG32(CB_COLOR6_BASE, 0); 3525 WREG32(CB_COLOR7_BASE, 0); 3526 WREG32(CB_COLOR8_BASE, 0); 3527 WREG32(CB_COLOR9_BASE, 0); 3528 WREG32(CB_COLOR10_BASE, 0); 3529 WREG32(CB_COLOR11_BASE, 0); 3530 3531 /* set the shader const cache sizes to 0 */ 3532 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4) 3533 WREG32(i, 0); 3534 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4) 3535 WREG32(i, 0); 3536 3537 tmp = RREG32(HDP_MISC_CNTL); 3538 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3539 WREG32(HDP_MISC_CNTL, tmp); 3540 3541 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3542 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3543 3544 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3545 3546 udelay(50); 3547 3548 } 3549 3550 int evergreen_mc_init(struct radeon_device *rdev) 3551 { 3552 u32 tmp; 3553 int chansize, numchan; 3554 3555 /* Get VRAM informations */ 3556 rdev->mc.vram_is_ddr = true; 3557 if ((rdev->family == CHIP_PALM) || 3558 (rdev->family == CHIP_SUMO) || 3559 (rdev->family == CHIP_SUMO2)) 3560 tmp = RREG32(FUS_MC_ARB_RAMCFG); 3561 else 3562 tmp = RREG32(MC_ARB_RAMCFG); 3563 if (tmp & CHANSIZE_OVERRIDE) { 3564 chansize = 16; 3565 } else if (tmp & CHANSIZE_MASK) { 3566 chansize = 64; 3567 } else { 3568 chansize = 32; 3569 } 3570 tmp = RREG32(MC_SHARED_CHMAP); 3571 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 3572 case 0: 3573 default: 3574 numchan = 1; 3575 break; 3576 case 1: 3577 numchan = 2; 3578 break; 3579 case 2: 3580 numchan = 4; 3581 break; 3582 case 3: 3583 numchan = 8; 3584 break; 3585 } 3586 rdev->mc.vram_width = numchan * chansize; 3587 /* Could aper size report 0 ? */ 3588 rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0); 3589 rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0); 3590 /* Setup GPU memory space */ 3591 if ((rdev->family == CHIP_PALM) || 3592 (rdev->family == CHIP_SUMO) || 3593 (rdev->family == CHIP_SUMO2)) { 3594 /* size in bytes on fusion */ 3595 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE); 3596 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE); 3597 } else { 3598 /* size in MB on evergreen/cayman/tn */ 3599 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3600 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3601 } 3602 rdev->mc.visible_vram_size = rdev->mc.aper_size; 3603 r700_vram_gtt_location(rdev, &rdev->mc); 3604 radeon_update_bandwidth_info(rdev); 3605 3606 return 0; 3607 } 3608 3609 void evergreen_print_gpu_status_regs(struct radeon_device *rdev) 3610 { 3611 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n", 3612 RREG32(GRBM_STATUS)); 3613 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n", 3614 RREG32(GRBM_STATUS_SE0)); 3615 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n", 3616 RREG32(GRBM_STATUS_SE1)); 3617 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n", 3618 RREG32(SRBM_STATUS)); 3619 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n", 3620 RREG32(SRBM_STATUS2)); 3621 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n", 3622 RREG32(CP_STALLED_STAT1)); 3623 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n", 3624 RREG32(CP_STALLED_STAT2)); 3625 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n", 3626 RREG32(CP_BUSY_STAT)); 3627 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n", 3628 RREG32(CP_STAT)); 3629 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", 3630 RREG32(DMA_STATUS_REG)); 3631 if (rdev->family >= CHIP_CAYMAN) { 3632 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n", 3633 RREG32(DMA_STATUS_REG + 0x800)); 3634 } 3635 } 3636 3637 bool evergreen_is_display_hung(struct radeon_device *rdev) 3638 { 3639 u32 crtc_hung = 0; 3640 u32 crtc_status[6]; 3641 u32 i, j, tmp; 3642 3643 for (i = 0; i < rdev->num_crtc; i++) { 3644 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) { 3645 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); 3646 crtc_hung |= (1 << i); 3647 } 3648 } 3649 3650 for (j = 0; j < 10; j++) { 3651 for (i = 0; i < rdev->num_crtc; i++) { 3652 if (crtc_hung & (1 << i)) { 3653 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); 3654 if (tmp != crtc_status[i]) 3655 crtc_hung &= ~(1 << i); 3656 } 3657 } 3658 if (crtc_hung == 0) 3659 return false; 3660 udelay(100); 3661 } 3662 3663 return true; 3664 } 3665 3666 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev) 3667 { 3668 u32 reset_mask = 0; 3669 u32 tmp; 3670 3671 /* GRBM_STATUS */ 3672 tmp = RREG32(GRBM_STATUS); 3673 if (tmp & (PA_BUSY | SC_BUSY | 3674 SH_BUSY | SX_BUSY | 3675 TA_BUSY | VGT_BUSY | 3676 DB_BUSY | CB_BUSY | 3677 SPI_BUSY | VGT_BUSY_NO_DMA)) 3678 reset_mask |= RADEON_RESET_GFX; 3679 3680 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | 3681 CP_BUSY | CP_COHERENCY_BUSY)) 3682 reset_mask |= RADEON_RESET_CP; 3683 3684 if (tmp & GRBM_EE_BUSY) 3685 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; 3686 3687 /* DMA_STATUS_REG */ 3688 tmp = RREG32(DMA_STATUS_REG); 3689 if (!(tmp & DMA_IDLE)) 3690 reset_mask |= RADEON_RESET_DMA; 3691 3692 /* SRBM_STATUS2 */ 3693 tmp = RREG32(SRBM_STATUS2); 3694 if (tmp & DMA_BUSY) 3695 reset_mask |= RADEON_RESET_DMA; 3696 3697 /* SRBM_STATUS */ 3698 tmp = RREG32(SRBM_STATUS); 3699 if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) 3700 reset_mask |= RADEON_RESET_RLC; 3701 3702 if (tmp & IH_BUSY) 3703 reset_mask |= RADEON_RESET_IH; 3704 3705 if (tmp & SEM_BUSY) 3706 reset_mask |= RADEON_RESET_SEM; 3707 3708 if (tmp & GRBM_RQ_PENDING) 3709 reset_mask |= RADEON_RESET_GRBM; 3710 3711 if (tmp & VMC_BUSY) 3712 reset_mask |= RADEON_RESET_VMC; 3713 3714 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 3715 MCC_BUSY | MCD_BUSY)) 3716 reset_mask |= RADEON_RESET_MC; 3717 3718 if (evergreen_is_display_hung(rdev)) 3719 reset_mask |= RADEON_RESET_DISPLAY; 3720 3721 /* VM_L2_STATUS */ 3722 tmp = RREG32(VM_L2_STATUS); 3723 if (tmp & L2_BUSY) 3724 reset_mask |= RADEON_RESET_VMC; 3725 3726 /* Skip MC reset as it's mostly likely not hung, just busy */ 3727 if (reset_mask & RADEON_RESET_MC) { 3728 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 3729 reset_mask &= ~RADEON_RESET_MC; 3730 } 3731 3732 return reset_mask; 3733 } 3734 3735 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 3736 { 3737 struct evergreen_mc_save save; 3738 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 3739 u32 tmp; 3740 3741 if (reset_mask == 0) 3742 return; 3743 3744 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 3745 3746 evergreen_print_gpu_status_regs(rdev); 3747 3748 /* Disable CP parsing/prefetching */ 3749 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 3750 3751 if (reset_mask & RADEON_RESET_DMA) { 3752 /* Disable DMA */ 3753 tmp = RREG32(DMA_RB_CNTL); 3754 tmp &= ~DMA_RB_ENABLE; 3755 WREG32(DMA_RB_CNTL, tmp); 3756 } 3757 3758 udelay(50); 3759 3760 evergreen_mc_stop(rdev, &save); 3761 if (evergreen_mc_wait_for_idle(rdev)) { 3762 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3763 } 3764 3765 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) { 3766 grbm_soft_reset |= SOFT_RESET_DB | 3767 SOFT_RESET_CB | 3768 SOFT_RESET_PA | 3769 SOFT_RESET_SC | 3770 SOFT_RESET_SPI | 3771 SOFT_RESET_SX | 3772 SOFT_RESET_SH | 3773 SOFT_RESET_TC | 3774 SOFT_RESET_TA | 3775 SOFT_RESET_VC | 3776 SOFT_RESET_VGT; 3777 } 3778 3779 if (reset_mask & RADEON_RESET_CP) { 3780 grbm_soft_reset |= SOFT_RESET_CP | 3781 SOFT_RESET_VGT; 3782 3783 srbm_soft_reset |= SOFT_RESET_GRBM; 3784 } 3785 3786 if (reset_mask & RADEON_RESET_DMA) 3787 srbm_soft_reset |= SOFT_RESET_DMA; 3788 3789 if (reset_mask & RADEON_RESET_DISPLAY) 3790 srbm_soft_reset |= SOFT_RESET_DC; 3791 3792 if (reset_mask & RADEON_RESET_RLC) 3793 srbm_soft_reset |= SOFT_RESET_RLC; 3794 3795 if (reset_mask & RADEON_RESET_SEM) 3796 srbm_soft_reset |= SOFT_RESET_SEM; 3797 3798 if (reset_mask & RADEON_RESET_IH) 3799 srbm_soft_reset |= SOFT_RESET_IH; 3800 3801 if (reset_mask & RADEON_RESET_GRBM) 3802 srbm_soft_reset |= SOFT_RESET_GRBM; 3803 3804 if (reset_mask & RADEON_RESET_VMC) 3805 srbm_soft_reset |= SOFT_RESET_VMC; 3806 3807 if (!(rdev->flags & RADEON_IS_IGP)) { 3808 if (reset_mask & RADEON_RESET_MC) 3809 srbm_soft_reset |= SOFT_RESET_MC; 3810 } 3811 3812 if (grbm_soft_reset) { 3813 tmp = RREG32(GRBM_SOFT_RESET); 3814 tmp |= grbm_soft_reset; 3815 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 3816 WREG32(GRBM_SOFT_RESET, tmp); 3817 tmp = RREG32(GRBM_SOFT_RESET); 3818 3819 udelay(50); 3820 3821 tmp &= ~grbm_soft_reset; 3822 WREG32(GRBM_SOFT_RESET, tmp); 3823 tmp = RREG32(GRBM_SOFT_RESET); 3824 } 3825 3826 if (srbm_soft_reset) { 3827 tmp = RREG32(SRBM_SOFT_RESET); 3828 tmp |= srbm_soft_reset; 3829 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 3830 WREG32(SRBM_SOFT_RESET, tmp); 3831 tmp = RREG32(SRBM_SOFT_RESET); 3832 3833 udelay(50); 3834 3835 tmp &= ~srbm_soft_reset; 3836 WREG32(SRBM_SOFT_RESET, tmp); 3837 tmp = RREG32(SRBM_SOFT_RESET); 3838 } 3839 3840 /* Wait a little for things to settle down */ 3841 udelay(50); 3842 3843 evergreen_mc_resume(rdev, &save); 3844 udelay(50); 3845 3846 evergreen_print_gpu_status_regs(rdev); 3847 } 3848 3849 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev) 3850 { 3851 struct evergreen_mc_save save; 3852 u32 tmp, i; 3853 3854 dev_info(rdev->dev, "GPU pci config reset\n"); 3855 3856 /* disable dpm? */ 3857 3858 /* Disable CP parsing/prefetching */ 3859 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 3860 udelay(50); 3861 /* Disable DMA */ 3862 tmp = RREG32(DMA_RB_CNTL); 3863 tmp &= ~DMA_RB_ENABLE; 3864 WREG32(DMA_RB_CNTL, tmp); 3865 /* XXX other engines? */ 3866 3867 /* halt the rlc */ 3868 r600_rlc_stop(rdev); 3869 3870 udelay(50); 3871 3872 /* set mclk/sclk to bypass */ 3873 rv770_set_clk_bypass_mode(rdev); 3874 /* disable BM */ 3875 pci_disable_busmaster(rdev->pdev->dev); 3876 /* disable mem access */ 3877 evergreen_mc_stop(rdev, &save); 3878 if (evergreen_mc_wait_for_idle(rdev)) { 3879 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 3880 } 3881 /* reset */ 3882 radeon_pci_config_reset(rdev); 3883 /* wait for asic to come out of reset */ 3884 for (i = 0; i < rdev->usec_timeout; i++) { 3885 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 3886 break; 3887 udelay(1); 3888 } 3889 } 3890 3891 int evergreen_asic_reset(struct radeon_device *rdev) 3892 { 3893 u32 reset_mask; 3894 3895 reset_mask = evergreen_gpu_check_soft_reset(rdev); 3896 3897 if (reset_mask) 3898 r600_set_bios_scratch_engine_hung(rdev, true); 3899 3900 /* try soft reset */ 3901 evergreen_gpu_soft_reset(rdev, reset_mask); 3902 3903 reset_mask = evergreen_gpu_check_soft_reset(rdev); 3904 3905 /* try pci config reset */ 3906 if (reset_mask && radeon_hard_reset) 3907 evergreen_gpu_pci_config_reset(rdev); 3908 3909 reset_mask = evergreen_gpu_check_soft_reset(rdev); 3910 3911 if (!reset_mask) 3912 r600_set_bios_scratch_engine_hung(rdev, false); 3913 3914 return 0; 3915 } 3916 3917 /** 3918 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up 3919 * 3920 * @rdev: radeon_device pointer 3921 * @ring: radeon_ring structure holding ring information 3922 * 3923 * Check if the GFX engine is locked up. 3924 * Returns true if the engine appears to be locked up, false if not. 3925 */ 3926 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 3927 { 3928 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev); 3929 3930 if (!(reset_mask & (RADEON_RESET_GFX | 3931 RADEON_RESET_COMPUTE | 3932 RADEON_RESET_CP))) { 3933 radeon_ring_lockup_update(rdev, ring); 3934 return false; 3935 } 3936 return radeon_ring_test_lockup(rdev, ring); 3937 } 3938 3939 /* 3940 * RLC 3941 */ 3942 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000 3943 #define RLC_CLEAR_STATE_END_MARKER 0x00000001 3944 3945 void sumo_rlc_fini(struct radeon_device *rdev) 3946 { 3947 int r; 3948 3949 /* save restore block */ 3950 if (rdev->rlc.save_restore_obj) { 3951 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 3952 if (unlikely(r != 0)) 3953 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r); 3954 radeon_bo_unpin(rdev->rlc.save_restore_obj); 3955 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 3956 3957 radeon_bo_unref(&rdev->rlc.save_restore_obj); 3958 rdev->rlc.save_restore_obj = NULL; 3959 } 3960 3961 /* clear state block */ 3962 if (rdev->rlc.clear_state_obj) { 3963 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 3964 if (unlikely(r != 0)) 3965 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r); 3966 radeon_bo_unpin(rdev->rlc.clear_state_obj); 3967 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 3968 3969 radeon_bo_unref(&rdev->rlc.clear_state_obj); 3970 rdev->rlc.clear_state_obj = NULL; 3971 } 3972 3973 /* clear state block */ 3974 if (rdev->rlc.cp_table_obj) { 3975 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); 3976 if (unlikely(r != 0)) 3977 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); 3978 radeon_bo_unpin(rdev->rlc.cp_table_obj); 3979 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 3980 3981 radeon_bo_unref(&rdev->rlc.cp_table_obj); 3982 rdev->rlc.cp_table_obj = NULL; 3983 } 3984 } 3985 3986 #define CP_ME_TABLE_SIZE 96 3987 3988 int sumo_rlc_init(struct radeon_device *rdev) 3989 { 3990 const u32 *src_ptr; 3991 volatile u32 *dst_ptr; 3992 u32 dws, data, i, j, k, reg_num; 3993 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0; 3994 u64 reg_list_mc_addr; 3995 const struct cs_section_def *cs_data; 3996 int r; 3997 void *vptr; 3998 3999 vptr = NULL; 4000 src_ptr = rdev->rlc.reg_list; 4001 dws = rdev->rlc.reg_list_size; 4002 if (rdev->family >= CHIP_BONAIRE) { 4003 dws += (5 * 16) + 48 + 48 + 64; 4004 } 4005 cs_data = rdev->rlc.cs_data; 4006 4007 if (src_ptr) { 4008 /* save restore block */ 4009 if (rdev->rlc.save_restore_obj == NULL) { 4010 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, 4011 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4012 &rdev->rlc.save_restore_obj); 4013 if (r) { 4014 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r); 4015 return r; 4016 } 4017 } 4018 4019 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 4020 if (unlikely(r != 0)) { 4021 sumo_rlc_fini(rdev); 4022 return r; 4023 } 4024 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM, 4025 &rdev->rlc.save_restore_gpu_addr); 4026 if (r) { 4027 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4028 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r); 4029 sumo_rlc_fini(rdev); 4030 return r; 4031 } 4032 4033 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void**)&vptr); 4034 if (r) { 4035 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r); 4036 sumo_rlc_fini(rdev); 4037 return r; 4038 } 4039 rdev->rlc.sr_ptr = vptr; 4040 /* write the sr buffer */ 4041 dst_ptr = rdev->rlc.sr_ptr; 4042 if (rdev->family >= CHIP_TAHITI) { 4043 /* SI */ 4044 for (i = 0; i < rdev->rlc.reg_list_size; i++) 4045 dst_ptr[i] = cpu_to_le32(src_ptr[i]); 4046 } else { 4047 /* ON/LN/TN */ 4048 /* format: 4049 * dw0: (reg2 << 16) | reg1 4050 * dw1: reg1 save space 4051 * dw2: reg2 save space 4052 */ 4053 for (i = 0; i < dws; i++) { 4054 data = src_ptr[i] >> 2; 4055 i++; 4056 if (i < dws) 4057 data |= (src_ptr[i] >> 2) << 16; 4058 j = (((i - 1) * 3) / 2); 4059 dst_ptr[j] = cpu_to_le32(data); 4060 } 4061 j = ((i * 3) / 2); 4062 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER); 4063 } 4064 radeon_bo_kunmap(rdev->rlc.save_restore_obj); 4065 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4066 } 4067 4068 if (cs_data) { 4069 /* clear state block */ 4070 if (rdev->family >= CHIP_BONAIRE) { 4071 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev); 4072 } else if (rdev->family >= CHIP_TAHITI) { 4073 rdev->rlc.clear_state_size = si_get_csb_size(rdev); 4074 dws = rdev->rlc.clear_state_size + (256 / 4); 4075 } else { 4076 reg_list_num = 0; 4077 dws = 0; 4078 for (i = 0; cs_data[i].section != NULL; i++) { 4079 for (j = 0; cs_data[i].section[j].extent != NULL; j++) { 4080 reg_list_num++; 4081 dws += cs_data[i].section[j].reg_count; 4082 } 4083 } 4084 reg_list_blk_index = (3 * reg_list_num + 2); 4085 dws += reg_list_blk_index; 4086 rdev->rlc.clear_state_size = dws; 4087 } 4088 4089 if (rdev->rlc.clear_state_obj == NULL) { 4090 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, 4091 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4092 &rdev->rlc.clear_state_obj); 4093 if (r) { 4094 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r); 4095 sumo_rlc_fini(rdev); 4096 return r; 4097 } 4098 } 4099 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 4100 if (unlikely(r != 0)) { 4101 sumo_rlc_fini(rdev); 4102 return r; 4103 } 4104 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM, 4105 &rdev->rlc.clear_state_gpu_addr); 4106 if (r) { 4107 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4108 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r); 4109 sumo_rlc_fini(rdev); 4110 return r; 4111 } 4112 4113 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void**)&vptr); 4114 if (r) { 4115 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r); 4116 sumo_rlc_fini(rdev); 4117 return r; 4118 } 4119 rdev->rlc.cs_ptr = vptr; 4120 /* set up the cs buffer */ 4121 dst_ptr = rdev->rlc.cs_ptr; 4122 if (rdev->family >= CHIP_BONAIRE) { 4123 cik_get_csb_buffer(rdev, dst_ptr); 4124 } else if (rdev->family >= CHIP_TAHITI) { 4125 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256; 4126 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr)); 4127 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr)); 4128 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size); 4129 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]); 4130 } else { 4131 reg_list_hdr_blk_index = 0; 4132 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4); 4133 data = upper_32_bits(reg_list_mc_addr); 4134 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4135 reg_list_hdr_blk_index++; 4136 for (i = 0; cs_data[i].section != NULL; i++) { 4137 for (j = 0; cs_data[i].section[j].extent != NULL; j++) { 4138 reg_num = cs_data[i].section[j].reg_count; 4139 data = reg_list_mc_addr & 0xffffffff; 4140 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4141 reg_list_hdr_blk_index++; 4142 4143 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff; 4144 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4145 reg_list_hdr_blk_index++; 4146 4147 data = 0x08000000 | (reg_num * 4); 4148 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4149 reg_list_hdr_blk_index++; 4150 4151 for (k = 0; k < reg_num; k++) { 4152 data = cs_data[i].section[j].extent[k]; 4153 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data); 4154 } 4155 reg_list_mc_addr += reg_num * 4; 4156 reg_list_blk_index += reg_num; 4157 } 4158 } 4159 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER); 4160 } 4161 radeon_bo_kunmap(rdev->rlc.clear_state_obj); 4162 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4163 } 4164 4165 if (rdev->rlc.cp_table_size) { 4166 if (rdev->rlc.cp_table_obj == NULL) { 4167 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, 4168 PAGE_SIZE, true, 4169 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4170 &rdev->rlc.cp_table_obj); 4171 if (r) { 4172 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r); 4173 sumo_rlc_fini(rdev); 4174 return r; 4175 } 4176 } 4177 4178 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); 4179 if (unlikely(r != 0)) { 4180 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); 4181 sumo_rlc_fini(rdev); 4182 return r; 4183 } 4184 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM, 4185 &rdev->rlc.cp_table_gpu_addr); 4186 if (r) { 4187 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4188 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r); 4189 sumo_rlc_fini(rdev); 4190 return r; 4191 } 4192 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void**)&vptr); 4193 if (r) { 4194 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r); 4195 sumo_rlc_fini(rdev); 4196 return r; 4197 } 4198 rdev->rlc.cp_table_ptr = vptr; 4199 4200 cik_init_cp_pg_table(rdev); 4201 4202 radeon_bo_kunmap(rdev->rlc.cp_table_obj); 4203 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4204 4205 } 4206 4207 return 0; 4208 } 4209 4210 static void evergreen_rlc_start(struct radeon_device *rdev) 4211 { 4212 u32 mask = RLC_ENABLE; 4213 4214 if (rdev->flags & RADEON_IS_IGP) { 4215 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC; 4216 } 4217 4218 WREG32(RLC_CNTL, mask); 4219 } 4220 4221 int evergreen_rlc_resume(struct radeon_device *rdev) 4222 { 4223 u32 i; 4224 const __be32 *fw_data; 4225 4226 if (!rdev->rlc_fw) 4227 return -EINVAL; 4228 4229 r600_rlc_stop(rdev); 4230 4231 WREG32(RLC_HB_CNTL, 0); 4232 4233 if (rdev->flags & RADEON_IS_IGP) { 4234 if (rdev->family == CHIP_ARUBA) { 4235 u32 always_on_bitmap = 4236 3 | (3 << (16 * rdev->config.cayman.max_shader_engines)); 4237 /* find out the number of active simds */ 4238 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; 4239 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se; 4240 tmp = hweight32(~tmp); 4241 if (tmp == rdev->config.cayman.max_simds_per_se) { 4242 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap); 4243 WREG32(TN_RLC_LB_PARAMS, 0x00601004); 4244 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff); 4245 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000); 4246 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000); 4247 } 4248 } else { 4249 WREG32(RLC_HB_WPTR_LSB_ADDR, 0); 4250 WREG32(RLC_HB_WPTR_MSB_ADDR, 0); 4251 } 4252 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 4253 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 4254 } else { 4255 WREG32(RLC_HB_BASE, 0); 4256 WREG32(RLC_HB_RPTR, 0); 4257 WREG32(RLC_HB_WPTR, 0); 4258 WREG32(RLC_HB_WPTR_LSB_ADDR, 0); 4259 WREG32(RLC_HB_WPTR_MSB_ADDR, 0); 4260 } 4261 WREG32(RLC_MC_CNTL, 0); 4262 WREG32(RLC_UCODE_CNTL, 0); 4263 4264 fw_data = (const __be32 *)rdev->rlc_fw->data; 4265 if (rdev->family >= CHIP_ARUBA) { 4266 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) { 4267 WREG32(RLC_UCODE_ADDR, i); 4268 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4269 } 4270 } else if (rdev->family >= CHIP_CAYMAN) { 4271 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) { 4272 WREG32(RLC_UCODE_ADDR, i); 4273 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4274 } 4275 } else { 4276 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) { 4277 WREG32(RLC_UCODE_ADDR, i); 4278 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4279 } 4280 } 4281 WREG32(RLC_UCODE_ADDR, 0); 4282 4283 evergreen_rlc_start(rdev); 4284 4285 return 0; 4286 } 4287 4288 /* Interrupts */ 4289 4290 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc) 4291 { 4292 if (crtc >= rdev->num_crtc) 4293 return 0; 4294 else 4295 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]); 4296 } 4297 4298 void evergreen_disable_interrupt_state(struct radeon_device *rdev) 4299 { 4300 u32 tmp; 4301 4302 if (rdev->family >= CHIP_CAYMAN) { 4303 cayman_cp_int_cntl_setup(rdev, 0, 4304 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4305 cayman_cp_int_cntl_setup(rdev, 1, 0); 4306 cayman_cp_int_cntl_setup(rdev, 2, 0); 4307 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; 4308 WREG32(CAYMAN_DMA1_CNTL, tmp); 4309 } else 4310 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4311 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE; 4312 WREG32(DMA_CNTL, tmp); 4313 WREG32(GRBM_INT_CNTL, 0); 4314 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 4315 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 4316 if (rdev->num_crtc >= 4) { 4317 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 4318 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 4319 } 4320 if (rdev->num_crtc >= 6) { 4321 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 4322 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 4323 } 4324 4325 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 4326 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 4327 if (rdev->num_crtc >= 4) { 4328 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 4329 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 4330 } 4331 if (rdev->num_crtc >= 6) { 4332 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 4333 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 4334 } 4335 4336 /* only one DAC on DCE5 */ 4337 if (!ASIC_IS_DCE5(rdev)) 4338 WREG32(DACA_AUTODETECT_INT_CONTROL, 0); 4339 WREG32(DACB_AUTODETECT_INT_CONTROL, 0); 4340 4341 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4342 WREG32(DC_HPD1_INT_CONTROL, tmp); 4343 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4344 WREG32(DC_HPD2_INT_CONTROL, tmp); 4345 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4346 WREG32(DC_HPD3_INT_CONTROL, tmp); 4347 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4348 WREG32(DC_HPD4_INT_CONTROL, tmp); 4349 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4350 WREG32(DC_HPD5_INT_CONTROL, tmp); 4351 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4352 WREG32(DC_HPD6_INT_CONTROL, tmp); 4353 4354 } 4355 4356 int evergreen_irq_set(struct radeon_device *rdev) 4357 { 4358 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; 4359 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; 4360 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 4361 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 4362 u32 grbm_int_cntl = 0; 4363 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0; 4364 u32 dma_cntl, dma_cntl1 = 0; 4365 u32 thermal_int = 0; 4366 4367 if (!rdev->irq.installed) { 4368 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 4369 return -EINVAL; 4370 } 4371 /* don't enable anything if the ih is disabled */ 4372 if (!rdev->ih.enabled) { 4373 r600_disable_interrupts(rdev); 4374 /* force the active interrupt state to all disabled */ 4375 evergreen_disable_interrupt_state(rdev); 4376 return 0; 4377 } 4378 4379 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; 4380 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; 4381 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; 4382 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; 4383 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 4384 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 4385 if (rdev->family == CHIP_ARUBA) 4386 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) & 4387 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 4388 else 4389 thermal_int = RREG32(CG_THERMAL_INT) & 4390 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 4391 4392 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4393 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4394 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4395 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4396 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4397 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4398 4399 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE; 4400 4401 if (rdev->family >= CHIP_CAYMAN) { 4402 /* enable CP interrupts on all rings */ 4403 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 4404 DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); 4405 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 4406 } 4407 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 4408 DRM_DEBUG("evergreen_irq_set: sw int cp1\n"); 4409 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE; 4410 } 4411 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 4412 DRM_DEBUG("evergreen_irq_set: sw int cp2\n"); 4413 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE; 4414 } 4415 } else { 4416 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 4417 DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); 4418 cp_int_cntl |= RB_INT_ENABLE; 4419 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 4420 } 4421 } 4422 4423 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 4424 DRM_DEBUG("r600_irq_set: sw int dma\n"); 4425 dma_cntl |= TRAP_ENABLE; 4426 } 4427 4428 if (rdev->family >= CHIP_CAYMAN) { 4429 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; 4430 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 4431 DRM_DEBUG("r600_irq_set: sw int dma1\n"); 4432 dma_cntl1 |= TRAP_ENABLE; 4433 } 4434 } 4435 4436 if (rdev->irq.dpm_thermal) { 4437 DRM_DEBUG("dpm thermal\n"); 4438 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; 4439 } 4440 4441 if (rdev->irq.crtc_vblank_int[0] || 4442 atomic_read(&rdev->irq.pflip[0])) { 4443 DRM_DEBUG("evergreen_irq_set: vblank 0\n"); 4444 crtc1 |= VBLANK_INT_MASK; 4445 } 4446 if (rdev->irq.crtc_vblank_int[1] || 4447 atomic_read(&rdev->irq.pflip[1])) { 4448 DRM_DEBUG("evergreen_irq_set: vblank 1\n"); 4449 crtc2 |= VBLANK_INT_MASK; 4450 } 4451 if (rdev->irq.crtc_vblank_int[2] || 4452 atomic_read(&rdev->irq.pflip[2])) { 4453 DRM_DEBUG("evergreen_irq_set: vblank 2\n"); 4454 crtc3 |= VBLANK_INT_MASK; 4455 } 4456 if (rdev->irq.crtc_vblank_int[3] || 4457 atomic_read(&rdev->irq.pflip[3])) { 4458 DRM_DEBUG("evergreen_irq_set: vblank 3\n"); 4459 crtc4 |= VBLANK_INT_MASK; 4460 } 4461 if (rdev->irq.crtc_vblank_int[4] || 4462 atomic_read(&rdev->irq.pflip[4])) { 4463 DRM_DEBUG("evergreen_irq_set: vblank 4\n"); 4464 crtc5 |= VBLANK_INT_MASK; 4465 } 4466 if (rdev->irq.crtc_vblank_int[5] || 4467 atomic_read(&rdev->irq.pflip[5])) { 4468 DRM_DEBUG("evergreen_irq_set: vblank 5\n"); 4469 crtc6 |= VBLANK_INT_MASK; 4470 } 4471 if (rdev->irq.hpd[0]) { 4472 DRM_DEBUG("evergreen_irq_set: hpd 1\n"); 4473 hpd1 |= DC_HPDx_INT_EN; 4474 } 4475 if (rdev->irq.hpd[1]) { 4476 DRM_DEBUG("evergreen_irq_set: hpd 2\n"); 4477 hpd2 |= DC_HPDx_INT_EN; 4478 } 4479 if (rdev->irq.hpd[2]) { 4480 DRM_DEBUG("evergreen_irq_set: hpd 3\n"); 4481 hpd3 |= DC_HPDx_INT_EN; 4482 } 4483 if (rdev->irq.hpd[3]) { 4484 DRM_DEBUG("evergreen_irq_set: hpd 4\n"); 4485 hpd4 |= DC_HPDx_INT_EN; 4486 } 4487 if (rdev->irq.hpd[4]) { 4488 DRM_DEBUG("evergreen_irq_set: hpd 5\n"); 4489 hpd5 |= DC_HPDx_INT_EN; 4490 } 4491 if (rdev->irq.hpd[5]) { 4492 DRM_DEBUG("evergreen_irq_set: hpd 6\n"); 4493 hpd6 |= DC_HPDx_INT_EN; 4494 } 4495 if (rdev->irq.afmt[0]) { 4496 DRM_DEBUG("evergreen_irq_set: hdmi 0\n"); 4497 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4498 } 4499 if (rdev->irq.afmt[1]) { 4500 DRM_DEBUG("evergreen_irq_set: hdmi 1\n"); 4501 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4502 } 4503 if (rdev->irq.afmt[2]) { 4504 DRM_DEBUG("evergreen_irq_set: hdmi 2\n"); 4505 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4506 } 4507 if (rdev->irq.afmt[3]) { 4508 DRM_DEBUG("evergreen_irq_set: hdmi 3\n"); 4509 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4510 } 4511 if (rdev->irq.afmt[4]) { 4512 DRM_DEBUG("evergreen_irq_set: hdmi 4\n"); 4513 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4514 } 4515 if (rdev->irq.afmt[5]) { 4516 DRM_DEBUG("evergreen_irq_set: hdmi 5\n"); 4517 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4518 } 4519 4520 if (rdev->family >= CHIP_CAYMAN) { 4521 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl); 4522 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1); 4523 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2); 4524 } else 4525 WREG32(CP_INT_CNTL, cp_int_cntl); 4526 4527 WREG32(DMA_CNTL, dma_cntl); 4528 4529 if (rdev->family >= CHIP_CAYMAN) 4530 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1); 4531 4532 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 4533 4534 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 4535 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 4536 if (rdev->num_crtc >= 4) { 4537 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 4538 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 4539 } 4540 if (rdev->num_crtc >= 6) { 4541 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 4542 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 4543 } 4544 4545 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 4546 GRPH_PFLIP_INT_MASK); 4547 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 4548 GRPH_PFLIP_INT_MASK); 4549 if (rdev->num_crtc >= 4) { 4550 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 4551 GRPH_PFLIP_INT_MASK); 4552 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 4553 GRPH_PFLIP_INT_MASK); 4554 } 4555 if (rdev->num_crtc >= 6) { 4556 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 4557 GRPH_PFLIP_INT_MASK); 4558 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 4559 GRPH_PFLIP_INT_MASK); 4560 } 4561 4562 WREG32(DC_HPD1_INT_CONTROL, hpd1); 4563 WREG32(DC_HPD2_INT_CONTROL, hpd2); 4564 WREG32(DC_HPD3_INT_CONTROL, hpd3); 4565 WREG32(DC_HPD4_INT_CONTROL, hpd4); 4566 WREG32(DC_HPD5_INT_CONTROL, hpd5); 4567 WREG32(DC_HPD6_INT_CONTROL, hpd6); 4568 if (rdev->family == CHIP_ARUBA) 4569 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int); 4570 else 4571 WREG32(CG_THERMAL_INT, thermal_int); 4572 4573 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1); 4574 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2); 4575 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3); 4576 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4); 4577 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5); 4578 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6); 4579 4580 return 0; 4581 } 4582 4583 static void evergreen_irq_ack(struct radeon_device *rdev) 4584 { 4585 u32 tmp; 4586 4587 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS); 4588 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 4589 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 4590 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 4591 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 4592 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 4593 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 4594 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 4595 if (rdev->num_crtc >= 4) { 4596 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 4597 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 4598 } 4599 if (rdev->num_crtc >= 6) { 4600 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 4601 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 4602 } 4603 4604 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 4605 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 4606 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 4607 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 4608 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 4609 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 4610 4611 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 4612 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4613 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 4614 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4615 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) 4616 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 4617 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) 4618 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 4619 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 4620 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 4621 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) 4622 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 4623 4624 if (rdev->num_crtc >= 4) { 4625 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 4626 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4627 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 4628 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4629 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 4630 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 4631 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 4632 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 4633 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 4634 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 4635 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 4636 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 4637 } 4638 4639 if (rdev->num_crtc >= 6) { 4640 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 4641 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4642 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 4643 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4644 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 4645 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 4646 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 4647 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 4648 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 4649 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 4650 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 4651 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 4652 } 4653 4654 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 4655 tmp = RREG32(DC_HPD1_INT_CONTROL); 4656 tmp |= DC_HPDx_INT_ACK; 4657 WREG32(DC_HPD1_INT_CONTROL, tmp); 4658 } 4659 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 4660 tmp = RREG32(DC_HPD2_INT_CONTROL); 4661 tmp |= DC_HPDx_INT_ACK; 4662 WREG32(DC_HPD2_INT_CONTROL, tmp); 4663 } 4664 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 4665 tmp = RREG32(DC_HPD3_INT_CONTROL); 4666 tmp |= DC_HPDx_INT_ACK; 4667 WREG32(DC_HPD3_INT_CONTROL, tmp); 4668 } 4669 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 4670 tmp = RREG32(DC_HPD4_INT_CONTROL); 4671 tmp |= DC_HPDx_INT_ACK; 4672 WREG32(DC_HPD4_INT_CONTROL, tmp); 4673 } 4674 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 4675 tmp = RREG32(DC_HPD5_INT_CONTROL); 4676 tmp |= DC_HPDx_INT_ACK; 4677 WREG32(DC_HPD5_INT_CONTROL, tmp); 4678 } 4679 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 4680 tmp = RREG32(DC_HPD5_INT_CONTROL); 4681 tmp |= DC_HPDx_INT_ACK; 4682 WREG32(DC_HPD6_INT_CONTROL, tmp); 4683 } 4684 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) { 4685 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET); 4686 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4687 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp); 4688 } 4689 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) { 4690 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET); 4691 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4692 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp); 4693 } 4694 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) { 4695 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET); 4696 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4697 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp); 4698 } 4699 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) { 4700 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET); 4701 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4702 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp); 4703 } 4704 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) { 4705 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET); 4706 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4707 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp); 4708 } 4709 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) { 4710 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET); 4711 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4712 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp); 4713 } 4714 } 4715 4716 static void evergreen_irq_disable(struct radeon_device *rdev) 4717 { 4718 r600_disable_interrupts(rdev); 4719 /* Wait and acknowledge irq */ 4720 mdelay(1); 4721 evergreen_irq_ack(rdev); 4722 evergreen_disable_interrupt_state(rdev); 4723 } 4724 4725 void evergreen_irq_suspend(struct radeon_device *rdev) 4726 { 4727 evergreen_irq_disable(rdev); 4728 r600_rlc_stop(rdev); 4729 } 4730 4731 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev) 4732 { 4733 u32 wptr, tmp; 4734 4735 if (rdev->wb.enabled) 4736 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 4737 else 4738 wptr = RREG32(IH_RB_WPTR); 4739 4740 if (wptr & RB_OVERFLOW) { 4741 wptr &= ~RB_OVERFLOW; 4742 /* When a ring buffer overflow happen start parsing interrupt 4743 * from the last not overwritten vector (wptr + 16). Hopefully 4744 * this should allow us to catchup. 4745 */ 4746 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", 4747 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); 4748 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 4749 tmp = RREG32(IH_RB_CNTL); 4750 tmp |= IH_WPTR_OVERFLOW_CLEAR; 4751 WREG32(IH_RB_CNTL, tmp); 4752 } 4753 return (wptr & rdev->ih.ptr_mask); 4754 } 4755 4756 irqreturn_t evergreen_irq_process(struct radeon_device *rdev) 4757 { 4758 u32 wptr; 4759 u32 rptr; 4760 u32 src_id, src_data; 4761 u32 ring_index; 4762 bool queue_hotplug = false; 4763 bool queue_hdmi = false; 4764 bool queue_thermal = false; 4765 u32 status, addr; 4766 4767 if (!rdev->ih.enabled || rdev->shutdown) 4768 return IRQ_NONE; 4769 4770 wptr = evergreen_get_ih_wptr(rdev); 4771 4772 restart_ih: 4773 /* is somebody else already processing irqs? */ 4774 if (atomic_xchg(&rdev->ih.lock, 1)) 4775 return IRQ_NONE; 4776 4777 rptr = rdev->ih.rptr; 4778 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 4779 4780 /* Order reading of wptr vs. reading of IH ring data */ 4781 rmb(); 4782 4783 /* display interrupts */ 4784 evergreen_irq_ack(rdev); 4785 4786 while (rptr != wptr) { 4787 /* wptr/rptr are in bytes! */ 4788 ring_index = rptr / 4; 4789 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 4790 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 4791 4792 switch (src_id) { 4793 case 1: /* D1 vblank/vline */ 4794 switch (src_data) { 4795 case 0: /* D1 vblank */ 4796 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) { 4797 if (rdev->irq.crtc_vblank_int[0]) { 4798 drm_handle_vblank(rdev->ddev, 0); 4799 rdev->pm.vblank_sync = true; 4800 wake_up(&rdev->irq.vblank_queue); 4801 } 4802 if (atomic_read(&rdev->irq.pflip[0])) 4803 radeon_crtc_handle_vblank(rdev, 0); 4804 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 4805 DRM_DEBUG("IH: D1 vblank\n"); 4806 } 4807 break; 4808 case 1: /* D1 vline */ 4809 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) { 4810 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; 4811 DRM_DEBUG("IH: D1 vline\n"); 4812 } 4813 break; 4814 default: 4815 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 4816 break; 4817 } 4818 break; 4819 case 2: /* D2 vblank/vline */ 4820 switch (src_data) { 4821 case 0: /* D2 vblank */ 4822 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 4823 if (rdev->irq.crtc_vblank_int[1]) { 4824 drm_handle_vblank(rdev->ddev, 1); 4825 rdev->pm.vblank_sync = true; 4826 wake_up(&rdev->irq.vblank_queue); 4827 } 4828 if (atomic_read(&rdev->irq.pflip[1])) 4829 radeon_crtc_handle_vblank(rdev, 1); 4830 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 4831 DRM_DEBUG("IH: D2 vblank\n"); 4832 } 4833 break; 4834 case 1: /* D2 vline */ 4835 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) { 4836 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 4837 DRM_DEBUG("IH: D2 vline\n"); 4838 } 4839 break; 4840 default: 4841 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 4842 break; 4843 } 4844 break; 4845 case 3: /* D3 vblank/vline */ 4846 switch (src_data) { 4847 case 0: /* D3 vblank */ 4848 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 4849 if (rdev->irq.crtc_vblank_int[2]) { 4850 drm_handle_vblank(rdev->ddev, 2); 4851 rdev->pm.vblank_sync = true; 4852 wake_up(&rdev->irq.vblank_queue); 4853 } 4854 if (atomic_read(&rdev->irq.pflip[2])) 4855 radeon_crtc_handle_vblank(rdev, 2); 4856 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 4857 DRM_DEBUG("IH: D3 vblank\n"); 4858 } 4859 break; 4860 case 1: /* D3 vline */ 4861 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 4862 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 4863 DRM_DEBUG("IH: D3 vline\n"); 4864 } 4865 break; 4866 default: 4867 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 4868 break; 4869 } 4870 break; 4871 case 4: /* D4 vblank/vline */ 4872 switch (src_data) { 4873 case 0: /* D4 vblank */ 4874 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 4875 if (rdev->irq.crtc_vblank_int[3]) { 4876 drm_handle_vblank(rdev->ddev, 3); 4877 rdev->pm.vblank_sync = true; 4878 wake_up(&rdev->irq.vblank_queue); 4879 } 4880 if (atomic_read(&rdev->irq.pflip[3])) 4881 radeon_crtc_handle_vblank(rdev, 3); 4882 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 4883 DRM_DEBUG("IH: D4 vblank\n"); 4884 } 4885 break; 4886 case 1: /* D4 vline */ 4887 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 4888 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 4889 DRM_DEBUG("IH: D4 vline\n"); 4890 } 4891 break; 4892 default: 4893 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 4894 break; 4895 } 4896 break; 4897 case 5: /* D5 vblank/vline */ 4898 switch (src_data) { 4899 case 0: /* D5 vblank */ 4900 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 4901 if (rdev->irq.crtc_vblank_int[4]) { 4902 drm_handle_vblank(rdev->ddev, 4); 4903 rdev->pm.vblank_sync = true; 4904 wake_up(&rdev->irq.vblank_queue); 4905 } 4906 if (atomic_read(&rdev->irq.pflip[4])) 4907 radeon_crtc_handle_vblank(rdev, 4); 4908 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 4909 DRM_DEBUG("IH: D5 vblank\n"); 4910 } 4911 break; 4912 case 1: /* D5 vline */ 4913 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 4914 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 4915 DRM_DEBUG("IH: D5 vline\n"); 4916 } 4917 break; 4918 default: 4919 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 4920 break; 4921 } 4922 break; 4923 case 6: /* D6 vblank/vline */ 4924 switch (src_data) { 4925 case 0: /* D6 vblank */ 4926 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 4927 if (rdev->irq.crtc_vblank_int[5]) { 4928 drm_handle_vblank(rdev->ddev, 5); 4929 rdev->pm.vblank_sync = true; 4930 wake_up(&rdev->irq.vblank_queue); 4931 } 4932 if (atomic_read(&rdev->irq.pflip[5])) 4933 radeon_crtc_handle_vblank(rdev, 5); 4934 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 4935 DRM_DEBUG("IH: D6 vblank\n"); 4936 } 4937 break; 4938 case 1: /* D6 vline */ 4939 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 4940 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 4941 DRM_DEBUG("IH: D6 vline\n"); 4942 } 4943 break; 4944 default: 4945 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 4946 break; 4947 } 4948 break; 4949 case 8: /* D1 page flip */ 4950 case 10: /* D2 page flip */ 4951 case 12: /* D3 page flip */ 4952 case 14: /* D4 page flip */ 4953 case 16: /* D5 page flip */ 4954 case 18: /* D6 page flip */ 4955 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); 4956 if (radeon_use_pflipirq > 0) 4957 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); 4958 break; 4959 case 42: /* HPD hotplug */ 4960 switch (src_data) { 4961 case 0: 4962 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 4963 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; 4964 queue_hotplug = true; 4965 DRM_DEBUG("IH: HPD1\n"); 4966 } 4967 break; 4968 case 1: 4969 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 4970 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; 4971 queue_hotplug = true; 4972 DRM_DEBUG("IH: HPD2\n"); 4973 } 4974 break; 4975 case 2: 4976 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 4977 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 4978 queue_hotplug = true; 4979 DRM_DEBUG("IH: HPD3\n"); 4980 } 4981 break; 4982 case 3: 4983 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 4984 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 4985 queue_hotplug = true; 4986 DRM_DEBUG("IH: HPD4\n"); 4987 } 4988 break; 4989 case 4: 4990 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 4991 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 4992 queue_hotplug = true; 4993 DRM_DEBUG("IH: HPD5\n"); 4994 } 4995 break; 4996 case 5: 4997 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 4998 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 4999 queue_hotplug = true; 5000 DRM_DEBUG("IH: HPD6\n"); 5001 } 5002 break; 5003 default: 5004 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5005 break; 5006 } 5007 break; 5008 case 44: /* hdmi */ 5009 switch (src_data) { 5010 case 0: 5011 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) { 5012 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG; 5013 queue_hdmi = true; 5014 DRM_DEBUG("IH: HDMI0\n"); 5015 } 5016 break; 5017 case 1: 5018 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) { 5019 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG; 5020 queue_hdmi = true; 5021 DRM_DEBUG("IH: HDMI1\n"); 5022 } 5023 break; 5024 case 2: 5025 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) { 5026 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG; 5027 queue_hdmi = true; 5028 DRM_DEBUG("IH: HDMI2\n"); 5029 } 5030 break; 5031 case 3: 5032 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) { 5033 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG; 5034 queue_hdmi = true; 5035 DRM_DEBUG("IH: HDMI3\n"); 5036 } 5037 break; 5038 case 4: 5039 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) { 5040 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG; 5041 queue_hdmi = true; 5042 DRM_DEBUG("IH: HDMI4\n"); 5043 } 5044 break; 5045 case 5: 5046 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) { 5047 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG; 5048 queue_hdmi = true; 5049 DRM_DEBUG("IH: HDMI5\n"); 5050 } 5051 break; 5052 default: 5053 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); 5054 break; 5055 } 5056 case 124: /* UVD */ 5057 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 5058 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 5059 break; 5060 case 146: 5061 case 147: 5062 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 5063 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 5064 /* reset addr and status */ 5065 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 5066 if (addr == 0x0 && status == 0x0) 5067 break; 5068 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 5069 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 5070 addr); 5071 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 5072 status); 5073 cayman_vm_decode_fault(rdev, status, addr); 5074 break; 5075 case 176: /* CP_INT in ring buffer */ 5076 case 177: /* CP_INT in IB1 */ 5077 case 178: /* CP_INT in IB2 */ 5078 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data); 5079 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 5080 break; 5081 case 181: /* CP EOP event */ 5082 DRM_DEBUG("IH: CP EOP\n"); 5083 if (rdev->family >= CHIP_CAYMAN) { 5084 switch (src_data) { 5085 case 0: 5086 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 5087 break; 5088 case 1: 5089 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 5090 break; 5091 case 2: 5092 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 5093 break; 5094 } 5095 } else 5096 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 5097 break; 5098 case 224: /* DMA trap event */ 5099 DRM_DEBUG("IH: DMA trap\n"); 5100 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 5101 break; 5102 case 230: /* thermal low to high */ 5103 DRM_DEBUG("IH: thermal low to high\n"); 5104 rdev->pm.dpm.thermal.high_to_low = false; 5105 queue_thermal = true; 5106 break; 5107 case 231: /* thermal high to low */ 5108 DRM_DEBUG("IH: thermal high to low\n"); 5109 rdev->pm.dpm.thermal.high_to_low = true; 5110 queue_thermal = true; 5111 break; 5112 case 233: /* GUI IDLE */ 5113 DRM_DEBUG("IH: GUI idle\n"); 5114 break; 5115 case 244: /* DMA trap event */ 5116 if (rdev->family >= CHIP_CAYMAN) { 5117 DRM_DEBUG("IH: DMA1 trap\n"); 5118 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 5119 } 5120 break; 5121 default: 5122 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5123 break; 5124 } 5125 5126 /* wptr/rptr are in bytes! */ 5127 rptr += 16; 5128 rptr &= rdev->ih.ptr_mask; 5129 WREG32(IH_RB_RPTR, rptr); 5130 } 5131 if (queue_hotplug) 5132 taskqueue_enqueue(rdev->tq, &rdev->hotplug_work); 5133 if (queue_hdmi) 5134 taskqueue_enqueue(rdev->tq, &rdev->audio_work); 5135 if (queue_thermal && rdev->pm.dpm_enabled) 5136 taskqueue_enqueue(rdev->tq, &rdev->pm.dpm.thermal.work); 5137 rdev->ih.rptr = rptr; 5138 atomic_set(&rdev->ih.lock, 0); 5139 5140 /* make sure wptr hasn't changed while processing */ 5141 wptr = evergreen_get_ih_wptr(rdev); 5142 if (wptr != rptr) 5143 goto restart_ih; 5144 5145 return IRQ_HANDLED; 5146 } 5147 5148 static int evergreen_startup(struct radeon_device *rdev) 5149 { 5150 struct radeon_ring *ring; 5151 int r; 5152 5153 /* enable pcie gen2 link */ 5154 evergreen_pcie_gen2_enable(rdev); 5155 /* enable aspm */ 5156 evergreen_program_aspm(rdev); 5157 5158 /* scratch needs to be initialized before MC */ 5159 r = r600_vram_scratch_init(rdev); 5160 if (r) 5161 return r; 5162 5163 evergreen_mc_program(rdev); 5164 5165 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) { 5166 r = ni_mc_load_microcode(rdev); 5167 if (r) { 5168 DRM_ERROR("Failed to load MC firmware!\n"); 5169 return r; 5170 } 5171 } 5172 5173 if (rdev->flags & RADEON_IS_AGP) { 5174 evergreen_agp_enable(rdev); 5175 } else { 5176 r = evergreen_pcie_gart_enable(rdev); 5177 if (r) 5178 return r; 5179 } 5180 evergreen_gpu_init(rdev); 5181 5182 /* allocate rlc buffers */ 5183 if (rdev->flags & RADEON_IS_IGP) { 5184 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list; 5185 rdev->rlc.reg_list_size = 5186 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list); 5187 rdev->rlc.cs_data = evergreen_cs_data; 5188 r = sumo_rlc_init(rdev); 5189 if (r) { 5190 DRM_ERROR("Failed to init rlc BOs!\n"); 5191 return r; 5192 } 5193 } 5194 5195 /* allocate wb buffer */ 5196 r = radeon_wb_init(rdev); 5197 if (r) 5198 return r; 5199 5200 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 5201 if (r) { 5202 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 5203 return r; 5204 } 5205 5206 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 5207 if (r) { 5208 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 5209 return r; 5210 } 5211 5212 r = uvd_v2_2_resume(rdev); 5213 if (!r) { 5214 r = radeon_fence_driver_start_ring(rdev, 5215 R600_RING_TYPE_UVD_INDEX); 5216 if (r) 5217 dev_err(rdev->dev, "UVD fences init error (%d).\n", r); 5218 } 5219 5220 if (r) 5221 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 5222 5223 /* Enable IRQ */ 5224 if (!rdev->irq.installed) { 5225 r = radeon_irq_kms_init(rdev); 5226 if (r) 5227 return r; 5228 } 5229 5230 r = r600_irq_init(rdev); 5231 if (r) { 5232 DRM_ERROR("radeon: IH init failed (%d).\n", r); 5233 radeon_irq_kms_fini(rdev); 5234 return r; 5235 } 5236 evergreen_irq_set(rdev); 5237 5238 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 5239 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 5240 RADEON_CP_PACKET2); 5241 if (r) 5242 return r; 5243 5244 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 5245 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 5246 DMA_PACKET(DMA_PACKET_NOP, 0, 0)); 5247 if (r) 5248 return r; 5249 5250 r = evergreen_cp_load_microcode(rdev); 5251 if (r) 5252 return r; 5253 r = evergreen_cp_resume(rdev); 5254 if (r) 5255 return r; 5256 r = r600_dma_resume(rdev); 5257 if (r) 5258 return r; 5259 5260 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 5261 if (ring->ring_size) { 5262 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 5263 RADEON_CP_PACKET2); 5264 if (!r) 5265 r = uvd_v1_0_init(rdev); 5266 5267 if (r) 5268 DRM_ERROR("radeon: error initializing UVD (%d).\n", r); 5269 } 5270 5271 r = radeon_ib_pool_init(rdev); 5272 if (r) { 5273 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 5274 return r; 5275 } 5276 5277 r = r600_audio_init(rdev); 5278 if (r) { 5279 DRM_ERROR("radeon: audio init failed\n"); 5280 return r; 5281 } 5282 5283 return 0; 5284 } 5285 5286 int evergreen_resume(struct radeon_device *rdev) 5287 { 5288 int r; 5289 5290 /* reset the asic, the gfx blocks are often in a bad state 5291 * after the driver is unloaded or after a resume 5292 */ 5293 if (radeon_asic_reset(rdev)) 5294 dev_warn(rdev->dev, "GPU reset failed !\n"); 5295 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw, 5296 * posting will perform necessary task to bring back GPU into good 5297 * shape. 5298 */ 5299 /* post card */ 5300 atom_asic_init(rdev->mode_info.atom_context); 5301 5302 /* init golden registers */ 5303 evergreen_init_golden_registers(rdev); 5304 5305 if (rdev->pm.pm_method == PM_METHOD_DPM) 5306 radeon_pm_resume(rdev); 5307 5308 rdev->accel_working = true; 5309 r = evergreen_startup(rdev); 5310 if (r) { 5311 DRM_ERROR("evergreen startup failed on resume\n"); 5312 rdev->accel_working = false; 5313 return r; 5314 } 5315 5316 return r; 5317 5318 } 5319 5320 int evergreen_suspend(struct radeon_device *rdev) 5321 { 5322 radeon_pm_suspend(rdev); 5323 r600_audio_fini(rdev); 5324 uvd_v1_0_fini(rdev); 5325 radeon_uvd_suspend(rdev); 5326 r700_cp_stop(rdev); 5327 r600_dma_stop(rdev); 5328 evergreen_irq_suspend(rdev); 5329 radeon_wb_disable(rdev); 5330 evergreen_pcie_gart_disable(rdev); 5331 5332 return 0; 5333 } 5334 5335 /* Plan is to move initialization in that function and use 5336 * helper function so that radeon_device_init pretty much 5337 * do nothing more than calling asic specific function. This 5338 * should also allow to remove a bunch of callback function 5339 * like vram_info. 5340 */ 5341 int evergreen_init(struct radeon_device *rdev) 5342 { 5343 int r; 5344 5345 /* Read BIOS */ 5346 if (!radeon_get_bios(rdev)) { 5347 if (ASIC_IS_AVIVO(rdev)) 5348 return -EINVAL; 5349 } 5350 /* Must be an ATOMBIOS */ 5351 if (!rdev->is_atom_bios) { 5352 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n"); 5353 return -EINVAL; 5354 } 5355 r = radeon_atombios_init(rdev); 5356 if (r) 5357 return r; 5358 /* reset the asic, the gfx blocks are often in a bad state 5359 * after the driver is unloaded or after a resume 5360 */ 5361 if (radeon_asic_reset(rdev)) 5362 dev_warn(rdev->dev, "GPU reset failed !\n"); 5363 /* Post card if necessary */ 5364 if (!radeon_card_posted(rdev)) { 5365 if (!rdev->bios) { 5366 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 5367 return -EINVAL; 5368 } 5369 DRM_INFO("GPU not posted. posting now...\n"); 5370 atom_asic_init(rdev->mode_info.atom_context); 5371 } 5372 /* init golden registers */ 5373 evergreen_init_golden_registers(rdev); 5374 /* Initialize scratch registers */ 5375 r600_scratch_init(rdev); 5376 /* Initialize surface registers */ 5377 radeon_surface_init(rdev); 5378 /* Initialize clocks */ 5379 radeon_get_clock_info(rdev->ddev); 5380 /* Fence driver */ 5381 r = radeon_fence_driver_init(rdev); 5382 if (r) 5383 return r; 5384 /* initialize AGP */ 5385 if (rdev->flags & RADEON_IS_AGP) { 5386 r = radeon_agp_init(rdev); 5387 if (r) 5388 radeon_agp_disable(rdev); 5389 } 5390 /* initialize memory controller */ 5391 r = evergreen_mc_init(rdev); 5392 if (r) 5393 return r; 5394 /* Memory manager */ 5395 r = radeon_bo_init(rdev); 5396 if (r) 5397 return r; 5398 5399 if (ASIC_IS_DCE5(rdev)) { 5400 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { 5401 r = ni_init_microcode(rdev); 5402 if (r) { 5403 DRM_ERROR("Failed to load firmware!\n"); 5404 return r; 5405 } 5406 } 5407 } else { 5408 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { 5409 r = r600_init_microcode(rdev); 5410 if (r) { 5411 DRM_ERROR("Failed to load firmware!\n"); 5412 return r; 5413 } 5414 } 5415 } 5416 5417 /* Initialize power management */ 5418 radeon_pm_init(rdev); 5419 5420 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL; 5421 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024); 5422 5423 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL; 5424 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024); 5425 5426 r = radeon_uvd_init(rdev); 5427 if (!r) { 5428 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL; 5429 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 5430 4096); 5431 } 5432 5433 rdev->ih.ring_obj = NULL; 5434 r600_ih_ring_init(rdev, 64 * 1024); 5435 5436 r = r600_pcie_gart_init(rdev); 5437 if (r) 5438 return r; 5439 5440 rdev->accel_working = true; 5441 r = evergreen_startup(rdev); 5442 if (r) { 5443 dev_err(rdev->dev, "disabling GPU acceleration\n"); 5444 r700_cp_fini(rdev); 5445 r600_dma_fini(rdev); 5446 r600_irq_fini(rdev); 5447 if (rdev->flags & RADEON_IS_IGP) 5448 sumo_rlc_fini(rdev); 5449 radeon_wb_fini(rdev); 5450 radeon_ib_pool_fini(rdev); 5451 radeon_irq_kms_fini(rdev); 5452 evergreen_pcie_gart_fini(rdev); 5453 rdev->accel_working = false; 5454 } 5455 5456 /* Don't start up if the MC ucode is missing on BTC parts. 5457 * The default clocks and voltages before the MC ucode 5458 * is loaded are not suffient for advanced operations. 5459 */ 5460 if (ASIC_IS_DCE5(rdev)) { 5461 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 5462 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 5463 return -EINVAL; 5464 } 5465 } 5466 5467 return 0; 5468 } 5469 5470 void evergreen_fini(struct radeon_device *rdev) 5471 { 5472 radeon_pm_fini(rdev); 5473 r600_audio_fini(rdev); 5474 r700_cp_fini(rdev); 5475 r600_dma_fini(rdev); 5476 r600_irq_fini(rdev); 5477 if (rdev->flags & RADEON_IS_IGP) 5478 sumo_rlc_fini(rdev); 5479 radeon_wb_fini(rdev); 5480 radeon_ib_pool_fini(rdev); 5481 radeon_irq_kms_fini(rdev); 5482 uvd_v1_0_fini(rdev); 5483 radeon_uvd_fini(rdev); 5484 evergreen_pcie_gart_fini(rdev); 5485 r600_vram_scratch_fini(rdev); 5486 radeon_gem_fini(rdev); 5487 radeon_fence_driver_fini(rdev); 5488 radeon_agp_fini(rdev); 5489 radeon_bo_fini(rdev); 5490 radeon_atombios_fini(rdev); 5491 if (ASIC_IS_DCE5(rdev)) 5492 ni_fini_microcode(rdev); 5493 else 5494 r600_fini_microcode(rdev); 5495 kfree(rdev->bios); 5496 rdev->bios = NULL; 5497 } 5498 5499 void evergreen_pcie_gen2_enable(struct radeon_device *rdev) 5500 { 5501 u32 link_width_cntl, speed_cntl, mask; 5502 int ret; 5503 5504 if (radeon_pcie_gen2 == 0) 5505 return; 5506 5507 if (rdev->flags & RADEON_IS_IGP) 5508 return; 5509 5510 if (!(rdev->flags & RADEON_IS_PCIE)) 5511 return; 5512 5513 /* x2 cards have a special sequence */ 5514 if (ASIC_IS_X2(rdev)) 5515 return; 5516 5517 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); 5518 if (ret != 0) 5519 return; 5520 5521 if (!(mask & DRM_PCIE_SPEED_50)) 5522 return; 5523 5524 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5525 if (speed_cntl & LC_CURRENT_DATA_RATE) { 5526 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 5527 return; 5528 } 5529 5530 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 5531 5532 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) || 5533 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) { 5534 5535 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5536 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 5537 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 5538 5539 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5540 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN; 5541 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5542 5543 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5544 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT; 5545 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5546 5547 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5548 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT; 5549 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5550 5551 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5552 speed_cntl |= LC_GEN2_EN_STRAP; 5553 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5554 5555 } else { 5556 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5557 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */ 5558 if (1) 5559 link_width_cntl |= LC_UPCONFIGURE_DIS; 5560 else 5561 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 5562 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 5563 } 5564 } 5565 5566 void evergreen_program_aspm(struct radeon_device *rdev) 5567 { 5568 u32 data, orig; 5569 u32 pcie_lc_cntl, pcie_lc_cntl_old; 5570 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false; 5571 /* fusion_platform = true 5572 * if the system is a fusion system 5573 * (APU or DGPU in a fusion system). 5574 * todo: check if the system is a fusion platform. 5575 */ 5576 bool fusion_platform = false; 5577 5578 if (radeon_aspm == 0) 5579 return; 5580 5581 if (!(rdev->flags & RADEON_IS_PCIE)) 5582 return; 5583 5584 switch (rdev->family) { 5585 case CHIP_CYPRESS: 5586 case CHIP_HEMLOCK: 5587 case CHIP_JUNIPER: 5588 case CHIP_REDWOOD: 5589 case CHIP_CEDAR: 5590 case CHIP_SUMO: 5591 case CHIP_SUMO2: 5592 case CHIP_PALM: 5593 case CHIP_ARUBA: 5594 disable_l0s = true; 5595 break; 5596 default: 5597 disable_l0s = false; 5598 break; 5599 } 5600 5601 if (rdev->flags & RADEON_IS_IGP) 5602 fusion_platform = true; /* XXX also dGPUs in a fusion system */ 5603 5604 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING); 5605 if (fusion_platform) 5606 data &= ~MULTI_PIF; 5607 else 5608 data |= MULTI_PIF; 5609 if (data != orig) 5610 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data); 5611 5612 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING); 5613 if (fusion_platform) 5614 data &= ~MULTI_PIF; 5615 else 5616 data |= MULTI_PIF; 5617 if (data != orig) 5618 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data); 5619 5620 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL); 5621 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 5622 if (!disable_l0s) { 5623 if (rdev->family >= CHIP_BARTS) 5624 pcie_lc_cntl |= LC_L0S_INACTIVITY(7); 5625 else 5626 pcie_lc_cntl |= LC_L0S_INACTIVITY(3); 5627 } 5628 5629 if (!disable_l1) { 5630 if (rdev->family >= CHIP_BARTS) 5631 pcie_lc_cntl |= LC_L1_INACTIVITY(7); 5632 else 5633 pcie_lc_cntl |= LC_L1_INACTIVITY(8); 5634 5635 if (!disable_plloff_in_l1) { 5636 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 5637 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 5638 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 5639 if (data != orig) 5640 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 5641 5642 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 5643 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 5644 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 5645 if (data != orig) 5646 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 5647 5648 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 5649 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 5650 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 5651 if (data != orig) 5652 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 5653 5654 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 5655 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 5656 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 5657 if (data != orig) 5658 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 5659 5660 if (rdev->family >= CHIP_BARTS) { 5661 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 5662 data &= ~PLL_RAMP_UP_TIME_0_MASK; 5663 data |= PLL_RAMP_UP_TIME_0(4); 5664 if (data != orig) 5665 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 5666 5667 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 5668 data &= ~PLL_RAMP_UP_TIME_1_MASK; 5669 data |= PLL_RAMP_UP_TIME_1(4); 5670 if (data != orig) 5671 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 5672 5673 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 5674 data &= ~PLL_RAMP_UP_TIME_0_MASK; 5675 data |= PLL_RAMP_UP_TIME_0(4); 5676 if (data != orig) 5677 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 5678 5679 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 5680 data &= ~PLL_RAMP_UP_TIME_1_MASK; 5681 data |= PLL_RAMP_UP_TIME_1(4); 5682 if (data != orig) 5683 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 5684 } 5685 5686 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5687 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 5688 data |= LC_DYN_LANES_PWR_STATE(3); 5689 if (data != orig) 5690 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 5691 5692 if (rdev->family >= CHIP_BARTS) { 5693 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL); 5694 data &= ~LS2_EXIT_TIME_MASK; 5695 data |= LS2_EXIT_TIME(1); 5696 if (data != orig) 5697 WREG32_PIF_PHY0(PB0_PIF_CNTL, data); 5698 5699 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL); 5700 data &= ~LS2_EXIT_TIME_MASK; 5701 data |= LS2_EXIT_TIME(1); 5702 if (data != orig) 5703 WREG32_PIF_PHY1(PB1_PIF_CNTL, data); 5704 } 5705 } 5706 } 5707 5708 /* evergreen parts only */ 5709 if (rdev->family < CHIP_BARTS) 5710 pcie_lc_cntl |= LC_PMI_TO_L1_DIS; 5711 5712 if (pcie_lc_cntl != pcie_lc_cntl_old) 5713 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl); 5714 } 5715