1 /* 2 * Copyright 2011 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 * 24 * $FreeBSD: head/sys/dev/drm2/radeon/si.c 254885 2013-08-25 19:37:15Z dumbbell $ 25 */ 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <drm/drmP.h> 29 #include "radeon.h" 30 #include "radeon_asic.h" 31 #include <uapi_drm/radeon_drm.h> 32 #include "sid.h" 33 #include "atom.h" 34 #include "si_blit_shaders.h" 35 #include "clearstate_si.h" 36 #include "radeon_ucode.h" 37 38 39 #define PCI_EXP_LNKCTL PCIER_LINKCTRL /* 16 */ 40 #define PCI_EXP_LNKCTL2 48 41 #define PCI_EXP_LNKCTL_HAWD PCIEM_LNKCTL_HAWD /* 0x0200 */ 42 #define PCI_EXP_DEVSTA PCIER_DEVSTS /* 10 */ 43 #define PCI_EXP_DEVSTA_TRPND 0x0020 44 #define PCI_EXP_LNKCAP_CLKPM 0x00040000 45 46 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin"); 47 MODULE_FIRMWARE("radeon/TAHITI_me.bin"); 48 MODULE_FIRMWARE("radeon/TAHITI_ce.bin"); 49 MODULE_FIRMWARE("radeon/TAHITI_mc.bin"); 50 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin"); 51 MODULE_FIRMWARE("radeon/TAHITI_smc.bin"); 52 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin"); 53 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin"); 54 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin"); 55 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin"); 56 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin"); 57 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin"); 58 MODULE_FIRMWARE("radeon/VERDE_pfp.bin"); 59 MODULE_FIRMWARE("radeon/VERDE_me.bin"); 60 MODULE_FIRMWARE("radeon/VERDE_ce.bin"); 61 MODULE_FIRMWARE("radeon/VERDE_mc.bin"); 62 MODULE_FIRMWARE("radeon/VERDE_rlc.bin"); 63 MODULE_FIRMWARE("radeon/VERDE_smc.bin"); 64 MODULE_FIRMWARE("radeon/OLAND_pfp.bin"); 65 MODULE_FIRMWARE("radeon/OLAND_me.bin"); 66 MODULE_FIRMWARE("radeon/OLAND_ce.bin"); 67 MODULE_FIRMWARE("radeon/OLAND_mc.bin"); 68 MODULE_FIRMWARE("radeon/OLAND_rlc.bin"); 69 MODULE_FIRMWARE("radeon/OLAND_smc.bin"); 70 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin"); 71 MODULE_FIRMWARE("radeon/HAINAN_me.bin"); 72 MODULE_FIRMWARE("radeon/HAINAN_ce.bin"); 73 MODULE_FIRMWARE("radeon/HAINAN_mc.bin"); 74 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin"); 75 MODULE_FIRMWARE("radeon/HAINAN_smc.bin"); 76 77 static void si_pcie_gen3_enable(struct radeon_device *rdev); 78 static void si_program_aspm(struct radeon_device *rdev); 79 80 static const u32 verde_rlc_save_restore_register_list[] = 81 { 82 (0x8000 << 16) | (0x98f4 >> 2), 83 0x00000000, 84 (0x8040 << 16) | (0x98f4 >> 2), 85 0x00000000, 86 (0x8000 << 16) | (0xe80 >> 2), 87 0x00000000, 88 (0x8040 << 16) | (0xe80 >> 2), 89 0x00000000, 90 (0x8000 << 16) | (0x89bc >> 2), 91 0x00000000, 92 (0x8040 << 16) | (0x89bc >> 2), 93 0x00000000, 94 (0x8000 << 16) | (0x8c1c >> 2), 95 0x00000000, 96 (0x8040 << 16) | (0x8c1c >> 2), 97 0x00000000, 98 (0x9c00 << 16) | (0x98f0 >> 2), 99 0x00000000, 100 (0x9c00 << 16) | (0xe7c >> 2), 101 0x00000000, 102 (0x8000 << 16) | (0x9148 >> 2), 103 0x00000000, 104 (0x8040 << 16) | (0x9148 >> 2), 105 0x00000000, 106 (0x9c00 << 16) | (0x9150 >> 2), 107 0x00000000, 108 (0x9c00 << 16) | (0x897c >> 2), 109 0x00000000, 110 (0x9c00 << 16) | (0x8d8c >> 2), 111 0x00000000, 112 (0x9c00 << 16) | (0xac54 >> 2), 113 0X00000000, 114 0x3, 115 (0x9c00 << 16) | (0x98f8 >> 2), 116 0x00000000, 117 (0x9c00 << 16) | (0x9910 >> 2), 118 0x00000000, 119 (0x9c00 << 16) | (0x9914 >> 2), 120 0x00000000, 121 (0x9c00 << 16) | (0x9918 >> 2), 122 0x00000000, 123 (0x9c00 << 16) | (0x991c >> 2), 124 0x00000000, 125 (0x9c00 << 16) | (0x9920 >> 2), 126 0x00000000, 127 (0x9c00 << 16) | (0x9924 >> 2), 128 0x00000000, 129 (0x9c00 << 16) | (0x9928 >> 2), 130 0x00000000, 131 (0x9c00 << 16) | (0x992c >> 2), 132 0x00000000, 133 (0x9c00 << 16) | (0x9930 >> 2), 134 0x00000000, 135 (0x9c00 << 16) | (0x9934 >> 2), 136 0x00000000, 137 (0x9c00 << 16) | (0x9938 >> 2), 138 0x00000000, 139 (0x9c00 << 16) | (0x993c >> 2), 140 0x00000000, 141 (0x9c00 << 16) | (0x9940 >> 2), 142 0x00000000, 143 (0x9c00 << 16) | (0x9944 >> 2), 144 0x00000000, 145 (0x9c00 << 16) | (0x9948 >> 2), 146 0x00000000, 147 (0x9c00 << 16) | (0x994c >> 2), 148 0x00000000, 149 (0x9c00 << 16) | (0x9950 >> 2), 150 0x00000000, 151 (0x9c00 << 16) | (0x9954 >> 2), 152 0x00000000, 153 (0x9c00 << 16) | (0x9958 >> 2), 154 0x00000000, 155 (0x9c00 << 16) | (0x995c >> 2), 156 0x00000000, 157 (0x9c00 << 16) | (0x9960 >> 2), 158 0x00000000, 159 (0x9c00 << 16) | (0x9964 >> 2), 160 0x00000000, 161 (0x9c00 << 16) | (0x9968 >> 2), 162 0x00000000, 163 (0x9c00 << 16) | (0x996c >> 2), 164 0x00000000, 165 (0x9c00 << 16) | (0x9970 >> 2), 166 0x00000000, 167 (0x9c00 << 16) | (0x9974 >> 2), 168 0x00000000, 169 (0x9c00 << 16) | (0x9978 >> 2), 170 0x00000000, 171 (0x9c00 << 16) | (0x997c >> 2), 172 0x00000000, 173 (0x9c00 << 16) | (0x9980 >> 2), 174 0x00000000, 175 (0x9c00 << 16) | (0x9984 >> 2), 176 0x00000000, 177 (0x9c00 << 16) | (0x9988 >> 2), 178 0x00000000, 179 (0x9c00 << 16) | (0x998c >> 2), 180 0x00000000, 181 (0x9c00 << 16) | (0x8c00 >> 2), 182 0x00000000, 183 (0x9c00 << 16) | (0x8c14 >> 2), 184 0x00000000, 185 (0x9c00 << 16) | (0x8c04 >> 2), 186 0x00000000, 187 (0x9c00 << 16) | (0x8c08 >> 2), 188 0x00000000, 189 (0x8000 << 16) | (0x9b7c >> 2), 190 0x00000000, 191 (0x8040 << 16) | (0x9b7c >> 2), 192 0x00000000, 193 (0x8000 << 16) | (0xe84 >> 2), 194 0x00000000, 195 (0x8040 << 16) | (0xe84 >> 2), 196 0x00000000, 197 (0x8000 << 16) | (0x89c0 >> 2), 198 0x00000000, 199 (0x8040 << 16) | (0x89c0 >> 2), 200 0x00000000, 201 (0x8000 << 16) | (0x914c >> 2), 202 0x00000000, 203 (0x8040 << 16) | (0x914c >> 2), 204 0x00000000, 205 (0x8000 << 16) | (0x8c20 >> 2), 206 0x00000000, 207 (0x8040 << 16) | (0x8c20 >> 2), 208 0x00000000, 209 (0x8000 << 16) | (0x9354 >> 2), 210 0x00000000, 211 (0x8040 << 16) | (0x9354 >> 2), 212 0x00000000, 213 (0x9c00 << 16) | (0x9060 >> 2), 214 0x00000000, 215 (0x9c00 << 16) | (0x9364 >> 2), 216 0x00000000, 217 (0x9c00 << 16) | (0x9100 >> 2), 218 0x00000000, 219 (0x9c00 << 16) | (0x913c >> 2), 220 0x00000000, 221 (0x8000 << 16) | (0x90e0 >> 2), 222 0x00000000, 223 (0x8000 << 16) | (0x90e4 >> 2), 224 0x00000000, 225 (0x8000 << 16) | (0x90e8 >> 2), 226 0x00000000, 227 (0x8040 << 16) | (0x90e0 >> 2), 228 0x00000000, 229 (0x8040 << 16) | (0x90e4 >> 2), 230 0x00000000, 231 (0x8040 << 16) | (0x90e8 >> 2), 232 0x00000000, 233 (0x9c00 << 16) | (0x8bcc >> 2), 234 0x00000000, 235 (0x9c00 << 16) | (0x8b24 >> 2), 236 0x00000000, 237 (0x9c00 << 16) | (0x88c4 >> 2), 238 0x00000000, 239 (0x9c00 << 16) | (0x8e50 >> 2), 240 0x00000000, 241 (0x9c00 << 16) | (0x8c0c >> 2), 242 0x00000000, 243 (0x9c00 << 16) | (0x8e58 >> 2), 244 0x00000000, 245 (0x9c00 << 16) | (0x8e5c >> 2), 246 0x00000000, 247 (0x9c00 << 16) | (0x9508 >> 2), 248 0x00000000, 249 (0x9c00 << 16) | (0x950c >> 2), 250 0x00000000, 251 (0x9c00 << 16) | (0x9494 >> 2), 252 0x00000000, 253 (0x9c00 << 16) | (0xac0c >> 2), 254 0x00000000, 255 (0x9c00 << 16) | (0xac10 >> 2), 256 0x00000000, 257 (0x9c00 << 16) | (0xac14 >> 2), 258 0x00000000, 259 (0x9c00 << 16) | (0xae00 >> 2), 260 0x00000000, 261 (0x9c00 << 16) | (0xac08 >> 2), 262 0x00000000, 263 (0x9c00 << 16) | (0x88d4 >> 2), 264 0x00000000, 265 (0x9c00 << 16) | (0x88c8 >> 2), 266 0x00000000, 267 (0x9c00 << 16) | (0x88cc >> 2), 268 0x00000000, 269 (0x9c00 << 16) | (0x89b0 >> 2), 270 0x00000000, 271 (0x9c00 << 16) | (0x8b10 >> 2), 272 0x00000000, 273 (0x9c00 << 16) | (0x8a14 >> 2), 274 0x00000000, 275 (0x9c00 << 16) | (0x9830 >> 2), 276 0x00000000, 277 (0x9c00 << 16) | (0x9834 >> 2), 278 0x00000000, 279 (0x9c00 << 16) | (0x9838 >> 2), 280 0x00000000, 281 (0x9c00 << 16) | (0x9a10 >> 2), 282 0x00000000, 283 (0x8000 << 16) | (0x9870 >> 2), 284 0x00000000, 285 (0x8000 << 16) | (0x9874 >> 2), 286 0x00000000, 287 (0x8001 << 16) | (0x9870 >> 2), 288 0x00000000, 289 (0x8001 << 16) | (0x9874 >> 2), 290 0x00000000, 291 (0x8040 << 16) | (0x9870 >> 2), 292 0x00000000, 293 (0x8040 << 16) | (0x9874 >> 2), 294 0x00000000, 295 (0x8041 << 16) | (0x9870 >> 2), 296 0x00000000, 297 (0x8041 << 16) | (0x9874 >> 2), 298 0x00000000, 299 0x00000000 300 }; 301 302 static const u32 tahiti_golden_rlc_registers[] = 303 { 304 0xc424, 0xffffffff, 0x00601005, 305 0xc47c, 0xffffffff, 0x10104040, 306 0xc488, 0xffffffff, 0x0100000a, 307 0xc314, 0xffffffff, 0x00000800, 308 0xc30c, 0xffffffff, 0x800000f4, 309 0xf4a8, 0xffffffff, 0x00000000 310 }; 311 312 static const u32 tahiti_golden_registers[] = 313 { 314 0x9a10, 0x00010000, 0x00018208, 315 0x9830, 0xffffffff, 0x00000000, 316 0x9834, 0xf00fffff, 0x00000400, 317 0x9838, 0x0002021c, 0x00020200, 318 0xc78, 0x00000080, 0x00000000, 319 0xd030, 0x000300c0, 0x00800040, 320 0xd830, 0x000300c0, 0x00800040, 321 0x5bb0, 0x000000f0, 0x00000070, 322 0x5bc0, 0x00200000, 0x50100000, 323 0x7030, 0x31000311, 0x00000011, 324 0x277c, 0x00000003, 0x000007ff, 325 0x240c, 0x000007ff, 0x00000000, 326 0x8a14, 0xf000001f, 0x00000007, 327 0x8b24, 0xffffffff, 0x00ffffff, 328 0x8b10, 0x0000ff0f, 0x00000000, 329 0x28a4c, 0x07ffffff, 0x4e000000, 330 0x28350, 0x3f3f3fff, 0x2a00126a, 331 0x30, 0x000000ff, 0x0040, 332 0x34, 0x00000040, 0x00004040, 333 0x9100, 0x07ffffff, 0x03000000, 334 0x8e88, 0x01ff1f3f, 0x00000000, 335 0x8e84, 0x01ff1f3f, 0x00000000, 336 0x9060, 0x0000007f, 0x00000020, 337 0x9508, 0x00010000, 0x00010000, 338 0xac14, 0x00000200, 0x000002fb, 339 0xac10, 0xffffffff, 0x0000543b, 340 0xac0c, 0xffffffff, 0xa9210876, 341 0x88d0, 0xffffffff, 0x000fff40, 342 0x88d4, 0x0000001f, 0x00000010, 343 0x1410, 0x20000000, 0x20fffed8, 344 0x15c0, 0x000c0fc0, 0x000c0400 345 }; 346 347 static const u32 tahiti_golden_registers2[] = 348 { 349 0xc64, 0x00000001, 0x00000001 350 }; 351 352 static const u32 pitcairn_golden_rlc_registers[] = 353 { 354 0xc424, 0xffffffff, 0x00601004, 355 0xc47c, 0xffffffff, 0x10102020, 356 0xc488, 0xffffffff, 0x01000020, 357 0xc314, 0xffffffff, 0x00000800, 358 0xc30c, 0xffffffff, 0x800000a4 359 }; 360 361 static const u32 pitcairn_golden_registers[] = 362 { 363 0x9a10, 0x00010000, 0x00018208, 364 0x9830, 0xffffffff, 0x00000000, 365 0x9834, 0xf00fffff, 0x00000400, 366 0x9838, 0x0002021c, 0x00020200, 367 0xc78, 0x00000080, 0x00000000, 368 0xd030, 0x000300c0, 0x00800040, 369 0xd830, 0x000300c0, 0x00800040, 370 0x5bb0, 0x000000f0, 0x00000070, 371 0x5bc0, 0x00200000, 0x50100000, 372 0x7030, 0x31000311, 0x00000011, 373 0x2ae4, 0x00073ffe, 0x000022a2, 374 0x240c, 0x000007ff, 0x00000000, 375 0x8a14, 0xf000001f, 0x00000007, 376 0x8b24, 0xffffffff, 0x00ffffff, 377 0x8b10, 0x0000ff0f, 0x00000000, 378 0x28a4c, 0x07ffffff, 0x4e000000, 379 0x28350, 0x3f3f3fff, 0x2a00126a, 380 0x30, 0x000000ff, 0x0040, 381 0x34, 0x00000040, 0x00004040, 382 0x9100, 0x07ffffff, 0x03000000, 383 0x9060, 0x0000007f, 0x00000020, 384 0x9508, 0x00010000, 0x00010000, 385 0xac14, 0x000003ff, 0x000000f7, 386 0xac10, 0xffffffff, 0x00000000, 387 0xac0c, 0xffffffff, 0x32761054, 388 0x88d4, 0x0000001f, 0x00000010, 389 0x15c0, 0x000c0fc0, 0x000c0400 390 }; 391 392 static const u32 verde_golden_rlc_registers[] = 393 { 394 0xc424, 0xffffffff, 0x033f1005, 395 0xc47c, 0xffffffff, 0x10808020, 396 0xc488, 0xffffffff, 0x00800008, 397 0xc314, 0xffffffff, 0x00001000, 398 0xc30c, 0xffffffff, 0x80010014 399 }; 400 401 static const u32 verde_golden_registers[] = 402 { 403 0x9a10, 0x00010000, 0x00018208, 404 0x9830, 0xffffffff, 0x00000000, 405 0x9834, 0xf00fffff, 0x00000400, 406 0x9838, 0x0002021c, 0x00020200, 407 0xc78, 0x00000080, 0x00000000, 408 0xd030, 0x000300c0, 0x00800040, 409 0xd030, 0x000300c0, 0x00800040, 410 0xd830, 0x000300c0, 0x00800040, 411 0xd830, 0x000300c0, 0x00800040, 412 0x5bb0, 0x000000f0, 0x00000070, 413 0x5bc0, 0x00200000, 0x50100000, 414 0x7030, 0x31000311, 0x00000011, 415 0x2ae4, 0x00073ffe, 0x000022a2, 416 0x2ae4, 0x00073ffe, 0x000022a2, 417 0x2ae4, 0x00073ffe, 0x000022a2, 418 0x240c, 0x000007ff, 0x00000000, 419 0x240c, 0x000007ff, 0x00000000, 420 0x240c, 0x000007ff, 0x00000000, 421 0x8a14, 0xf000001f, 0x00000007, 422 0x8a14, 0xf000001f, 0x00000007, 423 0x8a14, 0xf000001f, 0x00000007, 424 0x8b24, 0xffffffff, 0x00ffffff, 425 0x8b10, 0x0000ff0f, 0x00000000, 426 0x28a4c, 0x07ffffff, 0x4e000000, 427 0x28350, 0x3f3f3fff, 0x0000124a, 428 0x28350, 0x3f3f3fff, 0x0000124a, 429 0x28350, 0x3f3f3fff, 0x0000124a, 430 0x30, 0x000000ff, 0x0040, 431 0x34, 0x00000040, 0x00004040, 432 0x9100, 0x07ffffff, 0x03000000, 433 0x9100, 0x07ffffff, 0x03000000, 434 0x8e88, 0x01ff1f3f, 0x00000000, 435 0x8e88, 0x01ff1f3f, 0x00000000, 436 0x8e88, 0x01ff1f3f, 0x00000000, 437 0x8e84, 0x01ff1f3f, 0x00000000, 438 0x8e84, 0x01ff1f3f, 0x00000000, 439 0x8e84, 0x01ff1f3f, 0x00000000, 440 0x9060, 0x0000007f, 0x00000020, 441 0x9508, 0x00010000, 0x00010000, 442 0xac14, 0x000003ff, 0x00000003, 443 0xac14, 0x000003ff, 0x00000003, 444 0xac14, 0x000003ff, 0x00000003, 445 0xac10, 0xffffffff, 0x00000000, 446 0xac10, 0xffffffff, 0x00000000, 447 0xac10, 0xffffffff, 0x00000000, 448 0xac0c, 0xffffffff, 0x00001032, 449 0xac0c, 0xffffffff, 0x00001032, 450 0xac0c, 0xffffffff, 0x00001032, 451 0x88d4, 0x0000001f, 0x00000010, 452 0x88d4, 0x0000001f, 0x00000010, 453 0x88d4, 0x0000001f, 0x00000010, 454 0x15c0, 0x000c0fc0, 0x000c0400 455 }; 456 457 static const u32 oland_golden_rlc_registers[] = 458 { 459 0xc424, 0xffffffff, 0x00601005, 460 0xc47c, 0xffffffff, 0x10104040, 461 0xc488, 0xffffffff, 0x0100000a, 462 0xc314, 0xffffffff, 0x00000800, 463 0xc30c, 0xffffffff, 0x800000f4 464 }; 465 466 static const u32 oland_golden_registers[] = 467 { 468 0x9a10, 0x00010000, 0x00018208, 469 0x9830, 0xffffffff, 0x00000000, 470 0x9834, 0xf00fffff, 0x00000400, 471 0x9838, 0x0002021c, 0x00020200, 472 0xc78, 0x00000080, 0x00000000, 473 0xd030, 0x000300c0, 0x00800040, 474 0xd830, 0x000300c0, 0x00800040, 475 0x5bb0, 0x000000f0, 0x00000070, 476 0x5bc0, 0x00200000, 0x50100000, 477 0x7030, 0x31000311, 0x00000011, 478 0x2ae4, 0x00073ffe, 0x000022a2, 479 0x240c, 0x000007ff, 0x00000000, 480 0x8a14, 0xf000001f, 0x00000007, 481 0x8b24, 0xffffffff, 0x00ffffff, 482 0x8b10, 0x0000ff0f, 0x00000000, 483 0x28a4c, 0x07ffffff, 0x4e000000, 484 0x28350, 0x3f3f3fff, 0x00000082, 485 0x30, 0x000000ff, 0x0040, 486 0x34, 0x00000040, 0x00004040, 487 0x9100, 0x07ffffff, 0x03000000, 488 0x9060, 0x0000007f, 0x00000020, 489 0x9508, 0x00010000, 0x00010000, 490 0xac14, 0x000003ff, 0x000000f3, 491 0xac10, 0xffffffff, 0x00000000, 492 0xac0c, 0xffffffff, 0x00003210, 493 0x88d4, 0x0000001f, 0x00000010, 494 0x15c0, 0x000c0fc0, 0x000c0400 495 }; 496 497 static const u32 hainan_golden_registers[] = 498 { 499 0x9a10, 0x00010000, 0x00018208, 500 0x9830, 0xffffffff, 0x00000000, 501 0x9834, 0xf00fffff, 0x00000400, 502 0x9838, 0x0002021c, 0x00020200, 503 0xd0c0, 0xff000fff, 0x00000100, 504 0xd030, 0x000300c0, 0x00800040, 505 0xd8c0, 0xff000fff, 0x00000100, 506 0xd830, 0x000300c0, 0x00800040, 507 0x2ae4, 0x00073ffe, 0x000022a2, 508 0x240c, 0x000007ff, 0x00000000, 509 0x8a14, 0xf000001f, 0x00000007, 510 0x8b24, 0xffffffff, 0x00ffffff, 511 0x8b10, 0x0000ff0f, 0x00000000, 512 0x28a4c, 0x07ffffff, 0x4e000000, 513 0x28350, 0x3f3f3fff, 0x00000000, 514 0x30, 0x000000ff, 0x0040, 515 0x34, 0x00000040, 0x00004040, 516 0x9100, 0x03e00000, 0x03600000, 517 0x9060, 0x0000007f, 0x00000020, 518 0x9508, 0x00010000, 0x00010000, 519 0xac14, 0x000003ff, 0x000000f1, 520 0xac10, 0xffffffff, 0x00000000, 521 0xac0c, 0xffffffff, 0x00003210, 522 0x88d4, 0x0000001f, 0x00000010, 523 0x15c0, 0x000c0fc0, 0x000c0400 524 }; 525 526 static const u32 hainan_golden_registers2[] = 527 { 528 0x98f8, 0xffffffff, 0x02010001 529 }; 530 531 static const u32 tahiti_mgcg_cgcg_init[] = 532 { 533 0xc400, 0xffffffff, 0xfffffffc, 534 0x802c, 0xffffffff, 0xe0000000, 535 0x9a60, 0xffffffff, 0x00000100, 536 0x92a4, 0xffffffff, 0x00000100, 537 0xc164, 0xffffffff, 0x00000100, 538 0x9774, 0xffffffff, 0x00000100, 539 0x8984, 0xffffffff, 0x06000100, 540 0x8a18, 0xffffffff, 0x00000100, 541 0x92a0, 0xffffffff, 0x00000100, 542 0xc380, 0xffffffff, 0x00000100, 543 0x8b28, 0xffffffff, 0x00000100, 544 0x9144, 0xffffffff, 0x00000100, 545 0x8d88, 0xffffffff, 0x00000100, 546 0x8d8c, 0xffffffff, 0x00000100, 547 0x9030, 0xffffffff, 0x00000100, 548 0x9034, 0xffffffff, 0x00000100, 549 0x9038, 0xffffffff, 0x00000100, 550 0x903c, 0xffffffff, 0x00000100, 551 0xad80, 0xffffffff, 0x00000100, 552 0xac54, 0xffffffff, 0x00000100, 553 0x897c, 0xffffffff, 0x06000100, 554 0x9868, 0xffffffff, 0x00000100, 555 0x9510, 0xffffffff, 0x00000100, 556 0xaf04, 0xffffffff, 0x00000100, 557 0xae04, 0xffffffff, 0x00000100, 558 0x949c, 0xffffffff, 0x00000100, 559 0x802c, 0xffffffff, 0xe0000000, 560 0x9160, 0xffffffff, 0x00010000, 561 0x9164, 0xffffffff, 0x00030002, 562 0x9168, 0xffffffff, 0x00040007, 563 0x916c, 0xffffffff, 0x00060005, 564 0x9170, 0xffffffff, 0x00090008, 565 0x9174, 0xffffffff, 0x00020001, 566 0x9178, 0xffffffff, 0x00040003, 567 0x917c, 0xffffffff, 0x00000007, 568 0x9180, 0xffffffff, 0x00060005, 569 0x9184, 0xffffffff, 0x00090008, 570 0x9188, 0xffffffff, 0x00030002, 571 0x918c, 0xffffffff, 0x00050004, 572 0x9190, 0xffffffff, 0x00000008, 573 0x9194, 0xffffffff, 0x00070006, 574 0x9198, 0xffffffff, 0x000a0009, 575 0x919c, 0xffffffff, 0x00040003, 576 0x91a0, 0xffffffff, 0x00060005, 577 0x91a4, 0xffffffff, 0x00000009, 578 0x91a8, 0xffffffff, 0x00080007, 579 0x91ac, 0xffffffff, 0x000b000a, 580 0x91b0, 0xffffffff, 0x00050004, 581 0x91b4, 0xffffffff, 0x00070006, 582 0x91b8, 0xffffffff, 0x0008000b, 583 0x91bc, 0xffffffff, 0x000a0009, 584 0x91c0, 0xffffffff, 0x000d000c, 585 0x91c4, 0xffffffff, 0x00060005, 586 0x91c8, 0xffffffff, 0x00080007, 587 0x91cc, 0xffffffff, 0x0000000b, 588 0x91d0, 0xffffffff, 0x000a0009, 589 0x91d4, 0xffffffff, 0x000d000c, 590 0x91d8, 0xffffffff, 0x00070006, 591 0x91dc, 0xffffffff, 0x00090008, 592 0x91e0, 0xffffffff, 0x0000000c, 593 0x91e4, 0xffffffff, 0x000b000a, 594 0x91e8, 0xffffffff, 0x000e000d, 595 0x91ec, 0xffffffff, 0x00080007, 596 0x91f0, 0xffffffff, 0x000a0009, 597 0x91f4, 0xffffffff, 0x0000000d, 598 0x91f8, 0xffffffff, 0x000c000b, 599 0x91fc, 0xffffffff, 0x000f000e, 600 0x9200, 0xffffffff, 0x00090008, 601 0x9204, 0xffffffff, 0x000b000a, 602 0x9208, 0xffffffff, 0x000c000f, 603 0x920c, 0xffffffff, 0x000e000d, 604 0x9210, 0xffffffff, 0x00110010, 605 0x9214, 0xffffffff, 0x000a0009, 606 0x9218, 0xffffffff, 0x000c000b, 607 0x921c, 0xffffffff, 0x0000000f, 608 0x9220, 0xffffffff, 0x000e000d, 609 0x9224, 0xffffffff, 0x00110010, 610 0x9228, 0xffffffff, 0x000b000a, 611 0x922c, 0xffffffff, 0x000d000c, 612 0x9230, 0xffffffff, 0x00000010, 613 0x9234, 0xffffffff, 0x000f000e, 614 0x9238, 0xffffffff, 0x00120011, 615 0x923c, 0xffffffff, 0x000c000b, 616 0x9240, 0xffffffff, 0x000e000d, 617 0x9244, 0xffffffff, 0x00000011, 618 0x9248, 0xffffffff, 0x0010000f, 619 0x924c, 0xffffffff, 0x00130012, 620 0x9250, 0xffffffff, 0x000d000c, 621 0x9254, 0xffffffff, 0x000f000e, 622 0x9258, 0xffffffff, 0x00100013, 623 0x925c, 0xffffffff, 0x00120011, 624 0x9260, 0xffffffff, 0x00150014, 625 0x9264, 0xffffffff, 0x000e000d, 626 0x9268, 0xffffffff, 0x0010000f, 627 0x926c, 0xffffffff, 0x00000013, 628 0x9270, 0xffffffff, 0x00120011, 629 0x9274, 0xffffffff, 0x00150014, 630 0x9278, 0xffffffff, 0x000f000e, 631 0x927c, 0xffffffff, 0x00110010, 632 0x9280, 0xffffffff, 0x00000014, 633 0x9284, 0xffffffff, 0x00130012, 634 0x9288, 0xffffffff, 0x00160015, 635 0x928c, 0xffffffff, 0x0010000f, 636 0x9290, 0xffffffff, 0x00120011, 637 0x9294, 0xffffffff, 0x00000015, 638 0x9298, 0xffffffff, 0x00140013, 639 0x929c, 0xffffffff, 0x00170016, 640 0x9150, 0xffffffff, 0x96940200, 641 0x8708, 0xffffffff, 0x00900100, 642 0xc478, 0xffffffff, 0x00000080, 643 0xc404, 0xffffffff, 0x0020003f, 644 0x30, 0xffffffff, 0x0000001c, 645 0x34, 0x000f0000, 0x000f0000, 646 0x160c, 0xffffffff, 0x00000100, 647 0x1024, 0xffffffff, 0x00000100, 648 0x102c, 0x00000101, 0x00000000, 649 0x20a8, 0xffffffff, 0x00000104, 650 0x264c, 0x000c0000, 0x000c0000, 651 0x2648, 0x000c0000, 0x000c0000, 652 0x55e4, 0xff000fff, 0x00000100, 653 0x55e8, 0x00000001, 0x00000001, 654 0x2f50, 0x00000001, 0x00000001, 655 0x30cc, 0xc0000fff, 0x00000104, 656 0xc1e4, 0x00000001, 0x00000001, 657 0xd0c0, 0xfffffff0, 0x00000100, 658 0xd8c0, 0xfffffff0, 0x00000100 659 }; 660 661 static const u32 pitcairn_mgcg_cgcg_init[] = 662 { 663 0xc400, 0xffffffff, 0xfffffffc, 664 0x802c, 0xffffffff, 0xe0000000, 665 0x9a60, 0xffffffff, 0x00000100, 666 0x92a4, 0xffffffff, 0x00000100, 667 0xc164, 0xffffffff, 0x00000100, 668 0x9774, 0xffffffff, 0x00000100, 669 0x8984, 0xffffffff, 0x06000100, 670 0x8a18, 0xffffffff, 0x00000100, 671 0x92a0, 0xffffffff, 0x00000100, 672 0xc380, 0xffffffff, 0x00000100, 673 0x8b28, 0xffffffff, 0x00000100, 674 0x9144, 0xffffffff, 0x00000100, 675 0x8d88, 0xffffffff, 0x00000100, 676 0x8d8c, 0xffffffff, 0x00000100, 677 0x9030, 0xffffffff, 0x00000100, 678 0x9034, 0xffffffff, 0x00000100, 679 0x9038, 0xffffffff, 0x00000100, 680 0x903c, 0xffffffff, 0x00000100, 681 0xad80, 0xffffffff, 0x00000100, 682 0xac54, 0xffffffff, 0x00000100, 683 0x897c, 0xffffffff, 0x06000100, 684 0x9868, 0xffffffff, 0x00000100, 685 0x9510, 0xffffffff, 0x00000100, 686 0xaf04, 0xffffffff, 0x00000100, 687 0xae04, 0xffffffff, 0x00000100, 688 0x949c, 0xffffffff, 0x00000100, 689 0x802c, 0xffffffff, 0xe0000000, 690 0x9160, 0xffffffff, 0x00010000, 691 0x9164, 0xffffffff, 0x00030002, 692 0x9168, 0xffffffff, 0x00040007, 693 0x916c, 0xffffffff, 0x00060005, 694 0x9170, 0xffffffff, 0x00090008, 695 0x9174, 0xffffffff, 0x00020001, 696 0x9178, 0xffffffff, 0x00040003, 697 0x917c, 0xffffffff, 0x00000007, 698 0x9180, 0xffffffff, 0x00060005, 699 0x9184, 0xffffffff, 0x00090008, 700 0x9188, 0xffffffff, 0x00030002, 701 0x918c, 0xffffffff, 0x00050004, 702 0x9190, 0xffffffff, 0x00000008, 703 0x9194, 0xffffffff, 0x00070006, 704 0x9198, 0xffffffff, 0x000a0009, 705 0x919c, 0xffffffff, 0x00040003, 706 0x91a0, 0xffffffff, 0x00060005, 707 0x91a4, 0xffffffff, 0x00000009, 708 0x91a8, 0xffffffff, 0x00080007, 709 0x91ac, 0xffffffff, 0x000b000a, 710 0x91b0, 0xffffffff, 0x00050004, 711 0x91b4, 0xffffffff, 0x00070006, 712 0x91b8, 0xffffffff, 0x0008000b, 713 0x91bc, 0xffffffff, 0x000a0009, 714 0x91c0, 0xffffffff, 0x000d000c, 715 0x9200, 0xffffffff, 0x00090008, 716 0x9204, 0xffffffff, 0x000b000a, 717 0x9208, 0xffffffff, 0x000c000f, 718 0x920c, 0xffffffff, 0x000e000d, 719 0x9210, 0xffffffff, 0x00110010, 720 0x9214, 0xffffffff, 0x000a0009, 721 0x9218, 0xffffffff, 0x000c000b, 722 0x921c, 0xffffffff, 0x0000000f, 723 0x9220, 0xffffffff, 0x000e000d, 724 0x9224, 0xffffffff, 0x00110010, 725 0x9228, 0xffffffff, 0x000b000a, 726 0x922c, 0xffffffff, 0x000d000c, 727 0x9230, 0xffffffff, 0x00000010, 728 0x9234, 0xffffffff, 0x000f000e, 729 0x9238, 0xffffffff, 0x00120011, 730 0x923c, 0xffffffff, 0x000c000b, 731 0x9240, 0xffffffff, 0x000e000d, 732 0x9244, 0xffffffff, 0x00000011, 733 0x9248, 0xffffffff, 0x0010000f, 734 0x924c, 0xffffffff, 0x00130012, 735 0x9250, 0xffffffff, 0x000d000c, 736 0x9254, 0xffffffff, 0x000f000e, 737 0x9258, 0xffffffff, 0x00100013, 738 0x925c, 0xffffffff, 0x00120011, 739 0x9260, 0xffffffff, 0x00150014, 740 0x9150, 0xffffffff, 0x96940200, 741 0x8708, 0xffffffff, 0x00900100, 742 0xc478, 0xffffffff, 0x00000080, 743 0xc404, 0xffffffff, 0x0020003f, 744 0x30, 0xffffffff, 0x0000001c, 745 0x34, 0x000f0000, 0x000f0000, 746 0x160c, 0xffffffff, 0x00000100, 747 0x1024, 0xffffffff, 0x00000100, 748 0x102c, 0x00000101, 0x00000000, 749 0x20a8, 0xffffffff, 0x00000104, 750 0x55e4, 0xff000fff, 0x00000100, 751 0x55e8, 0x00000001, 0x00000001, 752 0x2f50, 0x00000001, 0x00000001, 753 0x30cc, 0xc0000fff, 0x00000104, 754 0xc1e4, 0x00000001, 0x00000001, 755 0xd0c0, 0xfffffff0, 0x00000100, 756 0xd8c0, 0xfffffff0, 0x00000100 757 }; 758 759 static const u32 verde_mgcg_cgcg_init[] = 760 { 761 0xc400, 0xffffffff, 0xfffffffc, 762 0x802c, 0xffffffff, 0xe0000000, 763 0x9a60, 0xffffffff, 0x00000100, 764 0x92a4, 0xffffffff, 0x00000100, 765 0xc164, 0xffffffff, 0x00000100, 766 0x9774, 0xffffffff, 0x00000100, 767 0x8984, 0xffffffff, 0x06000100, 768 0x8a18, 0xffffffff, 0x00000100, 769 0x92a0, 0xffffffff, 0x00000100, 770 0xc380, 0xffffffff, 0x00000100, 771 0x8b28, 0xffffffff, 0x00000100, 772 0x9144, 0xffffffff, 0x00000100, 773 0x8d88, 0xffffffff, 0x00000100, 774 0x8d8c, 0xffffffff, 0x00000100, 775 0x9030, 0xffffffff, 0x00000100, 776 0x9034, 0xffffffff, 0x00000100, 777 0x9038, 0xffffffff, 0x00000100, 778 0x903c, 0xffffffff, 0x00000100, 779 0xad80, 0xffffffff, 0x00000100, 780 0xac54, 0xffffffff, 0x00000100, 781 0x897c, 0xffffffff, 0x06000100, 782 0x9868, 0xffffffff, 0x00000100, 783 0x9510, 0xffffffff, 0x00000100, 784 0xaf04, 0xffffffff, 0x00000100, 785 0xae04, 0xffffffff, 0x00000100, 786 0x949c, 0xffffffff, 0x00000100, 787 0x802c, 0xffffffff, 0xe0000000, 788 0x9160, 0xffffffff, 0x00010000, 789 0x9164, 0xffffffff, 0x00030002, 790 0x9168, 0xffffffff, 0x00040007, 791 0x916c, 0xffffffff, 0x00060005, 792 0x9170, 0xffffffff, 0x00090008, 793 0x9174, 0xffffffff, 0x00020001, 794 0x9178, 0xffffffff, 0x00040003, 795 0x917c, 0xffffffff, 0x00000007, 796 0x9180, 0xffffffff, 0x00060005, 797 0x9184, 0xffffffff, 0x00090008, 798 0x9188, 0xffffffff, 0x00030002, 799 0x918c, 0xffffffff, 0x00050004, 800 0x9190, 0xffffffff, 0x00000008, 801 0x9194, 0xffffffff, 0x00070006, 802 0x9198, 0xffffffff, 0x000a0009, 803 0x919c, 0xffffffff, 0x00040003, 804 0x91a0, 0xffffffff, 0x00060005, 805 0x91a4, 0xffffffff, 0x00000009, 806 0x91a8, 0xffffffff, 0x00080007, 807 0x91ac, 0xffffffff, 0x000b000a, 808 0x91b0, 0xffffffff, 0x00050004, 809 0x91b4, 0xffffffff, 0x00070006, 810 0x91b8, 0xffffffff, 0x0008000b, 811 0x91bc, 0xffffffff, 0x000a0009, 812 0x91c0, 0xffffffff, 0x000d000c, 813 0x9200, 0xffffffff, 0x00090008, 814 0x9204, 0xffffffff, 0x000b000a, 815 0x9208, 0xffffffff, 0x000c000f, 816 0x920c, 0xffffffff, 0x000e000d, 817 0x9210, 0xffffffff, 0x00110010, 818 0x9214, 0xffffffff, 0x000a0009, 819 0x9218, 0xffffffff, 0x000c000b, 820 0x921c, 0xffffffff, 0x0000000f, 821 0x9220, 0xffffffff, 0x000e000d, 822 0x9224, 0xffffffff, 0x00110010, 823 0x9228, 0xffffffff, 0x000b000a, 824 0x922c, 0xffffffff, 0x000d000c, 825 0x9230, 0xffffffff, 0x00000010, 826 0x9234, 0xffffffff, 0x000f000e, 827 0x9238, 0xffffffff, 0x00120011, 828 0x923c, 0xffffffff, 0x000c000b, 829 0x9240, 0xffffffff, 0x000e000d, 830 0x9244, 0xffffffff, 0x00000011, 831 0x9248, 0xffffffff, 0x0010000f, 832 0x924c, 0xffffffff, 0x00130012, 833 0x9250, 0xffffffff, 0x000d000c, 834 0x9254, 0xffffffff, 0x000f000e, 835 0x9258, 0xffffffff, 0x00100013, 836 0x925c, 0xffffffff, 0x00120011, 837 0x9260, 0xffffffff, 0x00150014, 838 0x9150, 0xffffffff, 0x96940200, 839 0x8708, 0xffffffff, 0x00900100, 840 0xc478, 0xffffffff, 0x00000080, 841 0xc404, 0xffffffff, 0x0020003f, 842 0x30, 0xffffffff, 0x0000001c, 843 0x34, 0x000f0000, 0x000f0000, 844 0x160c, 0xffffffff, 0x00000100, 845 0x1024, 0xffffffff, 0x00000100, 846 0x102c, 0x00000101, 0x00000000, 847 0x20a8, 0xffffffff, 0x00000104, 848 0x264c, 0x000c0000, 0x000c0000, 849 0x2648, 0x000c0000, 0x000c0000, 850 0x55e4, 0xff000fff, 0x00000100, 851 0x55e8, 0x00000001, 0x00000001, 852 0x2f50, 0x00000001, 0x00000001, 853 0x30cc, 0xc0000fff, 0x00000104, 854 0xc1e4, 0x00000001, 0x00000001, 855 0xd0c0, 0xfffffff0, 0x00000100, 856 0xd8c0, 0xfffffff0, 0x00000100 857 }; 858 859 static const u32 oland_mgcg_cgcg_init[] = 860 { 861 0xc400, 0xffffffff, 0xfffffffc, 862 0x802c, 0xffffffff, 0xe0000000, 863 0x9a60, 0xffffffff, 0x00000100, 864 0x92a4, 0xffffffff, 0x00000100, 865 0xc164, 0xffffffff, 0x00000100, 866 0x9774, 0xffffffff, 0x00000100, 867 0x8984, 0xffffffff, 0x06000100, 868 0x8a18, 0xffffffff, 0x00000100, 869 0x92a0, 0xffffffff, 0x00000100, 870 0xc380, 0xffffffff, 0x00000100, 871 0x8b28, 0xffffffff, 0x00000100, 872 0x9144, 0xffffffff, 0x00000100, 873 0x8d88, 0xffffffff, 0x00000100, 874 0x8d8c, 0xffffffff, 0x00000100, 875 0x9030, 0xffffffff, 0x00000100, 876 0x9034, 0xffffffff, 0x00000100, 877 0x9038, 0xffffffff, 0x00000100, 878 0x903c, 0xffffffff, 0x00000100, 879 0xad80, 0xffffffff, 0x00000100, 880 0xac54, 0xffffffff, 0x00000100, 881 0x897c, 0xffffffff, 0x06000100, 882 0x9868, 0xffffffff, 0x00000100, 883 0x9510, 0xffffffff, 0x00000100, 884 0xaf04, 0xffffffff, 0x00000100, 885 0xae04, 0xffffffff, 0x00000100, 886 0x949c, 0xffffffff, 0x00000100, 887 0x802c, 0xffffffff, 0xe0000000, 888 0x9160, 0xffffffff, 0x00010000, 889 0x9164, 0xffffffff, 0x00030002, 890 0x9168, 0xffffffff, 0x00040007, 891 0x916c, 0xffffffff, 0x00060005, 892 0x9170, 0xffffffff, 0x00090008, 893 0x9174, 0xffffffff, 0x00020001, 894 0x9178, 0xffffffff, 0x00040003, 895 0x917c, 0xffffffff, 0x00000007, 896 0x9180, 0xffffffff, 0x00060005, 897 0x9184, 0xffffffff, 0x00090008, 898 0x9188, 0xffffffff, 0x00030002, 899 0x918c, 0xffffffff, 0x00050004, 900 0x9190, 0xffffffff, 0x00000008, 901 0x9194, 0xffffffff, 0x00070006, 902 0x9198, 0xffffffff, 0x000a0009, 903 0x919c, 0xffffffff, 0x00040003, 904 0x91a0, 0xffffffff, 0x00060005, 905 0x91a4, 0xffffffff, 0x00000009, 906 0x91a8, 0xffffffff, 0x00080007, 907 0x91ac, 0xffffffff, 0x000b000a, 908 0x91b0, 0xffffffff, 0x00050004, 909 0x91b4, 0xffffffff, 0x00070006, 910 0x91b8, 0xffffffff, 0x0008000b, 911 0x91bc, 0xffffffff, 0x000a0009, 912 0x91c0, 0xffffffff, 0x000d000c, 913 0x91c4, 0xffffffff, 0x00060005, 914 0x91c8, 0xffffffff, 0x00080007, 915 0x91cc, 0xffffffff, 0x0000000b, 916 0x91d0, 0xffffffff, 0x000a0009, 917 0x91d4, 0xffffffff, 0x000d000c, 918 0x9150, 0xffffffff, 0x96940200, 919 0x8708, 0xffffffff, 0x00900100, 920 0xc478, 0xffffffff, 0x00000080, 921 0xc404, 0xffffffff, 0x0020003f, 922 0x30, 0xffffffff, 0x0000001c, 923 0x34, 0x000f0000, 0x000f0000, 924 0x160c, 0xffffffff, 0x00000100, 925 0x1024, 0xffffffff, 0x00000100, 926 0x102c, 0x00000101, 0x00000000, 927 0x20a8, 0xffffffff, 0x00000104, 928 0x264c, 0x000c0000, 0x000c0000, 929 0x2648, 0x000c0000, 0x000c0000, 930 0x55e4, 0xff000fff, 0x00000100, 931 0x55e8, 0x00000001, 0x00000001, 932 0x2f50, 0x00000001, 0x00000001, 933 0x30cc, 0xc0000fff, 0x00000104, 934 0xc1e4, 0x00000001, 0x00000001, 935 0xd0c0, 0xfffffff0, 0x00000100, 936 0xd8c0, 0xfffffff0, 0x00000100 937 }; 938 939 static const u32 hainan_mgcg_cgcg_init[] = 940 { 941 0xc400, 0xffffffff, 0xfffffffc, 942 0x802c, 0xffffffff, 0xe0000000, 943 0x9a60, 0xffffffff, 0x00000100, 944 0x92a4, 0xffffffff, 0x00000100, 945 0xc164, 0xffffffff, 0x00000100, 946 0x9774, 0xffffffff, 0x00000100, 947 0x8984, 0xffffffff, 0x06000100, 948 0x8a18, 0xffffffff, 0x00000100, 949 0x92a0, 0xffffffff, 0x00000100, 950 0xc380, 0xffffffff, 0x00000100, 951 0x8b28, 0xffffffff, 0x00000100, 952 0x9144, 0xffffffff, 0x00000100, 953 0x8d88, 0xffffffff, 0x00000100, 954 0x8d8c, 0xffffffff, 0x00000100, 955 0x9030, 0xffffffff, 0x00000100, 956 0x9034, 0xffffffff, 0x00000100, 957 0x9038, 0xffffffff, 0x00000100, 958 0x903c, 0xffffffff, 0x00000100, 959 0xad80, 0xffffffff, 0x00000100, 960 0xac54, 0xffffffff, 0x00000100, 961 0x897c, 0xffffffff, 0x06000100, 962 0x9868, 0xffffffff, 0x00000100, 963 0x9510, 0xffffffff, 0x00000100, 964 0xaf04, 0xffffffff, 0x00000100, 965 0xae04, 0xffffffff, 0x00000100, 966 0x949c, 0xffffffff, 0x00000100, 967 0x802c, 0xffffffff, 0xe0000000, 968 0x9160, 0xffffffff, 0x00010000, 969 0x9164, 0xffffffff, 0x00030002, 970 0x9168, 0xffffffff, 0x00040007, 971 0x916c, 0xffffffff, 0x00060005, 972 0x9170, 0xffffffff, 0x00090008, 973 0x9174, 0xffffffff, 0x00020001, 974 0x9178, 0xffffffff, 0x00040003, 975 0x917c, 0xffffffff, 0x00000007, 976 0x9180, 0xffffffff, 0x00060005, 977 0x9184, 0xffffffff, 0x00090008, 978 0x9188, 0xffffffff, 0x00030002, 979 0x918c, 0xffffffff, 0x00050004, 980 0x9190, 0xffffffff, 0x00000008, 981 0x9194, 0xffffffff, 0x00070006, 982 0x9198, 0xffffffff, 0x000a0009, 983 0x919c, 0xffffffff, 0x00040003, 984 0x91a0, 0xffffffff, 0x00060005, 985 0x91a4, 0xffffffff, 0x00000009, 986 0x91a8, 0xffffffff, 0x00080007, 987 0x91ac, 0xffffffff, 0x000b000a, 988 0x91b0, 0xffffffff, 0x00050004, 989 0x91b4, 0xffffffff, 0x00070006, 990 0x91b8, 0xffffffff, 0x0008000b, 991 0x91bc, 0xffffffff, 0x000a0009, 992 0x91c0, 0xffffffff, 0x000d000c, 993 0x91c4, 0xffffffff, 0x00060005, 994 0x91c8, 0xffffffff, 0x00080007, 995 0x91cc, 0xffffffff, 0x0000000b, 996 0x91d0, 0xffffffff, 0x000a0009, 997 0x91d4, 0xffffffff, 0x000d000c, 998 0x9150, 0xffffffff, 0x96940200, 999 0x8708, 0xffffffff, 0x00900100, 1000 0xc478, 0xffffffff, 0x00000080, 1001 0xc404, 0xffffffff, 0x0020003f, 1002 0x30, 0xffffffff, 0x0000001c, 1003 0x34, 0x000f0000, 0x000f0000, 1004 0x160c, 0xffffffff, 0x00000100, 1005 0x1024, 0xffffffff, 0x00000100, 1006 0x20a8, 0xffffffff, 0x00000104, 1007 0x264c, 0x000c0000, 0x000c0000, 1008 0x2648, 0x000c0000, 0x000c0000, 1009 0x2f50, 0x00000001, 0x00000001, 1010 0x30cc, 0xc0000fff, 0x00000104, 1011 0xc1e4, 0x00000001, 0x00000001, 1012 0xd0c0, 0xfffffff0, 0x00000100, 1013 0xd8c0, 0xfffffff0, 0x00000100 1014 }; 1015 1016 static u32 verde_pg_init[] = 1017 { 1018 0x353c, 0xffffffff, 0x40000, 1019 0x3538, 0xffffffff, 0x200010ff, 1020 0x353c, 0xffffffff, 0x0, 1021 0x353c, 0xffffffff, 0x0, 1022 0x353c, 0xffffffff, 0x0, 1023 0x353c, 0xffffffff, 0x0, 1024 0x353c, 0xffffffff, 0x0, 1025 0x353c, 0xffffffff, 0x7007, 1026 0x3538, 0xffffffff, 0x300010ff, 1027 0x353c, 0xffffffff, 0x0, 1028 0x353c, 0xffffffff, 0x0, 1029 0x353c, 0xffffffff, 0x0, 1030 0x353c, 0xffffffff, 0x0, 1031 0x353c, 0xffffffff, 0x0, 1032 0x353c, 0xffffffff, 0x400000, 1033 0x3538, 0xffffffff, 0x100010ff, 1034 0x353c, 0xffffffff, 0x0, 1035 0x353c, 0xffffffff, 0x0, 1036 0x353c, 0xffffffff, 0x0, 1037 0x353c, 0xffffffff, 0x0, 1038 0x353c, 0xffffffff, 0x0, 1039 0x353c, 0xffffffff, 0x120200, 1040 0x3538, 0xffffffff, 0x500010ff, 1041 0x353c, 0xffffffff, 0x0, 1042 0x353c, 0xffffffff, 0x0, 1043 0x353c, 0xffffffff, 0x0, 1044 0x353c, 0xffffffff, 0x0, 1045 0x353c, 0xffffffff, 0x0, 1046 0x353c, 0xffffffff, 0x1e1e16, 1047 0x3538, 0xffffffff, 0x600010ff, 1048 0x353c, 0xffffffff, 0x0, 1049 0x353c, 0xffffffff, 0x0, 1050 0x353c, 0xffffffff, 0x0, 1051 0x353c, 0xffffffff, 0x0, 1052 0x353c, 0xffffffff, 0x0, 1053 0x353c, 0xffffffff, 0x171f1e, 1054 0x3538, 0xffffffff, 0x700010ff, 1055 0x353c, 0xffffffff, 0x0, 1056 0x353c, 0xffffffff, 0x0, 1057 0x353c, 0xffffffff, 0x0, 1058 0x353c, 0xffffffff, 0x0, 1059 0x353c, 0xffffffff, 0x0, 1060 0x353c, 0xffffffff, 0x0, 1061 0x3538, 0xffffffff, 0x9ff, 1062 0x3500, 0xffffffff, 0x0, 1063 0x3504, 0xffffffff, 0x10000800, 1064 0x3504, 0xffffffff, 0xf, 1065 0x3504, 0xffffffff, 0xf, 1066 0x3500, 0xffffffff, 0x4, 1067 0x3504, 0xffffffff, 0x1000051e, 1068 0x3504, 0xffffffff, 0xffff, 1069 0x3504, 0xffffffff, 0xffff, 1070 0x3500, 0xffffffff, 0x8, 1071 0x3504, 0xffffffff, 0x80500, 1072 0x3500, 0xffffffff, 0x12, 1073 0x3504, 0xffffffff, 0x9050c, 1074 0x3500, 0xffffffff, 0x1d, 1075 0x3504, 0xffffffff, 0xb052c, 1076 0x3500, 0xffffffff, 0x2a, 1077 0x3504, 0xffffffff, 0x1053e, 1078 0x3500, 0xffffffff, 0x2d, 1079 0x3504, 0xffffffff, 0x10546, 1080 0x3500, 0xffffffff, 0x30, 1081 0x3504, 0xffffffff, 0xa054e, 1082 0x3500, 0xffffffff, 0x3c, 1083 0x3504, 0xffffffff, 0x1055f, 1084 0x3500, 0xffffffff, 0x3f, 1085 0x3504, 0xffffffff, 0x10567, 1086 0x3500, 0xffffffff, 0x42, 1087 0x3504, 0xffffffff, 0x1056f, 1088 0x3500, 0xffffffff, 0x45, 1089 0x3504, 0xffffffff, 0x10572, 1090 0x3500, 0xffffffff, 0x48, 1091 0x3504, 0xffffffff, 0x20575, 1092 0x3500, 0xffffffff, 0x4c, 1093 0x3504, 0xffffffff, 0x190801, 1094 0x3500, 0xffffffff, 0x67, 1095 0x3504, 0xffffffff, 0x1082a, 1096 0x3500, 0xffffffff, 0x6a, 1097 0x3504, 0xffffffff, 0x1b082d, 1098 0x3500, 0xffffffff, 0x87, 1099 0x3504, 0xffffffff, 0x310851, 1100 0x3500, 0xffffffff, 0xba, 1101 0x3504, 0xffffffff, 0x891, 1102 0x3500, 0xffffffff, 0xbc, 1103 0x3504, 0xffffffff, 0x893, 1104 0x3500, 0xffffffff, 0xbe, 1105 0x3504, 0xffffffff, 0x20895, 1106 0x3500, 0xffffffff, 0xc2, 1107 0x3504, 0xffffffff, 0x20899, 1108 0x3500, 0xffffffff, 0xc6, 1109 0x3504, 0xffffffff, 0x2089d, 1110 0x3500, 0xffffffff, 0xca, 1111 0x3504, 0xffffffff, 0x8a1, 1112 0x3500, 0xffffffff, 0xcc, 1113 0x3504, 0xffffffff, 0x8a3, 1114 0x3500, 0xffffffff, 0xce, 1115 0x3504, 0xffffffff, 0x308a5, 1116 0x3500, 0xffffffff, 0xd3, 1117 0x3504, 0xffffffff, 0x6d08cd, 1118 0x3500, 0xffffffff, 0x142, 1119 0x3504, 0xffffffff, 0x2000095a, 1120 0x3504, 0xffffffff, 0x1, 1121 0x3500, 0xffffffff, 0x144, 1122 0x3504, 0xffffffff, 0x301f095b, 1123 0x3500, 0xffffffff, 0x165, 1124 0x3504, 0xffffffff, 0xc094d, 1125 0x3500, 0xffffffff, 0x173, 1126 0x3504, 0xffffffff, 0xf096d, 1127 0x3500, 0xffffffff, 0x184, 1128 0x3504, 0xffffffff, 0x15097f, 1129 0x3500, 0xffffffff, 0x19b, 1130 0x3504, 0xffffffff, 0xc0998, 1131 0x3500, 0xffffffff, 0x1a9, 1132 0x3504, 0xffffffff, 0x409a7, 1133 0x3500, 0xffffffff, 0x1af, 1134 0x3504, 0xffffffff, 0xcdc, 1135 0x3500, 0xffffffff, 0x1b1, 1136 0x3504, 0xffffffff, 0x800, 1137 0x3508, 0xffffffff, 0x6c9b2000, 1138 0x3510, 0xfc00, 0x2000, 1139 0x3544, 0xffffffff, 0xfc0, 1140 0x28d4, 0x00000100, 0x100 1141 }; 1142 1143 static void si_init_golden_registers(struct radeon_device *rdev) 1144 { 1145 switch (rdev->family) { 1146 case CHIP_TAHITI: 1147 radeon_program_register_sequence(rdev, 1148 tahiti_golden_registers, 1149 (const u32)ARRAY_SIZE(tahiti_golden_registers)); 1150 radeon_program_register_sequence(rdev, 1151 tahiti_golden_rlc_registers, 1152 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers)); 1153 radeon_program_register_sequence(rdev, 1154 tahiti_mgcg_cgcg_init, 1155 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init)); 1156 radeon_program_register_sequence(rdev, 1157 tahiti_golden_registers2, 1158 (const u32)ARRAY_SIZE(tahiti_golden_registers2)); 1159 break; 1160 case CHIP_PITCAIRN: 1161 radeon_program_register_sequence(rdev, 1162 pitcairn_golden_registers, 1163 (const u32)ARRAY_SIZE(pitcairn_golden_registers)); 1164 radeon_program_register_sequence(rdev, 1165 pitcairn_golden_rlc_registers, 1166 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers)); 1167 radeon_program_register_sequence(rdev, 1168 pitcairn_mgcg_cgcg_init, 1169 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init)); 1170 break; 1171 case CHIP_VERDE: 1172 radeon_program_register_sequence(rdev, 1173 verde_golden_registers, 1174 (const u32)ARRAY_SIZE(verde_golden_registers)); 1175 radeon_program_register_sequence(rdev, 1176 verde_golden_rlc_registers, 1177 (const u32)ARRAY_SIZE(verde_golden_rlc_registers)); 1178 radeon_program_register_sequence(rdev, 1179 verde_mgcg_cgcg_init, 1180 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init)); 1181 radeon_program_register_sequence(rdev, 1182 verde_pg_init, 1183 (const u32)ARRAY_SIZE(verde_pg_init)); 1184 break; 1185 case CHIP_OLAND: 1186 radeon_program_register_sequence(rdev, 1187 oland_golden_registers, 1188 (const u32)ARRAY_SIZE(oland_golden_registers)); 1189 radeon_program_register_sequence(rdev, 1190 oland_golden_rlc_registers, 1191 (const u32)ARRAY_SIZE(oland_golden_rlc_registers)); 1192 radeon_program_register_sequence(rdev, 1193 oland_mgcg_cgcg_init, 1194 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init)); 1195 break; 1196 case CHIP_HAINAN: 1197 radeon_program_register_sequence(rdev, 1198 hainan_golden_registers, 1199 (const u32)ARRAY_SIZE(hainan_golden_registers)); 1200 radeon_program_register_sequence(rdev, 1201 hainan_golden_registers2, 1202 (const u32)ARRAY_SIZE(hainan_golden_registers2)); 1203 radeon_program_register_sequence(rdev, 1204 hainan_mgcg_cgcg_init, 1205 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init)); 1206 break; 1207 default: 1208 break; 1209 } 1210 } 1211 1212 #define PCIE_BUS_CLK 10000 1213 #define TCLK (PCIE_BUS_CLK / 10) 1214 1215 /** 1216 * si_get_xclk - get the xclk 1217 * 1218 * @rdev: radeon_device pointer 1219 * 1220 * Returns the reference clock used by the gfx engine 1221 * (SI). 1222 */ 1223 u32 si_get_xclk(struct radeon_device *rdev) 1224 { 1225 u32 reference_clock = rdev->clock.spll.reference_freq; 1226 u32 tmp; 1227 1228 tmp = RREG32(CG_CLKPIN_CNTL_2); 1229 if (tmp & MUX_TCLK_TO_XCLK) 1230 return TCLK; 1231 1232 tmp = RREG32(CG_CLKPIN_CNTL); 1233 if (tmp & XTALIN_DIVIDE) 1234 return reference_clock / 4; 1235 1236 return reference_clock; 1237 } 1238 1239 /* get temperature in millidegrees */ 1240 int si_get_temp(struct radeon_device *rdev) 1241 { 1242 u32 temp; 1243 int actual_temp = 0; 1244 1245 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >> 1246 CTF_TEMP_SHIFT; 1247 1248 if (temp & 0x200) 1249 actual_temp = 255; 1250 else 1251 actual_temp = temp & 0x1ff; 1252 1253 actual_temp = (actual_temp * 1000); 1254 1255 return actual_temp; 1256 } 1257 1258 #define TAHITI_IO_MC_REGS_SIZE 36 1259 1260 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1261 {0x0000006f, 0x03044000}, 1262 {0x00000070, 0x0480c018}, 1263 {0x00000071, 0x00000040}, 1264 {0x00000072, 0x01000000}, 1265 {0x00000074, 0x000000ff}, 1266 {0x00000075, 0x00143400}, 1267 {0x00000076, 0x08ec0800}, 1268 {0x00000077, 0x040000cc}, 1269 {0x00000079, 0x00000000}, 1270 {0x0000007a, 0x21000409}, 1271 {0x0000007c, 0x00000000}, 1272 {0x0000007d, 0xe8000000}, 1273 {0x0000007e, 0x044408a8}, 1274 {0x0000007f, 0x00000003}, 1275 {0x00000080, 0x00000000}, 1276 {0x00000081, 0x01000000}, 1277 {0x00000082, 0x02000000}, 1278 {0x00000083, 0x00000000}, 1279 {0x00000084, 0xe3f3e4f4}, 1280 {0x00000085, 0x00052024}, 1281 {0x00000087, 0x00000000}, 1282 {0x00000088, 0x66036603}, 1283 {0x00000089, 0x01000000}, 1284 {0x0000008b, 0x1c0a0000}, 1285 {0x0000008c, 0xff010000}, 1286 {0x0000008e, 0xffffefff}, 1287 {0x0000008f, 0xfff3efff}, 1288 {0x00000090, 0xfff3efbf}, 1289 {0x00000094, 0x00101101}, 1290 {0x00000095, 0x00000fff}, 1291 {0x00000096, 0x00116fff}, 1292 {0x00000097, 0x60010000}, 1293 {0x00000098, 0x10010000}, 1294 {0x00000099, 0x00006000}, 1295 {0x0000009a, 0x00001000}, 1296 {0x0000009f, 0x00a77400} 1297 }; 1298 1299 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1300 {0x0000006f, 0x03044000}, 1301 {0x00000070, 0x0480c018}, 1302 {0x00000071, 0x00000040}, 1303 {0x00000072, 0x01000000}, 1304 {0x00000074, 0x000000ff}, 1305 {0x00000075, 0x00143400}, 1306 {0x00000076, 0x08ec0800}, 1307 {0x00000077, 0x040000cc}, 1308 {0x00000079, 0x00000000}, 1309 {0x0000007a, 0x21000409}, 1310 {0x0000007c, 0x00000000}, 1311 {0x0000007d, 0xe8000000}, 1312 {0x0000007e, 0x044408a8}, 1313 {0x0000007f, 0x00000003}, 1314 {0x00000080, 0x00000000}, 1315 {0x00000081, 0x01000000}, 1316 {0x00000082, 0x02000000}, 1317 {0x00000083, 0x00000000}, 1318 {0x00000084, 0xe3f3e4f4}, 1319 {0x00000085, 0x00052024}, 1320 {0x00000087, 0x00000000}, 1321 {0x00000088, 0x66036603}, 1322 {0x00000089, 0x01000000}, 1323 {0x0000008b, 0x1c0a0000}, 1324 {0x0000008c, 0xff010000}, 1325 {0x0000008e, 0xffffefff}, 1326 {0x0000008f, 0xfff3efff}, 1327 {0x00000090, 0xfff3efbf}, 1328 {0x00000094, 0x00101101}, 1329 {0x00000095, 0x00000fff}, 1330 {0x00000096, 0x00116fff}, 1331 {0x00000097, 0x60010000}, 1332 {0x00000098, 0x10010000}, 1333 {0x00000099, 0x00006000}, 1334 {0x0000009a, 0x00001000}, 1335 {0x0000009f, 0x00a47400} 1336 }; 1337 1338 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1339 {0x0000006f, 0x03044000}, 1340 {0x00000070, 0x0480c018}, 1341 {0x00000071, 0x00000040}, 1342 {0x00000072, 0x01000000}, 1343 {0x00000074, 0x000000ff}, 1344 {0x00000075, 0x00143400}, 1345 {0x00000076, 0x08ec0800}, 1346 {0x00000077, 0x040000cc}, 1347 {0x00000079, 0x00000000}, 1348 {0x0000007a, 0x21000409}, 1349 {0x0000007c, 0x00000000}, 1350 {0x0000007d, 0xe8000000}, 1351 {0x0000007e, 0x044408a8}, 1352 {0x0000007f, 0x00000003}, 1353 {0x00000080, 0x00000000}, 1354 {0x00000081, 0x01000000}, 1355 {0x00000082, 0x02000000}, 1356 {0x00000083, 0x00000000}, 1357 {0x00000084, 0xe3f3e4f4}, 1358 {0x00000085, 0x00052024}, 1359 {0x00000087, 0x00000000}, 1360 {0x00000088, 0x66036603}, 1361 {0x00000089, 0x01000000}, 1362 {0x0000008b, 0x1c0a0000}, 1363 {0x0000008c, 0xff010000}, 1364 {0x0000008e, 0xffffefff}, 1365 {0x0000008f, 0xfff3efff}, 1366 {0x00000090, 0xfff3efbf}, 1367 {0x00000094, 0x00101101}, 1368 {0x00000095, 0x00000fff}, 1369 {0x00000096, 0x00116fff}, 1370 {0x00000097, 0x60010000}, 1371 {0x00000098, 0x10010000}, 1372 {0x00000099, 0x00006000}, 1373 {0x0000009a, 0x00001000}, 1374 {0x0000009f, 0x00a37400} 1375 }; 1376 1377 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1378 {0x0000006f, 0x03044000}, 1379 {0x00000070, 0x0480c018}, 1380 {0x00000071, 0x00000040}, 1381 {0x00000072, 0x01000000}, 1382 {0x00000074, 0x000000ff}, 1383 {0x00000075, 0x00143400}, 1384 {0x00000076, 0x08ec0800}, 1385 {0x00000077, 0x040000cc}, 1386 {0x00000079, 0x00000000}, 1387 {0x0000007a, 0x21000409}, 1388 {0x0000007c, 0x00000000}, 1389 {0x0000007d, 0xe8000000}, 1390 {0x0000007e, 0x044408a8}, 1391 {0x0000007f, 0x00000003}, 1392 {0x00000080, 0x00000000}, 1393 {0x00000081, 0x01000000}, 1394 {0x00000082, 0x02000000}, 1395 {0x00000083, 0x00000000}, 1396 {0x00000084, 0xe3f3e4f4}, 1397 {0x00000085, 0x00052024}, 1398 {0x00000087, 0x00000000}, 1399 {0x00000088, 0x66036603}, 1400 {0x00000089, 0x01000000}, 1401 {0x0000008b, 0x1c0a0000}, 1402 {0x0000008c, 0xff010000}, 1403 {0x0000008e, 0xffffefff}, 1404 {0x0000008f, 0xfff3efff}, 1405 {0x00000090, 0xfff3efbf}, 1406 {0x00000094, 0x00101101}, 1407 {0x00000095, 0x00000fff}, 1408 {0x00000096, 0x00116fff}, 1409 {0x00000097, 0x60010000}, 1410 {0x00000098, 0x10010000}, 1411 {0x00000099, 0x00006000}, 1412 {0x0000009a, 0x00001000}, 1413 {0x0000009f, 0x00a17730} 1414 }; 1415 1416 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1417 {0x0000006f, 0x03044000}, 1418 {0x00000070, 0x0480c018}, 1419 {0x00000071, 0x00000040}, 1420 {0x00000072, 0x01000000}, 1421 {0x00000074, 0x000000ff}, 1422 {0x00000075, 0x00143400}, 1423 {0x00000076, 0x08ec0800}, 1424 {0x00000077, 0x040000cc}, 1425 {0x00000079, 0x00000000}, 1426 {0x0000007a, 0x21000409}, 1427 {0x0000007c, 0x00000000}, 1428 {0x0000007d, 0xe8000000}, 1429 {0x0000007e, 0x044408a8}, 1430 {0x0000007f, 0x00000003}, 1431 {0x00000080, 0x00000000}, 1432 {0x00000081, 0x01000000}, 1433 {0x00000082, 0x02000000}, 1434 {0x00000083, 0x00000000}, 1435 {0x00000084, 0xe3f3e4f4}, 1436 {0x00000085, 0x00052024}, 1437 {0x00000087, 0x00000000}, 1438 {0x00000088, 0x66036603}, 1439 {0x00000089, 0x01000000}, 1440 {0x0000008b, 0x1c0a0000}, 1441 {0x0000008c, 0xff010000}, 1442 {0x0000008e, 0xffffefff}, 1443 {0x0000008f, 0xfff3efff}, 1444 {0x00000090, 0xfff3efbf}, 1445 {0x00000094, 0x00101101}, 1446 {0x00000095, 0x00000fff}, 1447 {0x00000096, 0x00116fff}, 1448 {0x00000097, 0x60010000}, 1449 {0x00000098, 0x10010000}, 1450 {0x00000099, 0x00006000}, 1451 {0x0000009a, 0x00001000}, 1452 {0x0000009f, 0x00a07730} 1453 }; 1454 1455 /* ucode loading */ 1456 static int si_mc_load_microcode(struct radeon_device *rdev) 1457 { 1458 const __be32 *fw_data; 1459 u32 running, blackout = 0; 1460 u32 *io_mc_regs; 1461 int i, ucode_size, regs_size; 1462 1463 if (!rdev->mc_fw) 1464 return -EINVAL; 1465 1466 switch (rdev->family) { 1467 case CHIP_TAHITI: 1468 io_mc_regs = (u32 *)&tahiti_io_mc_regs; 1469 ucode_size = SI_MC_UCODE_SIZE; 1470 regs_size = TAHITI_IO_MC_REGS_SIZE; 1471 break; 1472 case CHIP_PITCAIRN: 1473 io_mc_regs = (u32 *)&pitcairn_io_mc_regs; 1474 ucode_size = SI_MC_UCODE_SIZE; 1475 regs_size = TAHITI_IO_MC_REGS_SIZE; 1476 break; 1477 case CHIP_VERDE: 1478 default: 1479 io_mc_regs = (u32 *)&verde_io_mc_regs; 1480 ucode_size = SI_MC_UCODE_SIZE; 1481 regs_size = TAHITI_IO_MC_REGS_SIZE; 1482 break; 1483 case CHIP_OLAND: 1484 io_mc_regs = (u32 *)&oland_io_mc_regs; 1485 ucode_size = OLAND_MC_UCODE_SIZE; 1486 regs_size = TAHITI_IO_MC_REGS_SIZE; 1487 break; 1488 case CHIP_HAINAN: 1489 io_mc_regs = (u32 *)&hainan_io_mc_regs; 1490 ucode_size = OLAND_MC_UCODE_SIZE; 1491 regs_size = TAHITI_IO_MC_REGS_SIZE; 1492 break; 1493 } 1494 1495 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; 1496 1497 if (running == 0) { 1498 if (running) { 1499 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 1500 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 1501 } 1502 1503 /* reset the engine and set to writable */ 1504 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1505 WREG32(MC_SEQ_SUP_CNTL, 0x00000010); 1506 1507 /* load mc io regs */ 1508 for (i = 0; i < regs_size; i++) { 1509 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); 1510 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); 1511 } 1512 /* load the MC ucode */ 1513 fw_data = (const __be32 *)rdev->mc_fw->data; 1514 for (i = 0; i < ucode_size; i++) 1515 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); 1516 1517 /* put the engine back into the active state */ 1518 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1519 WREG32(MC_SEQ_SUP_CNTL, 0x00000004); 1520 WREG32(MC_SEQ_SUP_CNTL, 0x00000001); 1521 1522 /* wait for training to complete */ 1523 for (i = 0; i < rdev->usec_timeout; i++) { 1524 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0) 1525 break; 1526 udelay(1); 1527 } 1528 for (i = 0; i < rdev->usec_timeout; i++) { 1529 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1) 1530 break; 1531 udelay(1); 1532 } 1533 1534 if (running) 1535 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout); 1536 } 1537 1538 return 0; 1539 } 1540 1541 static int si_init_microcode(struct radeon_device *rdev) 1542 { 1543 const char *chip_name; 1544 const char *rlc_chip_name; 1545 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size; 1546 size_t smc_req_size; 1547 char fw_name[30]; 1548 int err; 1549 1550 DRM_DEBUG("\n"); 1551 1552 switch (rdev->family) { 1553 case CHIP_TAHITI: 1554 chip_name = "TAHITI"; 1555 rlc_chip_name = "TAHITI"; 1556 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1557 me_req_size = SI_PM4_UCODE_SIZE * 4; 1558 ce_req_size = SI_CE_UCODE_SIZE * 4; 1559 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1560 mc_req_size = SI_MC_UCODE_SIZE * 4; 1561 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4); 1562 break; 1563 case CHIP_PITCAIRN: 1564 chip_name = "PITCAIRN"; 1565 rlc_chip_name = "PITCAIRN"; 1566 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1567 me_req_size = SI_PM4_UCODE_SIZE * 4; 1568 ce_req_size = SI_CE_UCODE_SIZE * 4; 1569 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1570 mc_req_size = SI_MC_UCODE_SIZE * 4; 1571 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4); 1572 break; 1573 case CHIP_VERDE: 1574 chip_name = "VERDE"; 1575 rlc_chip_name = "VERDE"; 1576 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1577 me_req_size = SI_PM4_UCODE_SIZE * 4; 1578 ce_req_size = SI_CE_UCODE_SIZE * 4; 1579 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1580 mc_req_size = SI_MC_UCODE_SIZE * 4; 1581 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4); 1582 break; 1583 case CHIP_OLAND: 1584 chip_name = "OLAND"; 1585 rlc_chip_name = "OLAND"; 1586 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1587 me_req_size = SI_PM4_UCODE_SIZE * 4; 1588 ce_req_size = SI_CE_UCODE_SIZE * 4; 1589 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1590 mc_req_size = OLAND_MC_UCODE_SIZE * 4; 1591 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4); 1592 break; 1593 case CHIP_HAINAN: 1594 chip_name = "HAINAN"; 1595 rlc_chip_name = "HAINAN"; 1596 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1597 me_req_size = SI_PM4_UCODE_SIZE * 4; 1598 ce_req_size = SI_CE_UCODE_SIZE * 4; 1599 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1600 mc_req_size = OLAND_MC_UCODE_SIZE * 4; 1601 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4); 1602 break; 1603 default: BUG(); 1604 } 1605 1606 DRM_INFO("Loading %s Microcode\n", chip_name); 1607 1608 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name); 1609 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 1610 if (err) 1611 goto out; 1612 if (rdev->pfp_fw->datasize != pfp_req_size) { 1613 printk(KERN_ERR 1614 "si_cp: Bogus length %zu in firmware \"%s\"\n", 1615 rdev->pfp_fw->datasize, fw_name); 1616 err = -EINVAL; 1617 goto out; 1618 } 1619 1620 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name); 1621 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 1622 if (err) 1623 goto out; 1624 if (rdev->me_fw->datasize != me_req_size) { 1625 printk(KERN_ERR 1626 "si_cp: Bogus length %zu in firmware \"%s\"\n", 1627 rdev->me_fw->datasize, fw_name); 1628 err = -EINVAL; 1629 } 1630 1631 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", chip_name); 1632 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 1633 if (err) 1634 goto out; 1635 if (rdev->ce_fw->datasize != ce_req_size) { 1636 printk(KERN_ERR 1637 "si_cp: Bogus length %zu in firmware \"%s\"\n", 1638 rdev->ce_fw->datasize, fw_name); 1639 err = -EINVAL; 1640 } 1641 1642 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", rlc_chip_name); 1643 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 1644 if (err) 1645 goto out; 1646 if (rdev->rlc_fw->datasize != rlc_req_size) { 1647 printk(KERN_ERR 1648 "si_rlc: Bogus length %zu in firmware \"%s\"\n", 1649 rdev->rlc_fw->datasize, fw_name); 1650 err = -EINVAL; 1651 } 1652 1653 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", chip_name); 1654 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 1655 if (err) 1656 goto out; 1657 if (rdev->mc_fw->datasize != mc_req_size) { 1658 printk(KERN_ERR 1659 "si_mc: Bogus length %zu in firmware \"%s\"\n", 1660 rdev->mc_fw->datasize, fw_name); 1661 err = -EINVAL; 1662 } 1663 1664 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", chip_name); 1665 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 1666 if (err) { 1667 printk(KERN_ERR 1668 "smc: error loading firmware \"%s\"\n", 1669 fw_name); 1670 release_firmware(rdev->smc_fw); 1671 rdev->smc_fw = NULL; 1672 } else if (rdev->smc_fw->datasize != smc_req_size) { 1673 printk(KERN_ERR 1674 "si_smc: Bogus length %zu in firmware \"%s\"\n", 1675 rdev->smc_fw->datasize, fw_name); 1676 err = -EINVAL; 1677 } 1678 1679 out: 1680 if (err) { 1681 if (err != -EINVAL) 1682 printk(KERN_ERR 1683 "si_cp: Failed to load firmware \"%s\"\n", 1684 fw_name); 1685 release_firmware(rdev->pfp_fw); 1686 rdev->pfp_fw = NULL; 1687 release_firmware(rdev->me_fw); 1688 rdev->me_fw = NULL; 1689 release_firmware(rdev->ce_fw); 1690 rdev->ce_fw = NULL; 1691 release_firmware(rdev->rlc_fw); 1692 rdev->rlc_fw = NULL; 1693 release_firmware(rdev->mc_fw); 1694 rdev->mc_fw = NULL; 1695 release_firmware(rdev->smc_fw); 1696 rdev->smc_fw = NULL; 1697 } 1698 return err; 1699 } 1700 1701 /** 1702 * si_fini_microcode - drop the firmwares image references 1703 * 1704 * @rdev: radeon_device pointer 1705 * 1706 * Drop the pfp, me, rlc, mc and ce firmware image references. 1707 * Called at driver shutdown. 1708 */ 1709 static void si_fini_microcode(struct radeon_device *rdev) 1710 { 1711 release_firmware(rdev->pfp_fw); 1712 rdev->pfp_fw = NULL; 1713 release_firmware(rdev->me_fw); 1714 rdev->me_fw = NULL; 1715 release_firmware(rdev->rlc_fw); 1716 rdev->rlc_fw = NULL; 1717 release_firmware(rdev->mc_fw); 1718 rdev->mc_fw = NULL; 1719 release_firmware(rdev->smc_fw); 1720 rdev->smc_fw = NULL; 1721 release_firmware(rdev->ce_fw); 1722 rdev->ce_fw = NULL; 1723 } 1724 1725 /* watermark setup */ 1726 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev, 1727 struct radeon_crtc *radeon_crtc, 1728 struct drm_display_mode *mode, 1729 struct drm_display_mode *other_mode) 1730 { 1731 u32 tmp; 1732 /* 1733 * Line Buffer Setup 1734 * There are 3 line buffers, each one shared by 2 display controllers. 1735 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between 1736 * the display controllers. The paritioning is done via one of four 1737 * preset allocations specified in bits 21:20: 1738 * 0 - half lb 1739 * 2 - whole lb, other crtc must be disabled 1740 */ 1741 /* this can get tricky if we have two large displays on a paired group 1742 * of crtcs. Ideally for multiple large displays we'd assign them to 1743 * non-linked crtcs for maximum line buffer allocation. 1744 */ 1745 if (radeon_crtc->base.enabled && mode) { 1746 if (other_mode) 1747 tmp = 0; /* 1/2 */ 1748 else 1749 tmp = 2; /* whole */ 1750 } else 1751 tmp = 0; 1752 1753 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, 1754 DC_LB_MEMORY_CONFIG(tmp)); 1755 1756 if (radeon_crtc->base.enabled && mode) { 1757 switch (tmp) { 1758 case 0: 1759 default: 1760 return 4096 * 2; 1761 case 2: 1762 return 8192 * 2; 1763 } 1764 } 1765 1766 /* controller not enabled, so no lb used */ 1767 return 0; 1768 } 1769 1770 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev) 1771 { 1772 u32 tmp = RREG32(MC_SHARED_CHMAP); 1773 1774 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 1775 case 0: 1776 default: 1777 return 1; 1778 case 1: 1779 return 2; 1780 case 2: 1781 return 4; 1782 case 3: 1783 return 8; 1784 case 4: 1785 return 3; 1786 case 5: 1787 return 6; 1788 case 6: 1789 return 10; 1790 case 7: 1791 return 12; 1792 case 8: 1793 return 16; 1794 } 1795 } 1796 1797 struct dce6_wm_params { 1798 u32 dram_channels; /* number of dram channels */ 1799 u32 yclk; /* bandwidth per dram data pin in kHz */ 1800 u32 sclk; /* engine clock in kHz */ 1801 u32 disp_clk; /* display clock in kHz */ 1802 u32 src_width; /* viewport width */ 1803 u32 active_time; /* active display time in ns */ 1804 u32 blank_time; /* blank time in ns */ 1805 bool interlaced; /* mode is interlaced */ 1806 fixed20_12 vsc; /* vertical scale ratio */ 1807 u32 num_heads; /* number of active crtcs */ 1808 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 1809 u32 lb_size; /* line buffer allocated to pipe */ 1810 u32 vtaps; /* vertical scaler taps */ 1811 }; 1812 1813 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm) 1814 { 1815 /* Calculate raw DRAM Bandwidth */ 1816 fixed20_12 dram_efficiency; /* 0.7 */ 1817 fixed20_12 yclk, dram_channels, bandwidth; 1818 fixed20_12 a; 1819 1820 a.full = dfixed_const(1000); 1821 yclk.full = dfixed_const(wm->yclk); 1822 yclk.full = dfixed_div(yclk, a); 1823 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1824 a.full = dfixed_const(10); 1825 dram_efficiency.full = dfixed_const(7); 1826 dram_efficiency.full = dfixed_div(dram_efficiency, a); 1827 bandwidth.full = dfixed_mul(dram_channels, yclk); 1828 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 1829 1830 return dfixed_trunc(bandwidth); 1831 } 1832 1833 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm) 1834 { 1835 /* Calculate DRAM Bandwidth and the part allocated to display. */ 1836 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 1837 fixed20_12 yclk, dram_channels, bandwidth; 1838 fixed20_12 a; 1839 1840 a.full = dfixed_const(1000); 1841 yclk.full = dfixed_const(wm->yclk); 1842 yclk.full = dfixed_div(yclk, a); 1843 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1844 a.full = dfixed_const(10); 1845 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 1846 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 1847 bandwidth.full = dfixed_mul(dram_channels, yclk); 1848 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 1849 1850 return dfixed_trunc(bandwidth); 1851 } 1852 1853 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm) 1854 { 1855 /* Calculate the display Data return Bandwidth */ 1856 fixed20_12 return_efficiency; /* 0.8 */ 1857 fixed20_12 sclk, bandwidth; 1858 fixed20_12 a; 1859 1860 a.full = dfixed_const(1000); 1861 sclk.full = dfixed_const(wm->sclk); 1862 sclk.full = dfixed_div(sclk, a); 1863 a.full = dfixed_const(10); 1864 return_efficiency.full = dfixed_const(8); 1865 return_efficiency.full = dfixed_div(return_efficiency, a); 1866 a.full = dfixed_const(32); 1867 bandwidth.full = dfixed_mul(a, sclk); 1868 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 1869 1870 return dfixed_trunc(bandwidth); 1871 } 1872 1873 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm) 1874 { 1875 return 32; 1876 } 1877 1878 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm) 1879 { 1880 /* Calculate the DMIF Request Bandwidth */ 1881 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 1882 fixed20_12 disp_clk, sclk, bandwidth; 1883 fixed20_12 a, b1, b2; 1884 u32 min_bandwidth; 1885 1886 a.full = dfixed_const(1000); 1887 disp_clk.full = dfixed_const(wm->disp_clk); 1888 disp_clk.full = dfixed_div(disp_clk, a); 1889 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2); 1890 b1.full = dfixed_mul(a, disp_clk); 1891 1892 a.full = dfixed_const(1000); 1893 sclk.full = dfixed_const(wm->sclk); 1894 sclk.full = dfixed_div(sclk, a); 1895 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm)); 1896 b2.full = dfixed_mul(a, sclk); 1897 1898 a.full = dfixed_const(10); 1899 disp_clk_request_efficiency.full = dfixed_const(8); 1900 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 1901 1902 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2)); 1903 1904 a.full = dfixed_const(min_bandwidth); 1905 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency); 1906 1907 return dfixed_trunc(bandwidth); 1908 } 1909 1910 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm) 1911 { 1912 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 1913 u32 dram_bandwidth = dce6_dram_bandwidth(wm); 1914 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm); 1915 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm); 1916 1917 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 1918 } 1919 1920 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm) 1921 { 1922 /* Calculate the display mode Average Bandwidth 1923 * DisplayMode should contain the source and destination dimensions, 1924 * timing, etc. 1925 */ 1926 fixed20_12 bpp; 1927 fixed20_12 line_time; 1928 fixed20_12 src_width; 1929 fixed20_12 bandwidth; 1930 fixed20_12 a; 1931 1932 a.full = dfixed_const(1000); 1933 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 1934 line_time.full = dfixed_div(line_time, a); 1935 bpp.full = dfixed_const(wm->bytes_per_pixel); 1936 src_width.full = dfixed_const(wm->src_width); 1937 bandwidth.full = dfixed_mul(src_width, bpp); 1938 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 1939 bandwidth.full = dfixed_div(bandwidth, line_time); 1940 1941 return dfixed_trunc(bandwidth); 1942 } 1943 1944 static u32 dce6_latency_watermark(struct dce6_wm_params *wm) 1945 { 1946 /* First calcualte the latency in ns */ 1947 u32 mc_latency = 2000; /* 2000 ns. */ 1948 u32 available_bandwidth = dce6_available_bandwidth(wm); 1949 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 1950 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 1951 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 1952 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 1953 (wm->num_heads * cursor_line_pair_return_time); 1954 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 1955 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 1956 u32 tmp, dmif_size = 12288; 1957 fixed20_12 a, b, c; 1958 1959 if (wm->num_heads == 0) 1960 return 0; 1961 1962 a.full = dfixed_const(2); 1963 b.full = dfixed_const(1); 1964 if ((wm->vsc.full > a.full) || 1965 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 1966 (wm->vtaps >= 5) || 1967 ((wm->vsc.full >= a.full) && wm->interlaced)) 1968 max_src_lines_per_dst_line = 4; 1969 else 1970 max_src_lines_per_dst_line = 2; 1971 1972 a.full = dfixed_const(available_bandwidth); 1973 b.full = dfixed_const(wm->num_heads); 1974 a.full = dfixed_div(a, b); 1975 1976 b.full = dfixed_const(mc_latency + 512); 1977 c.full = dfixed_const(wm->disp_clk); 1978 b.full = dfixed_div(b, c); 1979 1980 c.full = dfixed_const(dmif_size); 1981 b.full = dfixed_div(c, b); 1982 1983 tmp = min(dfixed_trunc(a), dfixed_trunc(b)); 1984 1985 b.full = dfixed_const(1000); 1986 c.full = dfixed_const(wm->disp_clk); 1987 b.full = dfixed_div(c, b); 1988 c.full = dfixed_const(wm->bytes_per_pixel); 1989 b.full = dfixed_mul(b, c); 1990 1991 lb_fill_bw = min(tmp, dfixed_trunc(b)); 1992 1993 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 1994 b.full = dfixed_const(1000); 1995 c.full = dfixed_const(lb_fill_bw); 1996 b.full = dfixed_div(c, b); 1997 a.full = dfixed_div(a, b); 1998 line_fill_time = dfixed_trunc(a); 1999 2000 if (line_fill_time < wm->active_time) 2001 return latency; 2002 else 2003 return latency + (line_fill_time - wm->active_time); 2004 2005 } 2006 2007 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm) 2008 { 2009 if (dce6_average_bandwidth(wm) <= 2010 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads)) 2011 return true; 2012 else 2013 return false; 2014 }; 2015 2016 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm) 2017 { 2018 if (dce6_average_bandwidth(wm) <= 2019 (dce6_available_bandwidth(wm) / wm->num_heads)) 2020 return true; 2021 else 2022 return false; 2023 }; 2024 2025 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm) 2026 { 2027 u32 lb_partitions = wm->lb_size / wm->src_width; 2028 u32 line_time = wm->active_time + wm->blank_time; 2029 u32 latency_tolerant_lines; 2030 u32 latency_hiding; 2031 fixed20_12 a; 2032 2033 a.full = dfixed_const(1); 2034 if (wm->vsc.full > a.full) 2035 latency_tolerant_lines = 1; 2036 else { 2037 if (lb_partitions <= (wm->vtaps + 1)) 2038 latency_tolerant_lines = 1; 2039 else 2040 latency_tolerant_lines = 2; 2041 } 2042 2043 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 2044 2045 if (dce6_latency_watermark(wm) <= latency_hiding) 2046 return true; 2047 else 2048 return false; 2049 } 2050 2051 static void dce6_program_watermarks(struct radeon_device *rdev, 2052 struct radeon_crtc *radeon_crtc, 2053 u32 lb_size, u32 num_heads) 2054 { 2055 struct drm_display_mode *mode = &radeon_crtc->base.mode; 2056 struct dce6_wm_params wm_low, wm_high; 2057 u32 dram_channels; 2058 u32 pixel_period; 2059 u32 line_time = 0; 2060 u32 latency_watermark_a = 0, latency_watermark_b = 0; 2061 u32 priority_a_mark = 0, priority_b_mark = 0; 2062 u32 priority_a_cnt = PRIORITY_OFF; 2063 u32 priority_b_cnt = PRIORITY_OFF; 2064 u32 tmp, arb_control3; 2065 fixed20_12 a, b, c; 2066 2067 if (radeon_crtc->base.enabled && num_heads && mode) { 2068 pixel_period = 1000000 / (u32)mode->clock; 2069 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 2070 priority_a_cnt = 0; 2071 priority_b_cnt = 0; 2072 2073 if (rdev->family == CHIP_ARUBA) 2074 dram_channels = evergreen_get_number_of_dram_channels(rdev); 2075 else 2076 dram_channels = si_get_number_of_dram_channels(rdev); 2077 2078 /* watermark for high clocks */ 2079 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2080 wm_high.yclk = 2081 radeon_dpm_get_mclk(rdev, false) * 10; 2082 wm_high.sclk = 2083 radeon_dpm_get_sclk(rdev, false) * 10; 2084 } else { 2085 wm_high.yclk = rdev->pm.current_mclk * 10; 2086 wm_high.sclk = rdev->pm.current_sclk * 10; 2087 } 2088 2089 wm_high.disp_clk = mode->clock; 2090 wm_high.src_width = mode->crtc_hdisplay; 2091 wm_high.active_time = mode->crtc_hdisplay * pixel_period; 2092 wm_high.blank_time = line_time - wm_high.active_time; 2093 wm_high.interlaced = false; 2094 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2095 wm_high.interlaced = true; 2096 wm_high.vsc = radeon_crtc->vsc; 2097 wm_high.vtaps = 1; 2098 if (radeon_crtc->rmx_type != RMX_OFF) 2099 wm_high.vtaps = 2; 2100 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2101 wm_high.lb_size = lb_size; 2102 wm_high.dram_channels = dram_channels; 2103 wm_high.num_heads = num_heads; 2104 2105 /* watermark for low clocks */ 2106 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2107 wm_low.yclk = 2108 radeon_dpm_get_mclk(rdev, true) * 10; 2109 wm_low.sclk = 2110 radeon_dpm_get_sclk(rdev, true) * 10; 2111 } else { 2112 wm_low.yclk = rdev->pm.current_mclk * 10; 2113 wm_low.sclk = rdev->pm.current_sclk * 10; 2114 } 2115 2116 wm_low.disp_clk = mode->clock; 2117 wm_low.src_width = mode->crtc_hdisplay; 2118 wm_low.active_time = mode->crtc_hdisplay * pixel_period; 2119 wm_low.blank_time = line_time - wm_low.active_time; 2120 wm_low.interlaced = false; 2121 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2122 wm_low.interlaced = true; 2123 wm_low.vsc = radeon_crtc->vsc; 2124 wm_low.vtaps = 1; 2125 if (radeon_crtc->rmx_type != RMX_OFF) 2126 wm_low.vtaps = 2; 2127 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2128 wm_low.lb_size = lb_size; 2129 wm_low.dram_channels = dram_channels; 2130 wm_low.num_heads = num_heads; 2131 2132 /* set for high clocks */ 2133 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535); 2134 /* set for low clocks */ 2135 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535); 2136 2137 /* possibly force display priority to high */ 2138 /* should really do this at mode validation time... */ 2139 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 2140 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) || 2141 !dce6_check_latency_hiding(&wm_high) || 2142 (rdev->disp_priority == 2)) { 2143 DRM_DEBUG_KMS("force priority to high\n"); 2144 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2145 priority_b_cnt |= PRIORITY_ALWAYS_ON; 2146 } 2147 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 2148 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) || 2149 !dce6_check_latency_hiding(&wm_low) || 2150 (rdev->disp_priority == 2)) { 2151 DRM_DEBUG_KMS("force priority to high\n"); 2152 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2153 priority_b_cnt |= PRIORITY_ALWAYS_ON; 2154 } 2155 2156 a.full = dfixed_const(1000); 2157 b.full = dfixed_const(mode->clock); 2158 b.full = dfixed_div(b, a); 2159 c.full = dfixed_const(latency_watermark_a); 2160 c.full = dfixed_mul(c, b); 2161 c.full = dfixed_mul(c, radeon_crtc->hsc); 2162 c.full = dfixed_div(c, a); 2163 a.full = dfixed_const(16); 2164 c.full = dfixed_div(c, a); 2165 priority_a_mark = dfixed_trunc(c); 2166 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK; 2167 2168 a.full = dfixed_const(1000); 2169 b.full = dfixed_const(mode->clock); 2170 b.full = dfixed_div(b, a); 2171 c.full = dfixed_const(latency_watermark_b); 2172 c.full = dfixed_mul(c, b); 2173 c.full = dfixed_mul(c, radeon_crtc->hsc); 2174 c.full = dfixed_div(c, a); 2175 a.full = dfixed_const(16); 2176 c.full = dfixed_div(c, a); 2177 priority_b_mark = dfixed_trunc(c); 2178 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; 2179 } 2180 2181 /* select wm A */ 2182 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset); 2183 tmp = arb_control3; 2184 tmp &= ~LATENCY_WATERMARK_MASK(3); 2185 tmp |= LATENCY_WATERMARK_MASK(1); 2186 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp); 2187 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 2188 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 2189 LATENCY_HIGH_WATERMARK(line_time))); 2190 /* select wm B */ 2191 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset); 2192 tmp &= ~LATENCY_WATERMARK_MASK(3); 2193 tmp |= LATENCY_WATERMARK_MASK(2); 2194 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp); 2195 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 2196 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 2197 LATENCY_HIGH_WATERMARK(line_time))); 2198 /* restore original selection */ 2199 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3); 2200 2201 /* write the priority marks */ 2202 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); 2203 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); 2204 2205 /* save values for DPM */ 2206 radeon_crtc->line_time = line_time; 2207 radeon_crtc->wm_high = latency_watermark_a; 2208 radeon_crtc->wm_low = latency_watermark_b; 2209 } 2210 2211 void dce6_bandwidth_update(struct radeon_device *rdev) 2212 { 2213 struct drm_display_mode *mode0 = NULL; 2214 struct drm_display_mode *mode1 = NULL; 2215 u32 num_heads = 0, lb_size; 2216 int i; 2217 2218 radeon_update_display_priority(rdev); 2219 2220 for (i = 0; i < rdev->num_crtc; i++) { 2221 if (rdev->mode_info.crtcs[i]->base.enabled) 2222 num_heads++; 2223 } 2224 for (i = 0; i < rdev->num_crtc; i += 2) { 2225 mode0 = &rdev->mode_info.crtcs[i]->base.mode; 2226 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode; 2227 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1); 2228 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 2229 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0); 2230 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads); 2231 } 2232 } 2233 2234 /* 2235 * Core functions 2236 */ 2237 static void si_tiling_mode_table_init(struct radeon_device *rdev) 2238 { 2239 const u32 num_tile_mode_states = 32; 2240 u32 reg_offset, gb_tile_moden, split_equal_to_row_size; 2241 2242 switch (rdev->config.si.mem_row_size_in_kb) { 2243 case 1: 2244 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB; 2245 break; 2246 case 2: 2247 default: 2248 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB; 2249 break; 2250 case 4: 2251 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB; 2252 break; 2253 } 2254 2255 if ((rdev->family == CHIP_TAHITI) || 2256 (rdev->family == CHIP_PITCAIRN)) { 2257 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2258 switch (reg_offset) { 2259 case 0: /* non-AA compressed depth or any compressed stencil */ 2260 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2261 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2262 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2263 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2264 NUM_BANKS(ADDR_SURF_16_BANK) | 2265 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2266 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2267 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2268 break; 2269 case 1: /* 2xAA/4xAA compressed depth only */ 2270 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2271 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2272 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2273 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2274 NUM_BANKS(ADDR_SURF_16_BANK) | 2275 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2276 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2277 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2278 break; 2279 case 2: /* 8xAA compressed depth only */ 2280 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2281 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2282 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2283 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2284 NUM_BANKS(ADDR_SURF_16_BANK) | 2285 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2286 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2287 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2288 break; 2289 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */ 2290 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2291 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2292 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2293 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2294 NUM_BANKS(ADDR_SURF_16_BANK) | 2295 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2296 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2297 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2298 break; 2299 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */ 2300 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2301 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2302 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2303 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2304 NUM_BANKS(ADDR_SURF_16_BANK) | 2305 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2306 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2307 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2308 break; 2309 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */ 2310 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2311 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2312 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2313 TILE_SPLIT(split_equal_to_row_size) | 2314 NUM_BANKS(ADDR_SURF_16_BANK) | 2315 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2316 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2317 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2318 break; 2319 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */ 2320 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2321 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2322 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2323 TILE_SPLIT(split_equal_to_row_size) | 2324 NUM_BANKS(ADDR_SURF_16_BANK) | 2325 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2326 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2327 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2328 break; 2329 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */ 2330 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2331 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2332 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2333 TILE_SPLIT(split_equal_to_row_size) | 2334 NUM_BANKS(ADDR_SURF_16_BANK) | 2335 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2336 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2337 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2338 break; 2339 case 8: /* 1D and 1D Array Surfaces */ 2340 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2341 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2342 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2343 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2344 NUM_BANKS(ADDR_SURF_16_BANK) | 2345 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2346 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2347 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2348 break; 2349 case 9: /* Displayable maps. */ 2350 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2351 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2352 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2353 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2354 NUM_BANKS(ADDR_SURF_16_BANK) | 2355 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2358 break; 2359 case 10: /* Display 8bpp. */ 2360 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2361 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2362 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2363 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2364 NUM_BANKS(ADDR_SURF_16_BANK) | 2365 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2366 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2367 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2368 break; 2369 case 11: /* Display 16bpp. */ 2370 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2371 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2372 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2373 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2374 NUM_BANKS(ADDR_SURF_16_BANK) | 2375 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2376 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2377 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2378 break; 2379 case 12: /* Display 32bpp. */ 2380 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2381 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2382 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2383 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2384 NUM_BANKS(ADDR_SURF_16_BANK) | 2385 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2386 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2387 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2388 break; 2389 case 13: /* Thin. */ 2390 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2391 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2392 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2393 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2394 NUM_BANKS(ADDR_SURF_16_BANK) | 2395 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2396 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2397 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2398 break; 2399 case 14: /* Thin 8 bpp. */ 2400 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2401 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2402 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2403 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2404 NUM_BANKS(ADDR_SURF_16_BANK) | 2405 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2406 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2407 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2408 break; 2409 case 15: /* Thin 16 bpp. */ 2410 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2411 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2412 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2413 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2414 NUM_BANKS(ADDR_SURF_16_BANK) | 2415 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2416 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2417 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2418 break; 2419 case 16: /* Thin 32 bpp. */ 2420 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2421 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2422 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2423 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2424 NUM_BANKS(ADDR_SURF_16_BANK) | 2425 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2426 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2427 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2428 break; 2429 case 17: /* Thin 64 bpp. */ 2430 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2431 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2432 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2433 TILE_SPLIT(split_equal_to_row_size) | 2434 NUM_BANKS(ADDR_SURF_16_BANK) | 2435 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2436 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2437 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2438 break; 2439 case 21: /* 8 bpp PRT. */ 2440 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2441 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2442 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2443 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2444 NUM_BANKS(ADDR_SURF_16_BANK) | 2445 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2446 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2447 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2448 break; 2449 case 22: /* 16 bpp PRT */ 2450 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2451 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2452 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2453 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2454 NUM_BANKS(ADDR_SURF_16_BANK) | 2455 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2456 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2457 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2458 break; 2459 case 23: /* 32 bpp PRT */ 2460 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2461 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2462 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2463 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2464 NUM_BANKS(ADDR_SURF_16_BANK) | 2465 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2466 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2467 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2468 break; 2469 case 24: /* 64 bpp PRT */ 2470 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2471 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2472 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2473 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2474 NUM_BANKS(ADDR_SURF_16_BANK) | 2475 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2476 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2477 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2478 break; 2479 case 25: /* 128 bpp PRT */ 2480 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2481 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2482 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2483 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) | 2484 NUM_BANKS(ADDR_SURF_8_BANK) | 2485 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2488 break; 2489 default: 2490 gb_tile_moden = 0; 2491 break; 2492 } 2493 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden; 2494 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2495 } 2496 } else if ((rdev->family == CHIP_VERDE) || 2497 (rdev->family == CHIP_OLAND) || 2498 (rdev->family == CHIP_HAINAN)) { 2499 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2500 switch (reg_offset) { 2501 case 0: /* non-AA compressed depth or any compressed stencil */ 2502 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2503 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2504 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2505 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2506 NUM_BANKS(ADDR_SURF_16_BANK) | 2507 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2508 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2509 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2510 break; 2511 case 1: /* 2xAA/4xAA compressed depth only */ 2512 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2513 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2514 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2515 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2516 NUM_BANKS(ADDR_SURF_16_BANK) | 2517 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2520 break; 2521 case 2: /* 8xAA compressed depth only */ 2522 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2523 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2524 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2525 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2526 NUM_BANKS(ADDR_SURF_16_BANK) | 2527 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2528 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2529 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2530 break; 2531 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */ 2532 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2533 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2534 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2535 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2536 NUM_BANKS(ADDR_SURF_16_BANK) | 2537 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2538 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2539 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2540 break; 2541 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */ 2542 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2543 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2544 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2545 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2546 NUM_BANKS(ADDR_SURF_16_BANK) | 2547 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2548 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2549 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2550 break; 2551 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */ 2552 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2553 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2554 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2555 TILE_SPLIT(split_equal_to_row_size) | 2556 NUM_BANKS(ADDR_SURF_16_BANK) | 2557 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2558 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2559 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2560 break; 2561 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */ 2562 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2563 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2564 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2565 TILE_SPLIT(split_equal_to_row_size) | 2566 NUM_BANKS(ADDR_SURF_16_BANK) | 2567 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2568 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2569 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2570 break; 2571 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */ 2572 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2573 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2574 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2575 TILE_SPLIT(split_equal_to_row_size) | 2576 NUM_BANKS(ADDR_SURF_16_BANK) | 2577 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2578 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2579 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2580 break; 2581 case 8: /* 1D and 1D Array Surfaces */ 2582 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2583 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2584 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2585 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2586 NUM_BANKS(ADDR_SURF_16_BANK) | 2587 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2588 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2589 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2590 break; 2591 case 9: /* Displayable maps. */ 2592 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2593 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2594 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2595 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2596 NUM_BANKS(ADDR_SURF_16_BANK) | 2597 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2598 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2599 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2600 break; 2601 case 10: /* Display 8bpp. */ 2602 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2603 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2604 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2605 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2606 NUM_BANKS(ADDR_SURF_16_BANK) | 2607 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2608 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2609 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2610 break; 2611 case 11: /* Display 16bpp. */ 2612 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2613 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2614 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2615 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2616 NUM_BANKS(ADDR_SURF_16_BANK) | 2617 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2618 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2619 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2620 break; 2621 case 12: /* Display 32bpp. */ 2622 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2623 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2624 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2625 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2626 NUM_BANKS(ADDR_SURF_16_BANK) | 2627 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2628 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2629 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2630 break; 2631 case 13: /* Thin. */ 2632 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2633 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2634 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2635 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2636 NUM_BANKS(ADDR_SURF_16_BANK) | 2637 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2638 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2639 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2640 break; 2641 case 14: /* Thin 8 bpp. */ 2642 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2643 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2644 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2645 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2646 NUM_BANKS(ADDR_SURF_16_BANK) | 2647 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2648 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2649 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2650 break; 2651 case 15: /* Thin 16 bpp. */ 2652 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2653 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2654 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2655 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2656 NUM_BANKS(ADDR_SURF_16_BANK) | 2657 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2658 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2659 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2660 break; 2661 case 16: /* Thin 32 bpp. */ 2662 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2663 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2664 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2665 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2666 NUM_BANKS(ADDR_SURF_16_BANK) | 2667 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2668 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2669 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2670 break; 2671 case 17: /* Thin 64 bpp. */ 2672 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2673 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2674 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2675 TILE_SPLIT(split_equal_to_row_size) | 2676 NUM_BANKS(ADDR_SURF_16_BANK) | 2677 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2678 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2679 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2680 break; 2681 case 21: /* 8 bpp PRT. */ 2682 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2683 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2684 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2685 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2686 NUM_BANKS(ADDR_SURF_16_BANK) | 2687 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2688 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2689 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2690 break; 2691 case 22: /* 16 bpp PRT */ 2692 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2693 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2694 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2695 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2696 NUM_BANKS(ADDR_SURF_16_BANK) | 2697 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2698 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2699 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2700 break; 2701 case 23: /* 32 bpp PRT */ 2702 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2703 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2704 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2705 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2706 NUM_BANKS(ADDR_SURF_16_BANK) | 2707 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2708 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2709 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2710 break; 2711 case 24: /* 64 bpp PRT */ 2712 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2713 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2714 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2715 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2716 NUM_BANKS(ADDR_SURF_16_BANK) | 2717 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2718 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2719 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2720 break; 2721 case 25: /* 128 bpp PRT */ 2722 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2723 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2724 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2725 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) | 2726 NUM_BANKS(ADDR_SURF_8_BANK) | 2727 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2728 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2729 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2730 break; 2731 default: 2732 gb_tile_moden = 0; 2733 break; 2734 } 2735 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden; 2736 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2737 } 2738 } else 2739 DRM_ERROR("unknown asic: 0x%x\n", rdev->family); 2740 } 2741 2742 static void si_select_se_sh(struct radeon_device *rdev, 2743 u32 se_num, u32 sh_num) 2744 { 2745 u32 data = INSTANCE_BROADCAST_WRITES; 2746 2747 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) 2748 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES; 2749 else if (se_num == 0xffffffff) 2750 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num); 2751 else if (sh_num == 0xffffffff) 2752 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num); 2753 else 2754 data |= SH_INDEX(sh_num) | SE_INDEX(se_num); 2755 WREG32(GRBM_GFX_INDEX, data); 2756 } 2757 2758 static u32 si_create_bitmask(u32 bit_width) 2759 { 2760 u32 i, mask = 0; 2761 2762 for (i = 0; i < bit_width; i++) { 2763 mask <<= 1; 2764 mask |= 1; 2765 } 2766 return mask; 2767 } 2768 2769 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh) 2770 { 2771 u32 data, mask; 2772 2773 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 2774 if (data & 1) 2775 data &= INACTIVE_CUS_MASK; 2776 else 2777 data = 0; 2778 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG); 2779 2780 data >>= INACTIVE_CUS_SHIFT; 2781 2782 mask = si_create_bitmask(cu_per_sh); 2783 2784 return ~data & mask; 2785 } 2786 2787 static void si_setup_spi(struct radeon_device *rdev, 2788 u32 se_num, u32 sh_per_se, 2789 u32 cu_per_sh) 2790 { 2791 int i, j, k; 2792 u32 data, mask, active_cu; 2793 2794 for (i = 0; i < se_num; i++) { 2795 for (j = 0; j < sh_per_se; j++) { 2796 si_select_se_sh(rdev, i, j); 2797 data = RREG32(SPI_STATIC_THREAD_MGMT_3); 2798 active_cu = si_get_cu_enabled(rdev, cu_per_sh); 2799 2800 mask = 1; 2801 for (k = 0; k < 16; k++) { 2802 mask <<= k; 2803 if (active_cu & mask) { 2804 data &= ~mask; 2805 WREG32(SPI_STATIC_THREAD_MGMT_3, data); 2806 break; 2807 } 2808 } 2809 } 2810 } 2811 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 2812 } 2813 2814 static u32 si_get_rb_disabled(struct radeon_device *rdev, 2815 u32 max_rb_num, u32 se_num, 2816 u32 sh_per_se) 2817 { 2818 u32 data, mask; 2819 2820 data = RREG32(CC_RB_BACKEND_DISABLE); 2821 if (data & 1) 2822 data &= BACKEND_DISABLE_MASK; 2823 else 2824 data = 0; 2825 data |= RREG32(GC_USER_RB_BACKEND_DISABLE); 2826 2827 data >>= BACKEND_DISABLE_SHIFT; 2828 2829 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se); 2830 2831 return data & mask; 2832 } 2833 2834 static void si_setup_rb(struct radeon_device *rdev, 2835 u32 se_num, u32 sh_per_se, 2836 u32 max_rb_num) 2837 { 2838 int i, j; 2839 u32 data, mask; 2840 u32 disabled_rbs = 0; 2841 u32 enabled_rbs = 0; 2842 2843 for (i = 0; i < se_num; i++) { 2844 for (j = 0; j < sh_per_se; j++) { 2845 si_select_se_sh(rdev, i, j); 2846 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se); 2847 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH); 2848 } 2849 } 2850 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 2851 2852 mask = 1; 2853 for (i = 0; i < max_rb_num; i++) { 2854 if (!(disabled_rbs & mask)) 2855 enabled_rbs |= mask; 2856 mask <<= 1; 2857 } 2858 2859 for (i = 0; i < se_num; i++) { 2860 si_select_se_sh(rdev, i, 0xffffffff); 2861 data = 0; 2862 for (j = 0; j < sh_per_se; j++) { 2863 switch (enabled_rbs & 3) { 2864 case 1: 2865 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2); 2866 break; 2867 case 2: 2868 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2); 2869 break; 2870 case 3: 2871 default: 2872 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2); 2873 break; 2874 } 2875 enabled_rbs >>= 2; 2876 } 2877 WREG32(PA_SC_RASTER_CONFIG, data); 2878 } 2879 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 2880 } 2881 2882 static void si_gpu_init(struct radeon_device *rdev) 2883 { 2884 u32 gb_addr_config = 0; 2885 u32 mc_shared_chmap, mc_arb_ramcfg; 2886 u32 sx_debug_1; 2887 u32 hdp_host_path_cntl; 2888 u32 tmp; 2889 int i, j; 2890 2891 switch (rdev->family) { 2892 case CHIP_TAHITI: 2893 rdev->config.si.max_shader_engines = 2; 2894 rdev->config.si.max_tile_pipes = 12; 2895 rdev->config.si.max_cu_per_sh = 8; 2896 rdev->config.si.max_sh_per_se = 2; 2897 rdev->config.si.max_backends_per_se = 4; 2898 rdev->config.si.max_texture_channel_caches = 12; 2899 rdev->config.si.max_gprs = 256; 2900 rdev->config.si.max_gs_threads = 32; 2901 rdev->config.si.max_hw_contexts = 8; 2902 2903 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2904 rdev->config.si.sc_prim_fifo_size_backend = 0x100; 2905 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2906 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2907 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN; 2908 break; 2909 case CHIP_PITCAIRN: 2910 rdev->config.si.max_shader_engines = 2; 2911 rdev->config.si.max_tile_pipes = 8; 2912 rdev->config.si.max_cu_per_sh = 5; 2913 rdev->config.si.max_sh_per_se = 2; 2914 rdev->config.si.max_backends_per_se = 4; 2915 rdev->config.si.max_texture_channel_caches = 8; 2916 rdev->config.si.max_gprs = 256; 2917 rdev->config.si.max_gs_threads = 32; 2918 rdev->config.si.max_hw_contexts = 8; 2919 2920 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2921 rdev->config.si.sc_prim_fifo_size_backend = 0x100; 2922 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2923 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2924 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN; 2925 break; 2926 case CHIP_VERDE: 2927 default: 2928 rdev->config.si.max_shader_engines = 1; 2929 rdev->config.si.max_tile_pipes = 4; 2930 rdev->config.si.max_cu_per_sh = 5; 2931 rdev->config.si.max_sh_per_se = 2; 2932 rdev->config.si.max_backends_per_se = 4; 2933 rdev->config.si.max_texture_channel_caches = 4; 2934 rdev->config.si.max_gprs = 256; 2935 rdev->config.si.max_gs_threads = 32; 2936 rdev->config.si.max_hw_contexts = 8; 2937 2938 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2939 rdev->config.si.sc_prim_fifo_size_backend = 0x40; 2940 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2941 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2942 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN; 2943 break; 2944 case CHIP_OLAND: 2945 rdev->config.si.max_shader_engines = 1; 2946 rdev->config.si.max_tile_pipes = 4; 2947 rdev->config.si.max_cu_per_sh = 6; 2948 rdev->config.si.max_sh_per_se = 1; 2949 rdev->config.si.max_backends_per_se = 2; 2950 rdev->config.si.max_texture_channel_caches = 4; 2951 rdev->config.si.max_gprs = 256; 2952 rdev->config.si.max_gs_threads = 16; 2953 rdev->config.si.max_hw_contexts = 8; 2954 2955 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2956 rdev->config.si.sc_prim_fifo_size_backend = 0x40; 2957 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2958 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2959 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN; 2960 break; 2961 case CHIP_HAINAN: 2962 rdev->config.si.max_shader_engines = 1; 2963 rdev->config.si.max_tile_pipes = 4; 2964 rdev->config.si.max_cu_per_sh = 5; 2965 rdev->config.si.max_sh_per_se = 1; 2966 rdev->config.si.max_backends_per_se = 1; 2967 rdev->config.si.max_texture_channel_caches = 2; 2968 rdev->config.si.max_gprs = 256; 2969 rdev->config.si.max_gs_threads = 16; 2970 rdev->config.si.max_hw_contexts = 8; 2971 2972 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2973 rdev->config.si.sc_prim_fifo_size_backend = 0x40; 2974 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2975 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2976 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN; 2977 break; 2978 } 2979 2980 /* Initialize HDP */ 2981 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 2982 WREG32((0x2c14 + j), 0x00000000); 2983 WREG32((0x2c18 + j), 0x00000000); 2984 WREG32((0x2c1c + j), 0x00000000); 2985 WREG32((0x2c20 + j), 0x00000000); 2986 WREG32((0x2c24 + j), 0x00000000); 2987 } 2988 2989 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 2990 2991 evergreen_fix_pci_max_read_req_size(rdev); 2992 2993 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 2994 2995 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 2996 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 2997 2998 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes; 2999 rdev->config.si.mem_max_burst_length_bytes = 256; 3000 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT; 3001 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 3002 if (rdev->config.si.mem_row_size_in_kb > 4) 3003 rdev->config.si.mem_row_size_in_kb = 4; 3004 /* XXX use MC settings? */ 3005 rdev->config.si.shader_engine_tile_size = 32; 3006 rdev->config.si.num_gpus = 1; 3007 rdev->config.si.multi_gpu_tile_size = 64; 3008 3009 /* fix up row size */ 3010 gb_addr_config &= ~ROW_SIZE_MASK; 3011 switch (rdev->config.si.mem_row_size_in_kb) { 3012 case 1: 3013 default: 3014 gb_addr_config |= ROW_SIZE(0); 3015 break; 3016 case 2: 3017 gb_addr_config |= ROW_SIZE(1); 3018 break; 3019 case 4: 3020 gb_addr_config |= ROW_SIZE(2); 3021 break; 3022 } 3023 3024 /* setup tiling info dword. gb_addr_config is not adequate since it does 3025 * not have bank info, so create a custom tiling dword. 3026 * bits 3:0 num_pipes 3027 * bits 7:4 num_banks 3028 * bits 11:8 group_size 3029 * bits 15:12 row_size 3030 */ 3031 rdev->config.si.tile_config = 0; 3032 switch (rdev->config.si.num_tile_pipes) { 3033 case 1: 3034 rdev->config.si.tile_config |= (0 << 0); 3035 break; 3036 case 2: 3037 rdev->config.si.tile_config |= (1 << 0); 3038 break; 3039 case 4: 3040 rdev->config.si.tile_config |= (2 << 0); 3041 break; 3042 case 8: 3043 default: 3044 /* XXX what about 12? */ 3045 rdev->config.si.tile_config |= (3 << 0); 3046 break; 3047 } 3048 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) { 3049 case 0: /* four banks */ 3050 rdev->config.si.tile_config |= 0 << 4; 3051 break; 3052 case 1: /* eight banks */ 3053 rdev->config.si.tile_config |= 1 << 4; 3054 break; 3055 case 2: /* sixteen banks */ 3056 default: 3057 rdev->config.si.tile_config |= 2 << 4; 3058 break; 3059 } 3060 rdev->config.si.tile_config |= 3061 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8; 3062 rdev->config.si.tile_config |= 3063 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12; 3064 3065 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3066 WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 3067 WREG32(DMIF_ADDR_CALC, gb_addr_config); 3068 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3069 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config); 3070 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config); 3071 if (rdev->has_uvd) { 3072 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3073 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3074 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3075 } 3076 3077 si_tiling_mode_table_init(rdev); 3078 3079 si_setup_rb(rdev, rdev->config.si.max_shader_engines, 3080 rdev->config.si.max_sh_per_se, 3081 rdev->config.si.max_backends_per_se); 3082 3083 si_setup_spi(rdev, rdev->config.si.max_shader_engines, 3084 rdev->config.si.max_sh_per_se, 3085 rdev->config.si.max_cu_per_sh); 3086 3087 3088 /* set HW defaults for 3D engine */ 3089 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | 3090 ROQ_IB2_START(0x2b))); 3091 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); 3092 3093 sx_debug_1 = RREG32(SX_DEBUG_1); 3094 WREG32(SX_DEBUG_1, sx_debug_1); 3095 3096 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3097 3098 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) | 3099 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) | 3100 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) | 3101 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size))); 3102 3103 WREG32(VGT_NUM_INSTANCES, 1); 3104 3105 WREG32(CP_PERFMON_CNTL, 0); 3106 3107 WREG32(SQ_CONFIG, 0); 3108 3109 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3110 FORCE_EOV_MAX_REZ_CNT(255))); 3111 3112 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) | 3113 AUTO_INVLD_EN(ES_AND_GS_AUTO)); 3114 3115 WREG32(VGT_GS_VERTEX_REUSE, 16); 3116 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3117 3118 WREG32(CB_PERFCOUNTER0_SELECT0, 0); 3119 WREG32(CB_PERFCOUNTER0_SELECT1, 0); 3120 WREG32(CB_PERFCOUNTER1_SELECT0, 0); 3121 WREG32(CB_PERFCOUNTER1_SELECT1, 0); 3122 WREG32(CB_PERFCOUNTER2_SELECT0, 0); 3123 WREG32(CB_PERFCOUNTER2_SELECT1, 0); 3124 WREG32(CB_PERFCOUNTER3_SELECT0, 0); 3125 WREG32(CB_PERFCOUNTER3_SELECT1, 0); 3126 3127 tmp = RREG32(HDP_MISC_CNTL); 3128 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3129 WREG32(HDP_MISC_CNTL, tmp); 3130 3131 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3132 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3133 3134 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3135 3136 udelay(50); 3137 } 3138 3139 /* 3140 * GPU scratch registers helpers function. 3141 */ 3142 static void si_scratch_init(struct radeon_device *rdev) 3143 { 3144 int i; 3145 3146 rdev->scratch.num_reg = 7; 3147 rdev->scratch.reg_base = SCRATCH_REG0; 3148 for (i = 0; i < rdev->scratch.num_reg; i++) { 3149 rdev->scratch.free[i] = true; 3150 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); 3151 } 3152 } 3153 3154 void si_fence_ring_emit(struct radeon_device *rdev, 3155 struct radeon_fence *fence) 3156 { 3157 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3158 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3159 3160 /* flush read cache over gart */ 3161 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3162 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2); 3163 radeon_ring_write(ring, 0); 3164 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 3165 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 3166 PACKET3_TC_ACTION_ENA | 3167 PACKET3_SH_KCACHE_ACTION_ENA | 3168 PACKET3_SH_ICACHE_ACTION_ENA); 3169 radeon_ring_write(ring, 0xFFFFFFFF); 3170 radeon_ring_write(ring, 0); 3171 radeon_ring_write(ring, 10); /* poll interval */ 3172 /* EVENT_WRITE_EOP - flush caches, send int */ 3173 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3174 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5)); 3175 radeon_ring_write(ring, addr & 0xffffffff); 3176 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2)); 3177 radeon_ring_write(ring, fence->seq); 3178 radeon_ring_write(ring, 0); 3179 } 3180 3181 /* 3182 * IB stuff 3183 */ 3184 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 3185 { 3186 struct radeon_ring *ring = &rdev->ring[ib->ring]; 3187 u32 header; 3188 3189 if (ib->is_const_ib) { 3190 /* set switch buffer packet before const IB */ 3191 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 3192 radeon_ring_write(ring, 0); 3193 3194 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 3195 } else { 3196 u32 next_rptr; 3197 if (ring->rptr_save_reg) { 3198 next_rptr = ring->wptr + 3 + 4 + 8; 3199 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3200 radeon_ring_write(ring, ((ring->rptr_save_reg - 3201 PACKET3_SET_CONFIG_REG_START) >> 2)); 3202 radeon_ring_write(ring, next_rptr); 3203 } else if (rdev->wb.enabled) { 3204 next_rptr = ring->wptr + 5 + 4 + 8; 3205 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 3206 radeon_ring_write(ring, (1 << 8)); 3207 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 3208 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff); 3209 radeon_ring_write(ring, next_rptr); 3210 } 3211 3212 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 3213 } 3214 3215 radeon_ring_write(ring, header); 3216 radeon_ring_write(ring, 3217 #ifdef __BIG_ENDIAN 3218 (2 << 0) | 3219 #endif 3220 (ib->gpu_addr & 0xFFFFFFFC)); 3221 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 3222 radeon_ring_write(ring, ib->length_dw | 3223 (ib->vm ? (ib->vm->id << 24) : 0)); 3224 3225 if (!ib->is_const_ib) { 3226 /* flush read cache over gart for this vmid */ 3227 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3228 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2); 3229 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0); 3230 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 3231 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 3232 PACKET3_TC_ACTION_ENA | 3233 PACKET3_SH_KCACHE_ACTION_ENA | 3234 PACKET3_SH_ICACHE_ACTION_ENA); 3235 radeon_ring_write(ring, 0xFFFFFFFF); 3236 radeon_ring_write(ring, 0); 3237 radeon_ring_write(ring, 10); /* poll interval */ 3238 } 3239 } 3240 3241 /* 3242 * CP. 3243 */ 3244 static void si_cp_enable(struct radeon_device *rdev, bool enable) 3245 { 3246 if (enable) 3247 WREG32(CP_ME_CNTL, 0); 3248 else { 3249 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 3250 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT)); 3251 WREG32(SCRATCH_UMSK, 0); 3252 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 3253 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 3254 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 3255 } 3256 udelay(50); 3257 } 3258 3259 static int si_cp_load_microcode(struct radeon_device *rdev) 3260 { 3261 const __be32 *fw_data; 3262 int i; 3263 3264 if (!rdev->me_fw || !rdev->pfp_fw) 3265 return -EINVAL; 3266 3267 si_cp_enable(rdev, false); 3268 3269 /* PFP */ 3270 fw_data = (const __be32 *)rdev->pfp_fw->data; 3271 WREG32(CP_PFP_UCODE_ADDR, 0); 3272 for (i = 0; i < SI_PFP_UCODE_SIZE; i++) 3273 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 3274 WREG32(CP_PFP_UCODE_ADDR, 0); 3275 3276 /* CE */ 3277 fw_data = (const __be32 *)rdev->ce_fw->data; 3278 WREG32(CP_CE_UCODE_ADDR, 0); 3279 for (i = 0; i < SI_CE_UCODE_SIZE; i++) 3280 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); 3281 WREG32(CP_CE_UCODE_ADDR, 0); 3282 3283 /* ME */ 3284 fw_data = (const __be32 *)rdev->me_fw->data; 3285 WREG32(CP_ME_RAM_WADDR, 0); 3286 for (i = 0; i < SI_PM4_UCODE_SIZE; i++) 3287 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 3288 WREG32(CP_ME_RAM_WADDR, 0); 3289 3290 WREG32(CP_PFP_UCODE_ADDR, 0); 3291 WREG32(CP_CE_UCODE_ADDR, 0); 3292 WREG32(CP_ME_RAM_WADDR, 0); 3293 WREG32(CP_ME_RAM_RADDR, 0); 3294 return 0; 3295 } 3296 3297 static int si_cp_start(struct radeon_device *rdev) 3298 { 3299 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3300 int r, i; 3301 3302 r = radeon_ring_lock(rdev, ring, 7 + 4); 3303 if (r) { 3304 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3305 return r; 3306 } 3307 /* init the CP */ 3308 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5)); 3309 radeon_ring_write(ring, 0x1); 3310 radeon_ring_write(ring, 0x0); 3311 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1); 3312 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 3313 radeon_ring_write(ring, 0); 3314 radeon_ring_write(ring, 0); 3315 3316 /* init the CE partitions */ 3317 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3318 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3319 radeon_ring_write(ring, 0xc000); 3320 radeon_ring_write(ring, 0xe000); 3321 radeon_ring_unlock_commit(rdev, ring); 3322 3323 si_cp_enable(rdev, true); 3324 3325 r = radeon_ring_lock(rdev, ring, si_default_size + 10); 3326 if (r) { 3327 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3328 return r; 3329 } 3330 3331 /* setup clear context state */ 3332 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3333 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3334 3335 for (i = 0; i < si_default_size; i++) 3336 radeon_ring_write(ring, si_default_state[i]); 3337 3338 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3339 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3340 3341 /* set clear context state */ 3342 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3343 radeon_ring_write(ring, 0); 3344 3345 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 3346 radeon_ring_write(ring, 0x00000316); 3347 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 3348 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */ 3349 3350 radeon_ring_unlock_commit(rdev, ring); 3351 3352 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) { 3353 ring = &rdev->ring[i]; 3354 r = radeon_ring_lock(rdev, ring, 2); 3355 3356 /* clear the compute context state */ 3357 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0)); 3358 radeon_ring_write(ring, 0); 3359 3360 radeon_ring_unlock_commit(rdev, ring); 3361 } 3362 3363 return 0; 3364 } 3365 3366 static void si_cp_fini(struct radeon_device *rdev) 3367 { 3368 struct radeon_ring *ring; 3369 si_cp_enable(rdev, false); 3370 3371 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3372 radeon_ring_fini(rdev, ring); 3373 radeon_scratch_free(rdev, ring->rptr_save_reg); 3374 3375 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 3376 radeon_ring_fini(rdev, ring); 3377 radeon_scratch_free(rdev, ring->rptr_save_reg); 3378 3379 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 3380 radeon_ring_fini(rdev, ring); 3381 radeon_scratch_free(rdev, ring->rptr_save_reg); 3382 } 3383 3384 static int si_cp_resume(struct radeon_device *rdev) 3385 { 3386 struct radeon_ring *ring; 3387 u32 tmp; 3388 u32 rb_bufsz; 3389 int r; 3390 3391 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */ 3392 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP | 3393 SOFT_RESET_PA | 3394 SOFT_RESET_VGT | 3395 SOFT_RESET_SPI | 3396 SOFT_RESET_SX)); 3397 RREG32(GRBM_SOFT_RESET); 3398 mdelay(15); 3399 WREG32(GRBM_SOFT_RESET, 0); 3400 RREG32(GRBM_SOFT_RESET); 3401 3402 WREG32(CP_SEM_WAIT_TIMER, 0x0); 3403 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 3404 3405 /* Set the write pointer delay */ 3406 WREG32(CP_RB_WPTR_DELAY, 0); 3407 3408 WREG32(CP_DEBUG, 0); 3409 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 3410 3411 /* ring 0 - compute and gfx */ 3412 /* Set ring buffer size */ 3413 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3414 rb_bufsz = drm_order(ring->ring_size / 8); 3415 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3416 #ifdef __BIG_ENDIAN 3417 tmp |= BUF_SWAP_32BIT; 3418 #endif 3419 WREG32(CP_RB0_CNTL, tmp); 3420 3421 /* Initialize the ring buffer's read and write pointers */ 3422 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA); 3423 ring->wptr = 0; 3424 WREG32(CP_RB0_WPTR, ring->wptr); 3425 3426 /* set the wb address whether it's enabled or not */ 3427 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); 3428 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 3429 3430 if (rdev->wb.enabled) 3431 WREG32(SCRATCH_UMSK, 0xff); 3432 else { 3433 tmp |= RB_NO_UPDATE; 3434 WREG32(SCRATCH_UMSK, 0); 3435 } 3436 3437 mdelay(1); 3438 WREG32(CP_RB0_CNTL, tmp); 3439 3440 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8); 3441 3442 ring->rptr = RREG32(CP_RB0_RPTR); 3443 3444 /* ring1 - compute only */ 3445 /* Set ring buffer size */ 3446 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 3447 rb_bufsz = drm_order(ring->ring_size / 8); 3448 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3449 #ifdef __BIG_ENDIAN 3450 tmp |= BUF_SWAP_32BIT; 3451 #endif 3452 WREG32(CP_RB1_CNTL, tmp); 3453 3454 /* Initialize the ring buffer's read and write pointers */ 3455 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA); 3456 ring->wptr = 0; 3457 WREG32(CP_RB1_WPTR, ring->wptr); 3458 3459 /* set the wb address whether it's enabled or not */ 3460 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC); 3461 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF); 3462 3463 mdelay(1); 3464 WREG32(CP_RB1_CNTL, tmp); 3465 3466 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8); 3467 3468 ring->rptr = RREG32(CP_RB1_RPTR); 3469 3470 /* ring2 - compute only */ 3471 /* Set ring buffer size */ 3472 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 3473 rb_bufsz = drm_order(ring->ring_size / 8); 3474 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3475 #ifdef __BIG_ENDIAN 3476 tmp |= BUF_SWAP_32BIT; 3477 #endif 3478 WREG32(CP_RB2_CNTL, tmp); 3479 3480 /* Initialize the ring buffer's read and write pointers */ 3481 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA); 3482 ring->wptr = 0; 3483 WREG32(CP_RB2_WPTR, ring->wptr); 3484 3485 /* set the wb address whether it's enabled or not */ 3486 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC); 3487 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF); 3488 3489 mdelay(1); 3490 WREG32(CP_RB2_CNTL, tmp); 3491 3492 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8); 3493 3494 ring->rptr = RREG32(CP_RB2_RPTR); 3495 3496 /* start the rings */ 3497 si_cp_start(rdev); 3498 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; 3499 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true; 3500 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true; 3501 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 3502 if (r) { 3503 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 3504 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 3505 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 3506 return r; 3507 } 3508 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]); 3509 if (r) { 3510 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 3511 } 3512 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]); 3513 if (r) { 3514 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 3515 } 3516 3517 return 0; 3518 } 3519 3520 static u32 si_gpu_check_soft_reset(struct radeon_device *rdev) 3521 { 3522 u32 reset_mask = 0; 3523 u32 tmp; 3524 3525 /* GRBM_STATUS */ 3526 tmp = RREG32(GRBM_STATUS); 3527 if (tmp & (PA_BUSY | SC_BUSY | 3528 BCI_BUSY | SX_BUSY | 3529 TA_BUSY | VGT_BUSY | 3530 DB_BUSY | CB_BUSY | 3531 GDS_BUSY | SPI_BUSY | 3532 IA_BUSY | IA_BUSY_NO_DMA)) 3533 reset_mask |= RADEON_RESET_GFX; 3534 3535 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | 3536 CP_BUSY | CP_COHERENCY_BUSY)) 3537 reset_mask |= RADEON_RESET_CP; 3538 3539 if (tmp & GRBM_EE_BUSY) 3540 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; 3541 3542 /* GRBM_STATUS2 */ 3543 tmp = RREG32(GRBM_STATUS2); 3544 if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) 3545 reset_mask |= RADEON_RESET_RLC; 3546 3547 /* DMA_STATUS_REG 0 */ 3548 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET); 3549 if (!(tmp & DMA_IDLE)) 3550 reset_mask |= RADEON_RESET_DMA; 3551 3552 /* DMA_STATUS_REG 1 */ 3553 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET); 3554 if (!(tmp & DMA_IDLE)) 3555 reset_mask |= RADEON_RESET_DMA1; 3556 3557 /* SRBM_STATUS2 */ 3558 tmp = RREG32(SRBM_STATUS2); 3559 if (tmp & DMA_BUSY) 3560 reset_mask |= RADEON_RESET_DMA; 3561 3562 if (tmp & DMA1_BUSY) 3563 reset_mask |= RADEON_RESET_DMA1; 3564 3565 /* SRBM_STATUS */ 3566 tmp = RREG32(SRBM_STATUS); 3567 3568 if (tmp & IH_BUSY) 3569 reset_mask |= RADEON_RESET_IH; 3570 3571 if (tmp & SEM_BUSY) 3572 reset_mask |= RADEON_RESET_SEM; 3573 3574 if (tmp & GRBM_RQ_PENDING) 3575 reset_mask |= RADEON_RESET_GRBM; 3576 3577 if (tmp & VMC_BUSY) 3578 reset_mask |= RADEON_RESET_VMC; 3579 3580 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 3581 MCC_BUSY | MCD_BUSY)) 3582 reset_mask |= RADEON_RESET_MC; 3583 3584 if (evergreen_is_display_hung(rdev)) 3585 reset_mask |= RADEON_RESET_DISPLAY; 3586 3587 /* VM_L2_STATUS */ 3588 tmp = RREG32(VM_L2_STATUS); 3589 if (tmp & L2_BUSY) 3590 reset_mask |= RADEON_RESET_VMC; 3591 3592 /* Skip MC reset as it's mostly likely not hung, just busy */ 3593 if (reset_mask & RADEON_RESET_MC) { 3594 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 3595 reset_mask &= ~RADEON_RESET_MC; 3596 } 3597 3598 return reset_mask; 3599 } 3600 3601 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 3602 { 3603 struct evergreen_mc_save save; 3604 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 3605 u32 tmp; 3606 3607 if (reset_mask == 0) 3608 return; 3609 3610 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 3611 3612 evergreen_print_gpu_status_regs(rdev); 3613 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 3614 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); 3615 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 3616 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); 3617 3618 /* Disable CP parsing/prefetching */ 3619 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 3620 3621 if (reset_mask & RADEON_RESET_DMA) { 3622 /* dma0 */ 3623 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); 3624 tmp &= ~DMA_RB_ENABLE; 3625 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp); 3626 } 3627 if (reset_mask & RADEON_RESET_DMA1) { 3628 /* dma1 */ 3629 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); 3630 tmp &= ~DMA_RB_ENABLE; 3631 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp); 3632 } 3633 3634 udelay(50); 3635 3636 evergreen_mc_stop(rdev, &save); 3637 if (evergreen_mc_wait_for_idle(rdev)) { 3638 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3639 } 3640 3641 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) { 3642 grbm_soft_reset = SOFT_RESET_CB | 3643 SOFT_RESET_DB | 3644 SOFT_RESET_GDS | 3645 SOFT_RESET_PA | 3646 SOFT_RESET_SC | 3647 SOFT_RESET_BCI | 3648 SOFT_RESET_SPI | 3649 SOFT_RESET_SX | 3650 SOFT_RESET_TC | 3651 SOFT_RESET_TA | 3652 SOFT_RESET_VGT | 3653 SOFT_RESET_IA; 3654 } 3655 3656 if (reset_mask & RADEON_RESET_CP) { 3657 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT; 3658 3659 srbm_soft_reset |= SOFT_RESET_GRBM; 3660 } 3661 3662 if (reset_mask & RADEON_RESET_DMA) 3663 srbm_soft_reset |= SOFT_RESET_DMA; 3664 3665 if (reset_mask & RADEON_RESET_DMA1) 3666 srbm_soft_reset |= SOFT_RESET_DMA1; 3667 3668 if (reset_mask & RADEON_RESET_DISPLAY) 3669 srbm_soft_reset |= SOFT_RESET_DC; 3670 3671 if (reset_mask & RADEON_RESET_RLC) 3672 grbm_soft_reset |= SOFT_RESET_RLC; 3673 3674 if (reset_mask & RADEON_RESET_SEM) 3675 srbm_soft_reset |= SOFT_RESET_SEM; 3676 3677 if (reset_mask & RADEON_RESET_IH) 3678 srbm_soft_reset |= SOFT_RESET_IH; 3679 3680 if (reset_mask & RADEON_RESET_GRBM) 3681 srbm_soft_reset |= SOFT_RESET_GRBM; 3682 3683 if (reset_mask & RADEON_RESET_VMC) 3684 srbm_soft_reset |= SOFT_RESET_VMC; 3685 3686 if (reset_mask & RADEON_RESET_MC) 3687 srbm_soft_reset |= SOFT_RESET_MC; 3688 3689 if (grbm_soft_reset) { 3690 tmp = RREG32(GRBM_SOFT_RESET); 3691 tmp |= grbm_soft_reset; 3692 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 3693 WREG32(GRBM_SOFT_RESET, tmp); 3694 tmp = RREG32(GRBM_SOFT_RESET); 3695 3696 udelay(50); 3697 3698 tmp &= ~grbm_soft_reset; 3699 WREG32(GRBM_SOFT_RESET, tmp); 3700 tmp = RREG32(GRBM_SOFT_RESET); 3701 } 3702 3703 if (srbm_soft_reset) { 3704 tmp = RREG32(SRBM_SOFT_RESET); 3705 tmp |= srbm_soft_reset; 3706 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 3707 WREG32(SRBM_SOFT_RESET, tmp); 3708 tmp = RREG32(SRBM_SOFT_RESET); 3709 3710 udelay(50); 3711 3712 tmp &= ~srbm_soft_reset; 3713 WREG32(SRBM_SOFT_RESET, tmp); 3714 tmp = RREG32(SRBM_SOFT_RESET); 3715 } 3716 3717 /* Wait a little for things to settle down */ 3718 udelay(50); 3719 3720 evergreen_mc_resume(rdev, &save); 3721 udelay(50); 3722 3723 evergreen_print_gpu_status_regs(rdev); 3724 } 3725 3726 int si_asic_reset(struct radeon_device *rdev) 3727 { 3728 u32 reset_mask; 3729 3730 reset_mask = si_gpu_check_soft_reset(rdev); 3731 3732 if (reset_mask) 3733 r600_set_bios_scratch_engine_hung(rdev, true); 3734 3735 si_gpu_soft_reset(rdev, reset_mask); 3736 3737 reset_mask = si_gpu_check_soft_reset(rdev); 3738 3739 if (!reset_mask) 3740 r600_set_bios_scratch_engine_hung(rdev, false); 3741 3742 return 0; 3743 } 3744 3745 /** 3746 * si_gfx_is_lockup - Check if the GFX engine is locked up 3747 * 3748 * @rdev: radeon_device pointer 3749 * @ring: radeon_ring structure holding ring information 3750 * 3751 * Check if the GFX engine is locked up. 3752 * Returns true if the engine appears to be locked up, false if not. 3753 */ 3754 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 3755 { 3756 u32 reset_mask = si_gpu_check_soft_reset(rdev); 3757 3758 if (!(reset_mask & (RADEON_RESET_GFX | 3759 RADEON_RESET_COMPUTE | 3760 RADEON_RESET_CP))) { 3761 radeon_ring_lockup_update(ring); 3762 return false; 3763 } 3764 /* force CP activities */ 3765 radeon_ring_force_activity(rdev, ring); 3766 return radeon_ring_test_lockup(rdev, ring); 3767 } 3768 3769 /** 3770 * si_dma_is_lockup - Check if the DMA engine is locked up 3771 * 3772 * @rdev: radeon_device pointer 3773 * @ring: radeon_ring structure holding ring information 3774 * 3775 * Check if the async DMA engine is locked up. 3776 * Returns true if the engine appears to be locked up, false if not. 3777 */ 3778 bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 3779 { 3780 u32 reset_mask = si_gpu_check_soft_reset(rdev); 3781 u32 mask; 3782 3783 if (ring->idx == R600_RING_TYPE_DMA_INDEX) 3784 mask = RADEON_RESET_DMA; 3785 else 3786 mask = RADEON_RESET_DMA1; 3787 3788 if (!(reset_mask & mask)) { 3789 radeon_ring_lockup_update(ring); 3790 return false; 3791 } 3792 /* force ring activities */ 3793 radeon_ring_force_activity(rdev, ring); 3794 return radeon_ring_test_lockup(rdev, ring); 3795 } 3796 3797 /* MC */ 3798 static void si_mc_program(struct radeon_device *rdev) 3799 { 3800 struct evergreen_mc_save save; 3801 u32 tmp; 3802 int i, j; 3803 3804 /* Initialize HDP */ 3805 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3806 WREG32((0x2c14 + j), 0x00000000); 3807 WREG32((0x2c18 + j), 0x00000000); 3808 WREG32((0x2c1c + j), 0x00000000); 3809 WREG32((0x2c20 + j), 0x00000000); 3810 WREG32((0x2c24 + j), 0x00000000); 3811 } 3812 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 3813 3814 evergreen_mc_stop(rdev, &save); 3815 if (radeon_mc_wait_for_idle(rdev)) { 3816 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3817 } 3818 if (!ASIC_IS_NODCE(rdev)) 3819 /* Lockout access through VGA aperture*/ 3820 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 3821 /* Update configuration */ 3822 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 3823 rdev->mc.vram_start >> 12); 3824 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 3825 rdev->mc.vram_end >> 12); 3826 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 3827 rdev->vram_scratch.gpu_addr >> 12); 3828 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 3829 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 3830 WREG32(MC_VM_FB_LOCATION, tmp); 3831 /* XXX double check these! */ 3832 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 3833 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 3834 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 3835 WREG32(MC_VM_AGP_BASE, 0); 3836 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 3837 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 3838 if (radeon_mc_wait_for_idle(rdev)) { 3839 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3840 } 3841 evergreen_mc_resume(rdev, &save); 3842 if (!ASIC_IS_NODCE(rdev)) { 3843 /* we need to own VRAM, so turn off the VGA renderer here 3844 * to stop it overwriting our objects */ 3845 rv515_vga_render_disable(rdev); 3846 } 3847 } 3848 3849 void si_vram_gtt_location(struct radeon_device *rdev, 3850 struct radeon_mc *mc) 3851 { 3852 if (mc->mc_vram_size > 0xFFC0000000ULL) { 3853 /* leave room for at least 1024M GTT */ 3854 dev_warn(rdev->dev, "limiting VRAM\n"); 3855 mc->real_vram_size = 0xFFC0000000ULL; 3856 mc->mc_vram_size = 0xFFC0000000ULL; 3857 } 3858 radeon_vram_location(rdev, &rdev->mc, 0); 3859 rdev->mc.gtt_base_align = 0; 3860 radeon_gtt_location(rdev, mc); 3861 } 3862 3863 static int si_mc_init(struct radeon_device *rdev) 3864 { 3865 u32 tmp; 3866 int chansize, numchan; 3867 3868 /* Get VRAM informations */ 3869 rdev->mc.vram_is_ddr = true; 3870 tmp = RREG32(MC_ARB_RAMCFG); 3871 if (tmp & CHANSIZE_OVERRIDE) { 3872 chansize = 16; 3873 } else if (tmp & CHANSIZE_MASK) { 3874 chansize = 64; 3875 } else { 3876 chansize = 32; 3877 } 3878 tmp = RREG32(MC_SHARED_CHMAP); 3879 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 3880 case 0: 3881 default: 3882 numchan = 1; 3883 break; 3884 case 1: 3885 numchan = 2; 3886 break; 3887 case 2: 3888 numchan = 4; 3889 break; 3890 case 3: 3891 numchan = 8; 3892 break; 3893 case 4: 3894 numchan = 3; 3895 break; 3896 case 5: 3897 numchan = 6; 3898 break; 3899 case 6: 3900 numchan = 10; 3901 break; 3902 case 7: 3903 numchan = 12; 3904 break; 3905 case 8: 3906 numchan = 16; 3907 break; 3908 } 3909 rdev->mc.vram_width = numchan * chansize; 3910 /* Could aper size report 0 ? */ 3911 rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0); 3912 rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0); 3913 /* size in MB on si */ 3914 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3915 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3916 rdev->mc.visible_vram_size = rdev->mc.aper_size; 3917 si_vram_gtt_location(rdev, &rdev->mc); 3918 radeon_update_bandwidth_info(rdev); 3919 3920 return 0; 3921 } 3922 3923 /* 3924 * GART 3925 */ 3926 void si_pcie_gart_tlb_flush(struct radeon_device *rdev) 3927 { 3928 /* flush hdp cache */ 3929 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 3930 3931 /* bits 0-15 are the VM contexts0-15 */ 3932 WREG32(VM_INVALIDATE_REQUEST, 1); 3933 } 3934 3935 static int si_pcie_gart_enable(struct radeon_device *rdev) 3936 { 3937 int r, i; 3938 3939 if (rdev->gart.robj == NULL) { 3940 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 3941 return -EINVAL; 3942 } 3943 r = radeon_gart_table_vram_pin(rdev); 3944 if (r) 3945 return r; 3946 radeon_gart_restore(rdev); 3947 /* Setup TLB control */ 3948 WREG32(MC_VM_MX_L1_TLB_CNTL, 3949 (0xA << 7) | 3950 ENABLE_L1_TLB | 3951 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 3952 ENABLE_ADVANCED_DRIVER_MODEL | 3953 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 3954 /* Setup L2 cache */ 3955 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | 3956 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 3957 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 3958 EFFECTIVE_L2_QUEUE_SIZE(7) | 3959 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 3960 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); 3961 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 3962 L2_CACHE_BIGK_FRAGMENT_SIZE(0)); 3963 /* setup context0 */ 3964 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 3965 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 3966 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 3967 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 3968 (u32)(rdev->dummy_page.addr >> 12)); 3969 WREG32(VM_CONTEXT0_CNTL2, 0); 3970 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 3971 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT)); 3972 3973 WREG32(0x15D4, 0); 3974 WREG32(0x15D8, 0); 3975 WREG32(0x15DC, 0); 3976 3977 /* empty context1-15 */ 3978 /* set vm size, must be a multiple of 4 */ 3979 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); 3980 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn); 3981 /* Assign the pt base to something valid for now; the pts used for 3982 * the VMs are determined by the application and setup and assigned 3983 * on the fly in the vm part of radeon_gart.c 3984 */ 3985 for (i = 1; i < 16; i++) { 3986 if (i < 8) 3987 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), 3988 rdev->gart.table_addr >> 12); 3989 else 3990 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2), 3991 rdev->gart.table_addr >> 12); 3992 } 3993 3994 /* enable context1-15 */ 3995 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, 3996 (u32)(rdev->dummy_page.addr >> 12)); 3997 WREG32(VM_CONTEXT1_CNTL2, 4); 3998 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | 3999 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 4000 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | 4001 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 4002 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT | 4003 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT | 4004 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT | 4005 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT | 4006 VALID_PROTECTION_FAULT_ENABLE_DEFAULT | 4007 READ_PROTECTION_FAULT_ENABLE_INTERRUPT | 4008 READ_PROTECTION_FAULT_ENABLE_DEFAULT | 4009 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT | 4010 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT); 4011 4012 si_pcie_gart_tlb_flush(rdev); 4013 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 4014 (unsigned)(rdev->mc.gtt_size >> 20), 4015 (unsigned long long)rdev->gart.table_addr); 4016 rdev->gart.ready = true; 4017 return 0; 4018 } 4019 4020 static void si_pcie_gart_disable(struct radeon_device *rdev) 4021 { 4022 /* Disable all tables */ 4023 WREG32(VM_CONTEXT0_CNTL, 0); 4024 WREG32(VM_CONTEXT1_CNTL, 0); 4025 /* Setup TLB control */ 4026 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS | 4027 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 4028 /* Setup L2 cache */ 4029 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 4030 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 4031 EFFECTIVE_L2_QUEUE_SIZE(7) | 4032 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 4033 WREG32(VM_L2_CNTL2, 0); 4034 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 4035 L2_CACHE_BIGK_FRAGMENT_SIZE(0)); 4036 radeon_gart_table_vram_unpin(rdev); 4037 } 4038 4039 static void si_pcie_gart_fini(struct radeon_device *rdev) 4040 { 4041 si_pcie_gart_disable(rdev); 4042 radeon_gart_table_vram_free(rdev); 4043 radeon_gart_fini(rdev); 4044 } 4045 4046 /* vm parser */ 4047 static bool si_vm_reg_valid(u32 reg) 4048 { 4049 /* context regs are fine */ 4050 if (reg >= 0x28000) 4051 return true; 4052 4053 /* check config regs */ 4054 switch (reg) { 4055 case GRBM_GFX_INDEX: 4056 case CP_STRMOUT_CNTL: 4057 case VGT_VTX_VECT_EJECT_REG: 4058 case VGT_CACHE_INVALIDATION: 4059 case VGT_ESGS_RING_SIZE: 4060 case VGT_GSVS_RING_SIZE: 4061 case VGT_GS_VERTEX_REUSE: 4062 case VGT_PRIMITIVE_TYPE: 4063 case VGT_INDEX_TYPE: 4064 case VGT_NUM_INDICES: 4065 case VGT_NUM_INSTANCES: 4066 case VGT_TF_RING_SIZE: 4067 case VGT_HS_OFFCHIP_PARAM: 4068 case VGT_TF_MEMORY_BASE: 4069 case PA_CL_ENHANCE: 4070 case PA_SU_LINE_STIPPLE_VALUE: 4071 case PA_SC_LINE_STIPPLE_STATE: 4072 case PA_SC_ENHANCE: 4073 case SQC_CACHES: 4074 case SPI_STATIC_THREAD_MGMT_1: 4075 case SPI_STATIC_THREAD_MGMT_2: 4076 case SPI_STATIC_THREAD_MGMT_3: 4077 case SPI_PS_MAX_WAVE_ID: 4078 case SPI_CONFIG_CNTL: 4079 case SPI_CONFIG_CNTL_1: 4080 case TA_CNTL_AUX: 4081 return true; 4082 default: 4083 DRM_ERROR("Invalid register 0x%x in CS\n", reg); 4084 return false; 4085 } 4086 } 4087 4088 static int si_vm_packet3_ce_check(struct radeon_device *rdev, 4089 u32 *ib, struct radeon_cs_packet *pkt) 4090 { 4091 switch (pkt->opcode) { 4092 case PACKET3_NOP: 4093 case PACKET3_SET_BASE: 4094 case PACKET3_SET_CE_DE_COUNTERS: 4095 case PACKET3_LOAD_CONST_RAM: 4096 case PACKET3_WRITE_CONST_RAM: 4097 case PACKET3_WRITE_CONST_RAM_OFFSET: 4098 case PACKET3_DUMP_CONST_RAM: 4099 case PACKET3_INCREMENT_CE_COUNTER: 4100 case PACKET3_WAIT_ON_DE_COUNTER: 4101 case PACKET3_CE_WRITE: 4102 break; 4103 default: 4104 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode); 4105 return -EINVAL; 4106 } 4107 return 0; 4108 } 4109 4110 static int si_vm_packet3_gfx_check(struct radeon_device *rdev, 4111 u32 *ib, struct radeon_cs_packet *pkt) 4112 { 4113 u32 idx = pkt->idx + 1; 4114 u32 idx_value = ib[idx]; 4115 u32 start_reg, end_reg, reg, i; 4116 u32 command, info; 4117 4118 switch (pkt->opcode) { 4119 case PACKET3_NOP: 4120 case PACKET3_SET_BASE: 4121 case PACKET3_CLEAR_STATE: 4122 case PACKET3_INDEX_BUFFER_SIZE: 4123 case PACKET3_DISPATCH_DIRECT: 4124 case PACKET3_DISPATCH_INDIRECT: 4125 case PACKET3_ALLOC_GDS: 4126 case PACKET3_WRITE_GDS_RAM: 4127 case PACKET3_ATOMIC_GDS: 4128 case PACKET3_ATOMIC: 4129 case PACKET3_OCCLUSION_QUERY: 4130 case PACKET3_SET_PREDICATION: 4131 case PACKET3_COND_EXEC: 4132 case PACKET3_PRED_EXEC: 4133 case PACKET3_DRAW_INDIRECT: 4134 case PACKET3_DRAW_INDEX_INDIRECT: 4135 case PACKET3_INDEX_BASE: 4136 case PACKET3_DRAW_INDEX_2: 4137 case PACKET3_CONTEXT_CONTROL: 4138 case PACKET3_INDEX_TYPE: 4139 case PACKET3_DRAW_INDIRECT_MULTI: 4140 case PACKET3_DRAW_INDEX_AUTO: 4141 case PACKET3_DRAW_INDEX_IMMD: 4142 case PACKET3_NUM_INSTANCES: 4143 case PACKET3_DRAW_INDEX_MULTI_AUTO: 4144 case PACKET3_STRMOUT_BUFFER_UPDATE: 4145 case PACKET3_DRAW_INDEX_OFFSET_2: 4146 case PACKET3_DRAW_INDEX_MULTI_ELEMENT: 4147 case PACKET3_DRAW_INDEX_INDIRECT_MULTI: 4148 case PACKET3_MPEG_INDEX: 4149 case PACKET3_WAIT_REG_MEM: 4150 case PACKET3_MEM_WRITE: 4151 case PACKET3_PFP_SYNC_ME: 4152 case PACKET3_SURFACE_SYNC: 4153 case PACKET3_EVENT_WRITE: 4154 case PACKET3_EVENT_WRITE_EOP: 4155 case PACKET3_EVENT_WRITE_EOS: 4156 case PACKET3_SET_CONTEXT_REG: 4157 case PACKET3_SET_CONTEXT_REG_INDIRECT: 4158 case PACKET3_SET_SH_REG: 4159 case PACKET3_SET_SH_REG_OFFSET: 4160 case PACKET3_INCREMENT_DE_COUNTER: 4161 case PACKET3_WAIT_ON_CE_COUNTER: 4162 case PACKET3_WAIT_ON_AVAIL_BUFFER: 4163 case PACKET3_ME_WRITE: 4164 break; 4165 case PACKET3_COPY_DATA: 4166 if ((idx_value & 0xf00) == 0) { 4167 reg = ib[idx + 3] * 4; 4168 if (!si_vm_reg_valid(reg)) 4169 return -EINVAL; 4170 } 4171 break; 4172 case PACKET3_WRITE_DATA: 4173 if ((idx_value & 0xf00) == 0) { 4174 start_reg = ib[idx + 1] * 4; 4175 if (idx_value & 0x10000) { 4176 if (!si_vm_reg_valid(start_reg)) 4177 return -EINVAL; 4178 } else { 4179 for (i = 0; i < (pkt->count - 2); i++) { 4180 reg = start_reg + (4 * i); 4181 if (!si_vm_reg_valid(reg)) 4182 return -EINVAL; 4183 } 4184 } 4185 } 4186 break; 4187 case PACKET3_COND_WRITE: 4188 if (idx_value & 0x100) { 4189 reg = ib[idx + 5] * 4; 4190 if (!si_vm_reg_valid(reg)) 4191 return -EINVAL; 4192 } 4193 break; 4194 case PACKET3_COPY_DW: 4195 if (idx_value & 0x2) { 4196 reg = ib[idx + 3] * 4; 4197 if (!si_vm_reg_valid(reg)) 4198 return -EINVAL; 4199 } 4200 break; 4201 case PACKET3_SET_CONFIG_REG: 4202 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START; 4203 end_reg = 4 * pkt->count + start_reg - 4; 4204 if ((start_reg < PACKET3_SET_CONFIG_REG_START) || 4205 (start_reg >= PACKET3_SET_CONFIG_REG_END) || 4206 (end_reg >= PACKET3_SET_CONFIG_REG_END)) { 4207 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n"); 4208 return -EINVAL; 4209 } 4210 for (i = 0; i < pkt->count; i++) { 4211 reg = start_reg + (4 * i); 4212 if (!si_vm_reg_valid(reg)) 4213 return -EINVAL; 4214 } 4215 break; 4216 case PACKET3_CP_DMA: 4217 command = ib[idx + 4]; 4218 info = ib[idx + 1]; 4219 if (command & PACKET3_CP_DMA_CMD_SAS) { 4220 /* src address space is register */ 4221 if (((info & 0x60000000) >> 29) == 0) { 4222 start_reg = idx_value << 2; 4223 if (command & PACKET3_CP_DMA_CMD_SAIC) { 4224 reg = start_reg; 4225 if (!si_vm_reg_valid(reg)) { 4226 DRM_ERROR("CP DMA Bad SRC register\n"); 4227 return -EINVAL; 4228 } 4229 } else { 4230 for (i = 0; i < (command & 0x1fffff); i++) { 4231 reg = start_reg + (4 * i); 4232 if (!si_vm_reg_valid(reg)) { 4233 DRM_ERROR("CP DMA Bad SRC register\n"); 4234 return -EINVAL; 4235 } 4236 } 4237 } 4238 } 4239 } 4240 if (command & PACKET3_CP_DMA_CMD_DAS) { 4241 /* dst address space is register */ 4242 if (((info & 0x00300000) >> 20) == 0) { 4243 start_reg = ib[idx + 2]; 4244 if (command & PACKET3_CP_DMA_CMD_DAIC) { 4245 reg = start_reg; 4246 if (!si_vm_reg_valid(reg)) { 4247 DRM_ERROR("CP DMA Bad DST register\n"); 4248 return -EINVAL; 4249 } 4250 } else { 4251 for (i = 0; i < (command & 0x1fffff); i++) { 4252 reg = start_reg + (4 * i); 4253 if (!si_vm_reg_valid(reg)) { 4254 DRM_ERROR("CP DMA Bad DST register\n"); 4255 return -EINVAL; 4256 } 4257 } 4258 } 4259 } 4260 } 4261 break; 4262 default: 4263 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode); 4264 return -EINVAL; 4265 } 4266 return 0; 4267 } 4268 4269 static int si_vm_packet3_compute_check(struct radeon_device *rdev, 4270 u32 *ib, struct radeon_cs_packet *pkt) 4271 { 4272 u32 idx = pkt->idx + 1; 4273 u32 idx_value = ib[idx]; 4274 u32 start_reg, reg, i; 4275 4276 switch (pkt->opcode) { 4277 case PACKET3_NOP: 4278 case PACKET3_SET_BASE: 4279 case PACKET3_CLEAR_STATE: 4280 case PACKET3_DISPATCH_DIRECT: 4281 case PACKET3_DISPATCH_INDIRECT: 4282 case PACKET3_ALLOC_GDS: 4283 case PACKET3_WRITE_GDS_RAM: 4284 case PACKET3_ATOMIC_GDS: 4285 case PACKET3_ATOMIC: 4286 case PACKET3_OCCLUSION_QUERY: 4287 case PACKET3_SET_PREDICATION: 4288 case PACKET3_COND_EXEC: 4289 case PACKET3_PRED_EXEC: 4290 case PACKET3_CONTEXT_CONTROL: 4291 case PACKET3_STRMOUT_BUFFER_UPDATE: 4292 case PACKET3_WAIT_REG_MEM: 4293 case PACKET3_MEM_WRITE: 4294 case PACKET3_PFP_SYNC_ME: 4295 case PACKET3_SURFACE_SYNC: 4296 case PACKET3_EVENT_WRITE: 4297 case PACKET3_EVENT_WRITE_EOP: 4298 case PACKET3_EVENT_WRITE_EOS: 4299 case PACKET3_SET_CONTEXT_REG: 4300 case PACKET3_SET_CONTEXT_REG_INDIRECT: 4301 case PACKET3_SET_SH_REG: 4302 case PACKET3_SET_SH_REG_OFFSET: 4303 case PACKET3_INCREMENT_DE_COUNTER: 4304 case PACKET3_WAIT_ON_CE_COUNTER: 4305 case PACKET3_WAIT_ON_AVAIL_BUFFER: 4306 case PACKET3_ME_WRITE: 4307 break; 4308 case PACKET3_COPY_DATA: 4309 if ((idx_value & 0xf00) == 0) { 4310 reg = ib[idx + 3] * 4; 4311 if (!si_vm_reg_valid(reg)) 4312 return -EINVAL; 4313 } 4314 break; 4315 case PACKET3_WRITE_DATA: 4316 if ((idx_value & 0xf00) == 0) { 4317 start_reg = ib[idx + 1] * 4; 4318 if (idx_value & 0x10000) { 4319 if (!si_vm_reg_valid(start_reg)) 4320 return -EINVAL; 4321 } else { 4322 for (i = 0; i < (pkt->count - 2); i++) { 4323 reg = start_reg + (4 * i); 4324 if (!si_vm_reg_valid(reg)) 4325 return -EINVAL; 4326 } 4327 } 4328 } 4329 break; 4330 case PACKET3_COND_WRITE: 4331 if (idx_value & 0x100) { 4332 reg = ib[idx + 5] * 4; 4333 if (!si_vm_reg_valid(reg)) 4334 return -EINVAL; 4335 } 4336 break; 4337 case PACKET3_COPY_DW: 4338 if (idx_value & 0x2) { 4339 reg = ib[idx + 3] * 4; 4340 if (!si_vm_reg_valid(reg)) 4341 return -EINVAL; 4342 } 4343 break; 4344 default: 4345 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode); 4346 return -EINVAL; 4347 } 4348 return 0; 4349 } 4350 4351 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) 4352 { 4353 int ret = 0; 4354 u32 idx = 0; 4355 struct radeon_cs_packet pkt; 4356 4357 do { 4358 pkt.idx = idx; 4359 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]); 4360 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]); 4361 pkt.one_reg_wr = 0; 4362 switch (pkt.type) { 4363 case RADEON_PACKET_TYPE0: 4364 dev_err(rdev->dev, "Packet0 not allowed!\n"); 4365 ret = -EINVAL; 4366 break; 4367 case RADEON_PACKET_TYPE2: 4368 idx += 1; 4369 break; 4370 case RADEON_PACKET_TYPE3: 4371 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]); 4372 if (ib->is_const_ib) 4373 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt); 4374 else { 4375 switch (ib->ring) { 4376 case RADEON_RING_TYPE_GFX_INDEX: 4377 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt); 4378 break; 4379 case CAYMAN_RING_TYPE_CP1_INDEX: 4380 case CAYMAN_RING_TYPE_CP2_INDEX: 4381 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt); 4382 break; 4383 default: 4384 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring); 4385 ret = -EINVAL; 4386 break; 4387 } 4388 } 4389 idx += pkt.count + 2; 4390 break; 4391 default: 4392 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type); 4393 ret = -EINVAL; 4394 break; 4395 } 4396 if (ret) 4397 break; 4398 } while (idx < ib->length_dw); 4399 4400 return ret; 4401 } 4402 4403 /* 4404 * vm 4405 */ 4406 int si_vm_init(struct radeon_device *rdev) 4407 { 4408 /* number of VMs */ 4409 rdev->vm_manager.nvm = 16; 4410 /* base offset of vram pages */ 4411 rdev->vm_manager.vram_base_offset = 0; 4412 4413 return 0; 4414 } 4415 4416 void si_vm_fini(struct radeon_device *rdev) 4417 { 4418 } 4419 4420 /** 4421 * si_vm_decode_fault - print human readable fault info 4422 * 4423 * @rdev: radeon_device pointer 4424 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value 4425 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value 4426 * 4427 * Print human readable fault information (SI). 4428 */ 4429 static void si_vm_decode_fault(struct radeon_device *rdev, 4430 u32 status, u32 addr) 4431 { 4432 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 4433 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; 4434 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; 4435 char *block; 4436 4437 if (rdev->family == CHIP_TAHITI) { 4438 switch (mc_id) { 4439 case 160: 4440 case 144: 4441 case 96: 4442 case 80: 4443 case 224: 4444 case 208: 4445 case 32: 4446 case 16: 4447 block = "CB"; 4448 break; 4449 case 161: 4450 case 145: 4451 case 97: 4452 case 81: 4453 case 225: 4454 case 209: 4455 case 33: 4456 case 17: 4457 block = "CB_FMASK"; 4458 break; 4459 case 162: 4460 case 146: 4461 case 98: 4462 case 82: 4463 case 226: 4464 case 210: 4465 case 34: 4466 case 18: 4467 block = "CB_CMASK"; 4468 break; 4469 case 163: 4470 case 147: 4471 case 99: 4472 case 83: 4473 case 227: 4474 case 211: 4475 case 35: 4476 case 19: 4477 block = "CB_IMMED"; 4478 break; 4479 case 164: 4480 case 148: 4481 case 100: 4482 case 84: 4483 case 228: 4484 case 212: 4485 case 36: 4486 case 20: 4487 block = "DB"; 4488 break; 4489 case 165: 4490 case 149: 4491 case 101: 4492 case 85: 4493 case 229: 4494 case 213: 4495 case 37: 4496 case 21: 4497 block = "DB_HTILE"; 4498 break; 4499 case 167: 4500 case 151: 4501 case 103: 4502 case 87: 4503 case 231: 4504 case 215: 4505 case 39: 4506 case 23: 4507 block = "DB_STEN"; 4508 break; 4509 case 72: 4510 case 68: 4511 case 64: 4512 case 8: 4513 case 4: 4514 case 0: 4515 case 136: 4516 case 132: 4517 case 128: 4518 case 200: 4519 case 196: 4520 case 192: 4521 block = "TC"; 4522 break; 4523 case 112: 4524 case 48: 4525 block = "CP"; 4526 break; 4527 case 49: 4528 case 177: 4529 case 50: 4530 case 178: 4531 block = "SH"; 4532 break; 4533 case 53: 4534 case 190: 4535 block = "VGT"; 4536 break; 4537 case 117: 4538 block = "IH"; 4539 break; 4540 case 51: 4541 case 115: 4542 block = "RLC"; 4543 break; 4544 case 119: 4545 case 183: 4546 block = "DMA0"; 4547 break; 4548 case 61: 4549 block = "DMA1"; 4550 break; 4551 case 248: 4552 case 120: 4553 block = "HDP"; 4554 break; 4555 default: 4556 block = "unknown"; 4557 break; 4558 } 4559 } else { 4560 switch (mc_id) { 4561 case 32: 4562 case 16: 4563 case 96: 4564 case 80: 4565 case 160: 4566 case 144: 4567 case 224: 4568 case 208: 4569 block = "CB"; 4570 break; 4571 case 33: 4572 case 17: 4573 case 97: 4574 case 81: 4575 case 161: 4576 case 145: 4577 case 225: 4578 case 209: 4579 block = "CB_FMASK"; 4580 break; 4581 case 34: 4582 case 18: 4583 case 98: 4584 case 82: 4585 case 162: 4586 case 146: 4587 case 226: 4588 case 210: 4589 block = "CB_CMASK"; 4590 break; 4591 case 35: 4592 case 19: 4593 case 99: 4594 case 83: 4595 case 163: 4596 case 147: 4597 case 227: 4598 case 211: 4599 block = "CB_IMMED"; 4600 break; 4601 case 36: 4602 case 20: 4603 case 100: 4604 case 84: 4605 case 164: 4606 case 148: 4607 case 228: 4608 case 212: 4609 block = "DB"; 4610 break; 4611 case 37: 4612 case 21: 4613 case 101: 4614 case 85: 4615 case 165: 4616 case 149: 4617 case 229: 4618 case 213: 4619 block = "DB_HTILE"; 4620 break; 4621 case 39: 4622 case 23: 4623 case 103: 4624 case 87: 4625 case 167: 4626 case 151: 4627 case 231: 4628 case 215: 4629 block = "DB_STEN"; 4630 break; 4631 case 72: 4632 case 68: 4633 case 8: 4634 case 4: 4635 case 136: 4636 case 132: 4637 case 200: 4638 case 196: 4639 block = "TC"; 4640 break; 4641 case 112: 4642 case 48: 4643 block = "CP"; 4644 break; 4645 case 49: 4646 case 177: 4647 case 50: 4648 case 178: 4649 block = "SH"; 4650 break; 4651 case 53: 4652 block = "VGT"; 4653 break; 4654 case 117: 4655 block = "IH"; 4656 break; 4657 case 51: 4658 case 115: 4659 block = "RLC"; 4660 break; 4661 case 119: 4662 case 183: 4663 block = "DMA0"; 4664 break; 4665 case 61: 4666 block = "DMA1"; 4667 break; 4668 case 248: 4669 case 120: 4670 block = "HDP"; 4671 break; 4672 default: 4673 block = "unknown"; 4674 break; 4675 } 4676 } 4677 4678 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n", 4679 protections, vmid, addr, 4680 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", 4681 block, mc_id); 4682 } 4683 4684 /** 4685 * si_vm_set_page - update the page tables using the CP 4686 * 4687 * @rdev: radeon_device pointer 4688 * @ib: indirect buffer to fill with commands 4689 * @pe: addr of the page entry 4690 * @addr: dst addr to write into pe 4691 * @count: number of page entries to update 4692 * @incr: increase next addr by incr bytes 4693 * @flags: access flags 4694 * 4695 * Update the page tables using the CP (SI). 4696 */ 4697 void si_vm_set_page(struct radeon_device *rdev, 4698 struct radeon_ib *ib, 4699 uint64_t pe, 4700 uint64_t addr, unsigned count, 4701 uint32_t incr, uint32_t flags) 4702 { 4703 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags); 4704 uint64_t value; 4705 unsigned ndw; 4706 4707 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) { 4708 while (count) { 4709 ndw = 2 + count * 2; 4710 if (ndw > 0x3FFE) 4711 ndw = 0x3FFE; 4712 4713 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw); 4714 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) | 4715 WRITE_DATA_DST_SEL(1)); 4716 ib->ptr[ib->length_dw++] = pe; 4717 ib->ptr[ib->length_dw++] = upper_32_bits(pe); 4718 for (; ndw > 2; ndw -= 2, --count, pe += 8) { 4719 if (flags & RADEON_VM_PAGE_SYSTEM) { 4720 value = radeon_vm_map_gart(rdev, addr); 4721 value &= 0xFFFFFFFFFFFFF000ULL; 4722 } else if (flags & RADEON_VM_PAGE_VALID) { 4723 value = addr; 4724 } else { 4725 value = 0; 4726 } 4727 addr += incr; 4728 value |= r600_flags; 4729 ib->ptr[ib->length_dw++] = value; 4730 ib->ptr[ib->length_dw++] = upper_32_bits(value); 4731 } 4732 } 4733 } else { 4734 /* DMA */ 4735 if (flags & RADEON_VM_PAGE_SYSTEM) { 4736 while (count) { 4737 ndw = count * 2; 4738 if (ndw > 0xFFFFE) 4739 ndw = 0xFFFFE; 4740 4741 /* for non-physically contiguous pages (system) */ 4742 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw); 4743 ib->ptr[ib->length_dw++] = pe; 4744 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; 4745 for (; ndw > 0; ndw -= 2, --count, pe += 8) { 4746 if (flags & RADEON_VM_PAGE_SYSTEM) { 4747 value = radeon_vm_map_gart(rdev, addr); 4748 value &= 0xFFFFFFFFFFFFF000ULL; 4749 } else if (flags & RADEON_VM_PAGE_VALID) { 4750 value = addr; 4751 } else { 4752 value = 0; 4753 } 4754 addr += incr; 4755 value |= r600_flags; 4756 ib->ptr[ib->length_dw++] = value; 4757 ib->ptr[ib->length_dw++] = upper_32_bits(value); 4758 } 4759 } 4760 } else { 4761 while (count) { 4762 ndw = count * 2; 4763 if (ndw > 0xFFFFE) 4764 ndw = 0xFFFFE; 4765 4766 if (flags & RADEON_VM_PAGE_VALID) 4767 value = addr; 4768 else 4769 value = 0; 4770 /* for physically contiguous pages (vram) */ 4771 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw); 4772 ib->ptr[ib->length_dw++] = pe; /* dst addr */ 4773 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; 4774 ib->ptr[ib->length_dw++] = r600_flags; /* mask */ 4775 ib->ptr[ib->length_dw++] = 0; 4776 ib->ptr[ib->length_dw++] = value; /* value */ 4777 ib->ptr[ib->length_dw++] = upper_32_bits(value); 4778 ib->ptr[ib->length_dw++] = incr; /* increment size */ 4779 ib->ptr[ib->length_dw++] = 0; 4780 pe += ndw * 4; 4781 addr += (ndw / 2) * incr; 4782 count -= ndw / 2; 4783 } 4784 } 4785 while (ib->length_dw & 0x7) 4786 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0); 4787 } 4788 } 4789 4790 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) 4791 { 4792 struct radeon_ring *ring = &rdev->ring[ridx]; 4793 4794 if (vm == NULL) 4795 return; 4796 4797 /* write new base address */ 4798 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 4799 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 4800 WRITE_DATA_DST_SEL(0))); 4801 4802 if (vm->id < 8) { 4803 radeon_ring_write(ring, 4804 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2); 4805 } else { 4806 radeon_ring_write(ring, 4807 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2); 4808 } 4809 radeon_ring_write(ring, 0); 4810 radeon_ring_write(ring, vm->pd_gpu_addr >> 12); 4811 4812 /* flush hdp cache */ 4813 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 4814 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 4815 WRITE_DATA_DST_SEL(0))); 4816 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2); 4817 radeon_ring_write(ring, 0); 4818 radeon_ring_write(ring, 0x1); 4819 4820 /* bits 0-15 are the VM contexts0-15 */ 4821 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 4822 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 4823 WRITE_DATA_DST_SEL(0))); 4824 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 4825 radeon_ring_write(ring, 0); 4826 radeon_ring_write(ring, 1 << vm->id); 4827 4828 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 4829 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 4830 radeon_ring_write(ring, 0x0); 4831 } 4832 4833 void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) 4834 { 4835 struct radeon_ring *ring = &rdev->ring[ridx]; 4836 4837 if (vm == NULL) 4838 return; 4839 4840 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); 4841 if (vm->id < 8) { 4842 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2)); 4843 } else { 4844 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2)); 4845 } 4846 radeon_ring_write(ring, vm->pd_gpu_addr >> 12); 4847 4848 /* flush hdp cache */ 4849 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); 4850 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2)); 4851 radeon_ring_write(ring, 1); 4852 4853 /* bits 0-7 are the VM contexts0-7 */ 4854 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); 4855 radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2)); 4856 radeon_ring_write(ring, 1 << vm->id); 4857 } 4858 4859 /* 4860 * Power and clock gating 4861 */ 4862 static void si_wait_for_rlc_serdes(struct radeon_device *rdev) 4863 { 4864 int i; 4865 4866 for (i = 0; i < rdev->usec_timeout; i++) { 4867 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0) 4868 break; 4869 udelay(1); 4870 } 4871 4872 for (i = 0; i < rdev->usec_timeout; i++) { 4873 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0) 4874 break; 4875 udelay(1); 4876 } 4877 } 4878 4879 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev, 4880 bool enable) 4881 { 4882 u32 tmp = RREG32(CP_INT_CNTL_RING0); 4883 u32 mask; 4884 int i; 4885 4886 if (enable) 4887 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4888 else 4889 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4890 WREG32(CP_INT_CNTL_RING0, tmp); 4891 4892 if (!enable) { 4893 /* read a gfx register */ 4894 tmp = RREG32(DB_DEPTH_INFO); 4895 4896 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS; 4897 for (i = 0; i < rdev->usec_timeout; i++) { 4898 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS)) 4899 break; 4900 udelay(1); 4901 } 4902 } 4903 } 4904 4905 static void si_set_uvd_dcm(struct radeon_device *rdev, 4906 bool sw_mode) 4907 { 4908 u32 tmp, tmp2; 4909 4910 tmp = RREG32(UVD_CGC_CTRL); 4911 tmp &= ~(CLK_OD_MASK | CG_DT_MASK); 4912 tmp |= DCM | CG_DT(1) | CLK_OD(4); 4913 4914 if (sw_mode) { 4915 tmp &= ~0x7ffff800; 4916 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7); 4917 } else { 4918 tmp |= 0x7ffff800; 4919 tmp2 = 0; 4920 } 4921 4922 WREG32(UVD_CGC_CTRL, tmp); 4923 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2); 4924 } 4925 4926 static void si_init_uvd_internal_cg(struct radeon_device *rdev) 4927 { 4928 bool hw_mode = true; 4929 4930 if (hw_mode) { 4931 si_set_uvd_dcm(rdev, false); 4932 } else { 4933 u32 tmp = RREG32(UVD_CGC_CTRL); 4934 tmp &= ~DCM; 4935 WREG32(UVD_CGC_CTRL, tmp); 4936 } 4937 } 4938 4939 static u32 si_halt_rlc(struct radeon_device *rdev) 4940 { 4941 u32 data, orig; 4942 4943 orig = data = RREG32(RLC_CNTL); 4944 4945 if (data & RLC_ENABLE) { 4946 data &= ~RLC_ENABLE; 4947 WREG32(RLC_CNTL, data); 4948 4949 si_wait_for_rlc_serdes(rdev); 4950 } 4951 4952 return orig; 4953 } 4954 4955 static void si_update_rlc(struct radeon_device *rdev, u32 rlc) 4956 { 4957 u32 tmp; 4958 4959 tmp = RREG32(RLC_CNTL); 4960 if (tmp != rlc) 4961 WREG32(RLC_CNTL, rlc); 4962 } 4963 4964 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable) 4965 { 4966 u32 data, orig; 4967 4968 orig = data = RREG32(DMA_PG); 4969 if (enable) 4970 data |= PG_CNTL_ENABLE; 4971 else 4972 data &= ~PG_CNTL_ENABLE; 4973 if (orig != data) 4974 WREG32(DMA_PG, data); 4975 } 4976 4977 static void si_init_dma_pg(struct radeon_device *rdev) 4978 { 4979 u32 tmp; 4980 4981 WREG32(DMA_PGFSM_WRITE, 0x00002000); 4982 WREG32(DMA_PGFSM_CONFIG, 0x100010ff); 4983 4984 for (tmp = 0; tmp < 5; tmp++) 4985 WREG32(DMA_PGFSM_WRITE, 0); 4986 } 4987 4988 static void si_enable_gfx_cgpg(struct radeon_device *rdev, 4989 bool enable) 4990 { 4991 u32 tmp; 4992 4993 if (enable) { 4994 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10); 4995 WREG32(RLC_TTOP_D, tmp); 4996 4997 tmp = RREG32(RLC_PG_CNTL); 4998 tmp |= GFX_PG_ENABLE; 4999 WREG32(RLC_PG_CNTL, tmp); 5000 5001 tmp = RREG32(RLC_AUTO_PG_CTRL); 5002 tmp |= AUTO_PG_EN; 5003 WREG32(RLC_AUTO_PG_CTRL, tmp); 5004 } else { 5005 tmp = RREG32(RLC_AUTO_PG_CTRL); 5006 tmp &= ~AUTO_PG_EN; 5007 WREG32(RLC_AUTO_PG_CTRL, tmp); 5008 5009 tmp = RREG32(DB_RENDER_CONTROL); 5010 } 5011 } 5012 5013 static void si_init_gfx_cgpg(struct radeon_device *rdev) 5014 { 5015 u32 tmp; 5016 5017 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 5018 5019 tmp = RREG32(RLC_PG_CNTL); 5020 tmp |= GFX_PG_SRC; 5021 WREG32(RLC_PG_CNTL, tmp); 5022 5023 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 5024 5025 tmp = RREG32(RLC_AUTO_PG_CTRL); 5026 5027 tmp &= ~GRBM_REG_SGIT_MASK; 5028 tmp |= GRBM_REG_SGIT(0x700); 5029 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK; 5030 WREG32(RLC_AUTO_PG_CTRL, tmp); 5031 } 5032 5033 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) 5034 { 5035 u32 mask = 0, tmp, tmp1; 5036 int i; 5037 5038 si_select_se_sh(rdev, se, sh); 5039 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 5040 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); 5041 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5042 5043 tmp &= 0xffff0000; 5044 5045 tmp |= tmp1; 5046 tmp >>= 16; 5047 5048 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) { 5049 mask <<= 1; 5050 mask |= 1; 5051 } 5052 5053 return (~tmp) & mask; 5054 } 5055 5056 static void si_init_ao_cu_mask(struct radeon_device *rdev) 5057 { 5058 u32 i, j, k, active_cu_number = 0; 5059 u32 mask, counter, cu_bitmap; 5060 u32 tmp = 0; 5061 5062 for (i = 0; i < rdev->config.si.max_shader_engines; i++) { 5063 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) { 5064 mask = 1; 5065 cu_bitmap = 0; 5066 counter = 0; 5067 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) { 5068 if (si_get_cu_active_bitmap(rdev, i, j) & mask) { 5069 if (counter < 2) 5070 cu_bitmap |= mask; 5071 counter++; 5072 } 5073 mask <<= 1; 5074 } 5075 5076 active_cu_number += counter; 5077 tmp |= (cu_bitmap << (i * 16 + j * 8)); 5078 } 5079 } 5080 5081 WREG32(RLC_PG_AO_CU_MASK, tmp); 5082 5083 tmp = RREG32(RLC_MAX_PG_CU); 5084 tmp &= ~MAX_PU_CU_MASK; 5085 tmp |= MAX_PU_CU(active_cu_number); 5086 WREG32(RLC_MAX_PG_CU, tmp); 5087 } 5088 5089 static void si_enable_cgcg(struct radeon_device *rdev, 5090 bool enable) 5091 { 5092 u32 data, orig, tmp; 5093 5094 orig = data = RREG32(RLC_CGCG_CGLS_CTRL); 5095 5096 si_enable_gui_idle_interrupt(rdev, enable); 5097 5098 if (enable) { 5099 WREG32(RLC_GCPM_GENERAL_3, 0x00000080); 5100 5101 tmp = si_halt_rlc(rdev); 5102 5103 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); 5104 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); 5105 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff); 5106 5107 si_wait_for_rlc_serdes(rdev); 5108 5109 si_update_rlc(rdev, tmp); 5110 5111 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff); 5112 5113 data |= CGCG_EN | CGLS_EN; 5114 } else { 5115 RREG32(CB_CGTT_SCLK_CTRL); 5116 RREG32(CB_CGTT_SCLK_CTRL); 5117 RREG32(CB_CGTT_SCLK_CTRL); 5118 RREG32(CB_CGTT_SCLK_CTRL); 5119 5120 data &= ~(CGCG_EN | CGLS_EN); 5121 } 5122 5123 if (orig != data) 5124 WREG32(RLC_CGCG_CGLS_CTRL, data); 5125 } 5126 5127 static void si_enable_mgcg(struct radeon_device *rdev, 5128 bool enable) 5129 { 5130 u32 data, orig, tmp = 0; 5131 5132 if (enable) { 5133 orig = data = RREG32(CGTS_SM_CTRL_REG); 5134 data = 0x96940200; 5135 if (orig != data) 5136 WREG32(CGTS_SM_CTRL_REG, data); 5137 5138 orig = data = RREG32(CP_MEM_SLP_CNTL); 5139 data |= CP_MEM_LS_EN; 5140 if (orig != data) 5141 WREG32(CP_MEM_SLP_CNTL, data); 5142 5143 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 5144 data &= 0xffffffc0; 5145 if (orig != data) 5146 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 5147 5148 tmp = si_halt_rlc(rdev); 5149 5150 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); 5151 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); 5152 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff); 5153 5154 si_update_rlc(rdev, tmp); 5155 } else { 5156 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 5157 data |= 0x00000003; 5158 if (orig != data) 5159 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 5160 5161 data = RREG32(CP_MEM_SLP_CNTL); 5162 if (data & CP_MEM_LS_EN) { 5163 data &= ~CP_MEM_LS_EN; 5164 WREG32(CP_MEM_SLP_CNTL, data); 5165 } 5166 orig = data = RREG32(CGTS_SM_CTRL_REG); 5167 data |= LS_OVERRIDE | OVERRIDE; 5168 if (orig != data) 5169 WREG32(CGTS_SM_CTRL_REG, data); 5170 5171 tmp = si_halt_rlc(rdev); 5172 5173 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); 5174 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); 5175 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff); 5176 5177 si_update_rlc(rdev, tmp); 5178 } 5179 } 5180 5181 static void si_enable_uvd_mgcg(struct radeon_device *rdev, 5182 bool enable) 5183 { 5184 u32 orig, data, tmp; 5185 5186 if (enable) { 5187 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 5188 tmp |= 0x3fff; 5189 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp); 5190 5191 orig = data = RREG32(UVD_CGC_CTRL); 5192 data |= DCM; 5193 if (orig != data) 5194 WREG32(UVD_CGC_CTRL, data); 5195 5196 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0); 5197 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0); 5198 } else { 5199 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 5200 tmp &= ~0x3fff; 5201 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp); 5202 5203 orig = data = RREG32(UVD_CGC_CTRL); 5204 data &= ~DCM; 5205 if (orig != data) 5206 WREG32(UVD_CGC_CTRL, data); 5207 5208 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff); 5209 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff); 5210 } 5211 } 5212 5213 static const u32 mc_cg_registers[] = 5214 { 5215 MC_HUB_MISC_HUB_CG, 5216 MC_HUB_MISC_SIP_CG, 5217 MC_HUB_MISC_VM_CG, 5218 MC_XPB_CLK_GAT, 5219 ATC_MISC_CG, 5220 MC_CITF_MISC_WR_CG, 5221 MC_CITF_MISC_RD_CG, 5222 MC_CITF_MISC_VM_CG, 5223 VM_L2_CG, 5224 }; 5225 5226 static void si_enable_mc_ls(struct radeon_device *rdev, 5227 bool enable) 5228 { 5229 int i; 5230 u32 orig, data; 5231 5232 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 5233 orig = data = RREG32(mc_cg_registers[i]); 5234 if (enable) 5235 data |= MC_LS_ENABLE; 5236 else 5237 data &= ~MC_LS_ENABLE; 5238 if (data != orig) 5239 WREG32(mc_cg_registers[i], data); 5240 } 5241 } 5242 5243 5244 static void si_init_cg(struct radeon_device *rdev) 5245 { 5246 si_enable_mgcg(rdev, true); 5247 si_enable_cgcg(rdev, false); 5248 /* disable MC LS on Tahiti */ 5249 if (rdev->family == CHIP_TAHITI) 5250 si_enable_mc_ls(rdev, false); 5251 if (rdev->has_uvd) { 5252 si_enable_uvd_mgcg(rdev, true); 5253 si_init_uvd_internal_cg(rdev); 5254 } 5255 } 5256 5257 static void si_fini_cg(struct radeon_device *rdev) 5258 { 5259 if (rdev->has_uvd) 5260 si_enable_uvd_mgcg(rdev, false); 5261 si_enable_cgcg(rdev, false); 5262 si_enable_mgcg(rdev, false); 5263 } 5264 5265 static void si_init_pg(struct radeon_device *rdev) 5266 { 5267 bool has_pg = false; 5268 #if 0 5269 /* only cape verde supports PG */ 5270 if (rdev->family == CHIP_VERDE) 5271 has_pg = true; 5272 #endif 5273 if (has_pg) { 5274 si_init_ao_cu_mask(rdev); 5275 si_init_dma_pg(rdev); 5276 si_enable_dma_pg(rdev, true); 5277 si_init_gfx_cgpg(rdev); 5278 si_enable_gfx_cgpg(rdev, true); 5279 } else { 5280 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 5281 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 5282 } 5283 } 5284 5285 static void si_fini_pg(struct radeon_device *rdev) 5286 { 5287 bool has_pg = false; 5288 5289 /* only cape verde supports PG */ 5290 if (rdev->family == CHIP_VERDE) 5291 has_pg = true; 5292 5293 if (has_pg) { 5294 si_enable_dma_pg(rdev, false); 5295 si_enable_gfx_cgpg(rdev, false); 5296 } 5297 } 5298 5299 /* 5300 * RLC 5301 */ 5302 void si_rlc_fini(struct radeon_device *rdev) 5303 { 5304 int r; 5305 5306 /* save restore block */ 5307 if (rdev->rlc.save_restore_obj) { 5308 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 5309 if (unlikely(r != 0)) 5310 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r); 5311 radeon_bo_unpin(rdev->rlc.save_restore_obj); 5312 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 5313 5314 radeon_bo_unref(&rdev->rlc.save_restore_obj); 5315 rdev->rlc.save_restore_obj = NULL; 5316 } 5317 5318 /* clear state block */ 5319 if (rdev->rlc.clear_state_obj) { 5320 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 5321 if (unlikely(r != 0)) 5322 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r); 5323 radeon_bo_unpin(rdev->rlc.clear_state_obj); 5324 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 5325 5326 radeon_bo_unref(&rdev->rlc.clear_state_obj); 5327 rdev->rlc.clear_state_obj = NULL; 5328 } 5329 } 5330 5331 #define RLC_CLEAR_STATE_END_MARKER 0x00000001 5332 5333 int si_rlc_init(struct radeon_device *rdev) 5334 { 5335 volatile u32 *dst_ptr; 5336 void *ptr; 5337 u32 dws, data, i, j, k, reg_num; 5338 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index; 5339 u64 reg_list_mc_addr; 5340 const struct cs_section_def *cs_data = si_cs_data; 5341 int r; 5342 5343 /* save restore block */ 5344 if (rdev->rlc.save_restore_obj == NULL) { 5345 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true, 5346 RADEON_GEM_DOMAIN_VRAM, NULL, 5347 &rdev->rlc.save_restore_obj); 5348 if (r) { 5349 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r); 5350 return r; 5351 } 5352 } 5353 5354 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 5355 if (unlikely(r != 0)) { 5356 si_rlc_fini(rdev); 5357 return r; 5358 } 5359 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM, 5360 &rdev->rlc.save_restore_gpu_addr); 5361 if (r) { 5362 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 5363 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r); 5364 si_rlc_fini(rdev); 5365 return r; 5366 } 5367 5368 if (rdev->family == CHIP_VERDE) { 5369 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&ptr); 5370 if (r) { 5371 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r); 5372 si_rlc_fini(rdev); 5373 return r; 5374 } 5375 rdev->rlc.sr_ptr = ptr; 5376 /* write the sr buffer */ 5377 dst_ptr = rdev->rlc.sr_ptr; 5378 for (i = 0; i < ARRAY_SIZE(verde_rlc_save_restore_register_list); i++) { 5379 dst_ptr[i] = verde_rlc_save_restore_register_list[i]; 5380 } 5381 radeon_bo_kunmap(rdev->rlc.save_restore_obj); 5382 } 5383 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 5384 5385 /* clear state block */ 5386 reg_list_num = 0; 5387 dws = 0; 5388 for (i = 0; cs_data[i].section != NULL; i++) { 5389 for (j = 0; cs_data[i].section[j].extent != NULL; j++) { 5390 reg_list_num++; 5391 dws += cs_data[i].section[j].reg_count; 5392 } 5393 } 5394 reg_list_blk_index = (3 * reg_list_num + 2); 5395 dws += reg_list_blk_index; 5396 5397 if (rdev->rlc.clear_state_obj == NULL) { 5398 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, 5399 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj); 5400 if (r) { 5401 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r); 5402 si_rlc_fini(rdev); 5403 return r; 5404 } 5405 } 5406 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 5407 if (unlikely(r != 0)) { 5408 si_rlc_fini(rdev); 5409 return r; 5410 } 5411 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM, 5412 &rdev->rlc.clear_state_gpu_addr); 5413 if (r) { 5414 5415 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 5416 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r); 5417 si_rlc_fini(rdev); 5418 return r; 5419 } 5420 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&ptr); 5421 if (r) { 5422 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r); 5423 si_rlc_fini(rdev); 5424 return r; 5425 } 5426 rdev->rlc.cs_ptr = ptr; 5427 /* set up the cs buffer */ 5428 dst_ptr = rdev->rlc.cs_ptr; 5429 reg_list_hdr_blk_index = 0; 5430 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4); 5431 data = upper_32_bits(reg_list_mc_addr); 5432 dst_ptr[reg_list_hdr_blk_index] = data; 5433 reg_list_hdr_blk_index++; 5434 for (i = 0; cs_data[i].section != NULL; i++) { 5435 for (j = 0; cs_data[i].section[j].extent != NULL; j++) { 5436 reg_num = cs_data[i].section[j].reg_count; 5437 data = reg_list_mc_addr & 0xffffffff; 5438 dst_ptr[reg_list_hdr_blk_index] = data; 5439 reg_list_hdr_blk_index++; 5440 5441 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff; 5442 dst_ptr[reg_list_hdr_blk_index] = data; 5443 reg_list_hdr_blk_index++; 5444 5445 data = 0x08000000 | (reg_num * 4); 5446 dst_ptr[reg_list_hdr_blk_index] = data; 5447 reg_list_hdr_blk_index++; 5448 5449 for (k = 0; k < reg_num; k++) { 5450 data = cs_data[i].section[j].extent[k]; 5451 dst_ptr[reg_list_blk_index + k] = data; 5452 } 5453 reg_list_mc_addr += reg_num * 4; 5454 reg_list_blk_index += reg_num; 5455 } 5456 } 5457 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER; 5458 5459 radeon_bo_kunmap(rdev->rlc.clear_state_obj); 5460 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 5461 5462 return 0; 5463 } 5464 5465 static void si_rlc_reset(struct radeon_device *rdev) 5466 { 5467 u32 tmp = RREG32(GRBM_SOFT_RESET); 5468 5469 tmp |= SOFT_RESET_RLC; 5470 WREG32(GRBM_SOFT_RESET, tmp); 5471 udelay(50); 5472 tmp &= ~SOFT_RESET_RLC; 5473 WREG32(GRBM_SOFT_RESET, tmp); 5474 udelay(50); 5475 } 5476 5477 static void si_rlc_stop(struct radeon_device *rdev) 5478 { 5479 WREG32(RLC_CNTL, 0); 5480 5481 si_enable_gui_idle_interrupt(rdev, false); 5482 5483 si_wait_for_rlc_serdes(rdev); 5484 } 5485 5486 static void si_rlc_start(struct radeon_device *rdev) 5487 { 5488 WREG32(RLC_CNTL, RLC_ENABLE); 5489 5490 si_enable_gui_idle_interrupt(rdev, true); 5491 5492 udelay(50); 5493 } 5494 5495 static bool si_lbpw_supported(struct radeon_device *rdev) 5496 { 5497 u32 tmp; 5498 5499 /* Enable LBPW only for DDR3 */ 5500 tmp = RREG32(MC_SEQ_MISC0); 5501 if ((tmp & 0xF0000000) == 0xB0000000) 5502 return true; 5503 return false; 5504 } 5505 5506 static void si_enable_lbpw(struct radeon_device *rdev, bool enable) 5507 { 5508 u32 tmp; 5509 5510 tmp = RREG32(RLC_LB_CNTL); 5511 if (enable) 5512 tmp |= LOAD_BALANCE_ENABLE; 5513 else 5514 tmp &= ~LOAD_BALANCE_ENABLE; 5515 WREG32(RLC_LB_CNTL, tmp); 5516 5517 if (!enable) { 5518 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5519 WREG32(SPI_LB_CU_MASK, 0x00ff); 5520 } 5521 } 5522 5523 static int si_rlc_resume(struct radeon_device *rdev) 5524 { 5525 u32 i; 5526 const __be32 *fw_data; 5527 5528 if (!rdev->rlc_fw) 5529 return -EINVAL; 5530 5531 si_rlc_stop(rdev); 5532 5533 si_rlc_reset(rdev); 5534 5535 si_init_pg(rdev); 5536 5537 si_init_cg(rdev); 5538 5539 WREG32(RLC_RL_BASE, 0); 5540 WREG32(RLC_RL_SIZE, 0); 5541 WREG32(RLC_LB_CNTL, 0); 5542 WREG32(RLC_LB_CNTR_MAX, 0xffffffff); 5543 WREG32(RLC_LB_CNTR_INIT, 0); 5544 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); 5545 5546 WREG32(RLC_MC_CNTL, 0); 5547 WREG32(RLC_UCODE_CNTL, 0); 5548 5549 fw_data = (const __be32 *)rdev->rlc_fw->data; 5550 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) { 5551 WREG32(RLC_UCODE_ADDR, i); 5552 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 5553 } 5554 WREG32(RLC_UCODE_ADDR, 0); 5555 5556 si_enable_lbpw(rdev, si_lbpw_supported(rdev)); 5557 5558 si_rlc_start(rdev); 5559 5560 return 0; 5561 } 5562 5563 static void si_enable_interrupts(struct radeon_device *rdev) 5564 { 5565 u32 ih_cntl = RREG32(IH_CNTL); 5566 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 5567 5568 ih_cntl |= ENABLE_INTR; 5569 ih_rb_cntl |= IH_RB_ENABLE; 5570 WREG32(IH_CNTL, ih_cntl); 5571 WREG32(IH_RB_CNTL, ih_rb_cntl); 5572 rdev->ih.enabled = true; 5573 } 5574 5575 static void si_disable_interrupts(struct radeon_device *rdev) 5576 { 5577 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 5578 u32 ih_cntl = RREG32(IH_CNTL); 5579 5580 ih_rb_cntl &= ~IH_RB_ENABLE; 5581 ih_cntl &= ~ENABLE_INTR; 5582 WREG32(IH_RB_CNTL, ih_rb_cntl); 5583 WREG32(IH_CNTL, ih_cntl); 5584 /* set rptr, wptr to 0 */ 5585 WREG32(IH_RB_RPTR, 0); 5586 WREG32(IH_RB_WPTR, 0); 5587 rdev->ih.enabled = false; 5588 rdev->ih.rptr = 0; 5589 } 5590 5591 static void si_disable_interrupt_state(struct radeon_device *rdev) 5592 { 5593 u32 tmp; 5594 5595 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 5596 WREG32(CP_INT_CNTL_RING1, 0); 5597 WREG32(CP_INT_CNTL_RING2, 0); 5598 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 5599 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp); 5600 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 5601 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp); 5602 WREG32(GRBM_INT_CNTL, 0); 5603 if (rdev->num_crtc >= 2) { 5604 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 5605 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 5606 } 5607 if (rdev->num_crtc >= 4) { 5608 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 5609 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 5610 } 5611 if (rdev->num_crtc >= 6) { 5612 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 5613 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 5614 } 5615 5616 if (rdev->num_crtc >= 2) { 5617 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 5618 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 5619 } 5620 if (rdev->num_crtc >= 4) { 5621 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 5622 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 5623 } 5624 if (rdev->num_crtc >= 6) { 5625 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 5626 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 5627 } 5628 5629 if (!ASIC_IS_NODCE(rdev)) { 5630 WREG32(DACA_AUTODETECT_INT_CONTROL, 0); 5631 5632 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5633 WREG32(DC_HPD1_INT_CONTROL, tmp); 5634 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5635 WREG32(DC_HPD2_INT_CONTROL, tmp); 5636 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5637 WREG32(DC_HPD3_INT_CONTROL, tmp); 5638 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5639 WREG32(DC_HPD4_INT_CONTROL, tmp); 5640 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5641 WREG32(DC_HPD5_INT_CONTROL, tmp); 5642 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5643 WREG32(DC_HPD6_INT_CONTROL, tmp); 5644 } 5645 } 5646 5647 static int si_irq_init(struct radeon_device *rdev) 5648 { 5649 int ret = 0; 5650 int rb_bufsz; 5651 u32 interrupt_cntl, ih_cntl, ih_rb_cntl; 5652 5653 /* allocate ring */ 5654 ret = r600_ih_ring_alloc(rdev); 5655 if (ret) 5656 return ret; 5657 5658 /* disable irqs */ 5659 si_disable_interrupts(rdev); 5660 5661 /* init rlc */ 5662 ret = si_rlc_resume(rdev); 5663 if (ret) { 5664 r600_ih_ring_fini(rdev); 5665 return ret; 5666 } 5667 5668 /* setup interrupt control */ 5669 /* set dummy read address to ring address */ 5670 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8); 5671 interrupt_cntl = RREG32(INTERRUPT_CNTL); 5672 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi 5673 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN 5674 */ 5675 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE; 5676 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */ 5677 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN; 5678 WREG32(INTERRUPT_CNTL, interrupt_cntl); 5679 5680 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); 5681 rb_bufsz = drm_order(rdev->ih.ring_size / 4); 5682 5683 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | 5684 IH_WPTR_OVERFLOW_CLEAR | 5685 (rb_bufsz << 1)); 5686 5687 if (rdev->wb.enabled) 5688 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE; 5689 5690 /* set the writeback address whether it's enabled or not */ 5691 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); 5692 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); 5693 5694 WREG32(IH_RB_CNTL, ih_rb_cntl); 5695 5696 /* set rptr, wptr to 0 */ 5697 WREG32(IH_RB_RPTR, 0); 5698 WREG32(IH_RB_WPTR, 0); 5699 5700 /* Default settings for IH_CNTL (disabled at first) */ 5701 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0); 5702 /* RPTR_REARM only works if msi's are enabled */ 5703 if (rdev->msi_enabled) 5704 ih_cntl |= RPTR_REARM; 5705 WREG32(IH_CNTL, ih_cntl); 5706 5707 /* force the active interrupt state to all disabled */ 5708 si_disable_interrupt_state(rdev); 5709 5710 pci_enable_busmaster(rdev->dev); 5711 5712 /* enable irqs */ 5713 si_enable_interrupts(rdev); 5714 5715 return ret; 5716 } 5717 5718 int si_irq_set(struct radeon_device *rdev) 5719 { 5720 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; 5721 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; 5722 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 5723 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0; 5724 u32 grbm_int_cntl = 0; 5725 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0; 5726 u32 dma_cntl, dma_cntl1; 5727 u32 thermal_int = 0; 5728 5729 if (!rdev->irq.installed) { 5730 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 5731 return -EINVAL; 5732 } 5733 /* don't enable anything if the ih is disabled */ 5734 if (!rdev->ih.enabled) { 5735 si_disable_interrupts(rdev); 5736 /* force the active interrupt state to all disabled */ 5737 si_disable_interrupt_state(rdev); 5738 return 0; 5739 } 5740 5741 if (!ASIC_IS_NODCE(rdev)) { 5742 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; 5743 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; 5744 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; 5745 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; 5746 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 5747 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 5748 } 5749 5750 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 5751 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 5752 5753 thermal_int = RREG32(CG_THERMAL_INT) & 5754 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 5755 5756 /* enable CP interrupts on all rings */ 5757 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 5758 DRM_DEBUG("si_irq_set: sw int gfx\n"); 5759 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 5760 } 5761 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 5762 DRM_DEBUG("si_irq_set: sw int cp1\n"); 5763 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE; 5764 } 5765 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 5766 DRM_DEBUG("si_irq_set: sw int cp2\n"); 5767 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE; 5768 } 5769 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 5770 DRM_DEBUG("si_irq_set: sw int dma\n"); 5771 dma_cntl |= TRAP_ENABLE; 5772 } 5773 5774 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 5775 DRM_DEBUG("si_irq_set: sw int dma1\n"); 5776 dma_cntl1 |= TRAP_ENABLE; 5777 } 5778 if (rdev->irq.crtc_vblank_int[0] || 5779 atomic_read(&rdev->irq.pflip[0])) { 5780 DRM_DEBUG("si_irq_set: vblank 0\n"); 5781 crtc1 |= VBLANK_INT_MASK; 5782 } 5783 if (rdev->irq.crtc_vblank_int[1] || 5784 atomic_read(&rdev->irq.pflip[1])) { 5785 DRM_DEBUG("si_irq_set: vblank 1\n"); 5786 crtc2 |= VBLANK_INT_MASK; 5787 } 5788 if (rdev->irq.crtc_vblank_int[2] || 5789 atomic_read(&rdev->irq.pflip[2])) { 5790 DRM_DEBUG("si_irq_set: vblank 2\n"); 5791 crtc3 |= VBLANK_INT_MASK; 5792 } 5793 if (rdev->irq.crtc_vblank_int[3] || 5794 atomic_read(&rdev->irq.pflip[3])) { 5795 DRM_DEBUG("si_irq_set: vblank 3\n"); 5796 crtc4 |= VBLANK_INT_MASK; 5797 } 5798 if (rdev->irq.crtc_vblank_int[4] || 5799 atomic_read(&rdev->irq.pflip[4])) { 5800 DRM_DEBUG("si_irq_set: vblank 4\n"); 5801 crtc5 |= VBLANK_INT_MASK; 5802 } 5803 if (rdev->irq.crtc_vblank_int[5] || 5804 atomic_read(&rdev->irq.pflip[5])) { 5805 DRM_DEBUG("si_irq_set: vblank 5\n"); 5806 crtc6 |= VBLANK_INT_MASK; 5807 } 5808 if (rdev->irq.hpd[0]) { 5809 DRM_DEBUG("si_irq_set: hpd 1\n"); 5810 hpd1 |= DC_HPDx_INT_EN; 5811 } 5812 if (rdev->irq.hpd[1]) { 5813 DRM_DEBUG("si_irq_set: hpd 2\n"); 5814 hpd2 |= DC_HPDx_INT_EN; 5815 } 5816 if (rdev->irq.hpd[2]) { 5817 DRM_DEBUG("si_irq_set: hpd 3\n"); 5818 hpd3 |= DC_HPDx_INT_EN; 5819 } 5820 if (rdev->irq.hpd[3]) { 5821 DRM_DEBUG("si_irq_set: hpd 4\n"); 5822 hpd4 |= DC_HPDx_INT_EN; 5823 } 5824 if (rdev->irq.hpd[4]) { 5825 DRM_DEBUG("si_irq_set: hpd 5\n"); 5826 hpd5 |= DC_HPDx_INT_EN; 5827 } 5828 if (rdev->irq.hpd[5]) { 5829 DRM_DEBUG("si_irq_set: hpd 6\n"); 5830 hpd6 |= DC_HPDx_INT_EN; 5831 } 5832 5833 WREG32(CP_INT_CNTL_RING0, cp_int_cntl); 5834 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1); 5835 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2); 5836 5837 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl); 5838 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1); 5839 5840 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 5841 5842 if (rdev->irq.dpm_thermal) { 5843 DRM_DEBUG("dpm thermal\n"); 5844 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; 5845 } 5846 5847 if (rdev->num_crtc >= 2) { 5848 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 5849 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 5850 } 5851 if (rdev->num_crtc >= 4) { 5852 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 5853 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 5854 } 5855 if (rdev->num_crtc >= 6) { 5856 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 5857 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 5858 } 5859 5860 if (rdev->num_crtc >= 2) { 5861 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1); 5862 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2); 5863 } 5864 if (rdev->num_crtc >= 4) { 5865 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3); 5866 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4); 5867 } 5868 if (rdev->num_crtc >= 6) { 5869 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5); 5870 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6); 5871 } 5872 5873 if (!ASIC_IS_NODCE(rdev)) { 5874 WREG32(DC_HPD1_INT_CONTROL, hpd1); 5875 WREG32(DC_HPD2_INT_CONTROL, hpd2); 5876 WREG32(DC_HPD3_INT_CONTROL, hpd3); 5877 WREG32(DC_HPD4_INT_CONTROL, hpd4); 5878 WREG32(DC_HPD5_INT_CONTROL, hpd5); 5879 WREG32(DC_HPD6_INT_CONTROL, hpd6); 5880 } 5881 5882 WREG32(CG_THERMAL_INT, thermal_int); 5883 5884 return 0; 5885 } 5886 5887 static inline void si_irq_ack(struct radeon_device *rdev) 5888 { 5889 u32 tmp; 5890 5891 if (ASIC_IS_NODCE(rdev)) 5892 return; 5893 5894 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS); 5895 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 5896 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 5897 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 5898 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 5899 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 5900 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 5901 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 5902 if (rdev->num_crtc >= 4) { 5903 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 5904 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 5905 } 5906 if (rdev->num_crtc >= 6) { 5907 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 5908 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 5909 } 5910 5911 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 5912 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5913 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 5914 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5915 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) 5916 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 5917 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) 5918 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 5919 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 5920 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 5921 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) 5922 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 5923 5924 if (rdev->num_crtc >= 4) { 5925 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 5926 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5927 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 5928 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5929 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 5930 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 5931 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 5932 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 5933 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 5934 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 5935 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 5936 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 5937 } 5938 5939 if (rdev->num_crtc >= 6) { 5940 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 5941 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5942 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 5943 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5944 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 5945 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 5946 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 5947 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 5948 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 5949 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 5950 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 5951 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 5952 } 5953 5954 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 5955 tmp = RREG32(DC_HPD1_INT_CONTROL); 5956 tmp |= DC_HPDx_INT_ACK; 5957 WREG32(DC_HPD1_INT_CONTROL, tmp); 5958 } 5959 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 5960 tmp = RREG32(DC_HPD2_INT_CONTROL); 5961 tmp |= DC_HPDx_INT_ACK; 5962 WREG32(DC_HPD2_INT_CONTROL, tmp); 5963 } 5964 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 5965 tmp = RREG32(DC_HPD3_INT_CONTROL); 5966 tmp |= DC_HPDx_INT_ACK; 5967 WREG32(DC_HPD3_INT_CONTROL, tmp); 5968 } 5969 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 5970 tmp = RREG32(DC_HPD4_INT_CONTROL); 5971 tmp |= DC_HPDx_INT_ACK; 5972 WREG32(DC_HPD4_INT_CONTROL, tmp); 5973 } 5974 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 5975 tmp = RREG32(DC_HPD5_INT_CONTROL); 5976 tmp |= DC_HPDx_INT_ACK; 5977 WREG32(DC_HPD5_INT_CONTROL, tmp); 5978 } 5979 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 5980 tmp = RREG32(DC_HPD5_INT_CONTROL); 5981 tmp |= DC_HPDx_INT_ACK; 5982 WREG32(DC_HPD6_INT_CONTROL, tmp); 5983 } 5984 } 5985 5986 static void si_irq_disable(struct radeon_device *rdev) 5987 { 5988 si_disable_interrupts(rdev); 5989 /* Wait and acknowledge irq */ 5990 mdelay(1); 5991 si_irq_ack(rdev); 5992 si_disable_interrupt_state(rdev); 5993 } 5994 5995 static void si_irq_suspend(struct radeon_device *rdev) 5996 { 5997 si_irq_disable(rdev); 5998 si_rlc_stop(rdev); 5999 } 6000 6001 static void si_irq_fini(struct radeon_device *rdev) 6002 { 6003 si_irq_suspend(rdev); 6004 r600_ih_ring_fini(rdev); 6005 } 6006 6007 static inline u32 si_get_ih_wptr(struct radeon_device *rdev) 6008 { 6009 u32 wptr, tmp; 6010 6011 if (rdev->wb.enabled) 6012 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 6013 else 6014 wptr = RREG32(IH_RB_WPTR); 6015 6016 if (wptr & RB_OVERFLOW) { 6017 /* When a ring buffer overflow happen start parsing interrupt 6018 * from the last not overwritten vector (wptr + 16). Hopefully 6019 * this should allow us to catchup. 6020 */ 6021 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n", 6022 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask); 6023 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 6024 tmp = RREG32(IH_RB_CNTL); 6025 tmp |= IH_WPTR_OVERFLOW_CLEAR; 6026 WREG32(IH_RB_CNTL, tmp); 6027 } 6028 return (wptr & rdev->ih.ptr_mask); 6029 } 6030 6031 /* SI IV Ring 6032 * Each IV ring entry is 128 bits: 6033 * [7:0] - interrupt source id 6034 * [31:8] - reserved 6035 * [59:32] - interrupt source data 6036 * [63:60] - reserved 6037 * [71:64] - RINGID 6038 * [79:72] - VMID 6039 * [127:80] - reserved 6040 */ 6041 irqreturn_t si_irq_process(struct radeon_device *rdev) 6042 { 6043 u32 wptr; 6044 u32 rptr; 6045 u32 src_id, src_data, ring_id; 6046 u32 ring_index; 6047 bool queue_hotplug = false; 6048 bool queue_thermal = false; 6049 u32 status, addr; 6050 6051 if (!rdev->ih.enabled || rdev->shutdown) 6052 return IRQ_NONE; 6053 6054 wptr = si_get_ih_wptr(rdev); 6055 6056 restart_ih: 6057 /* is somebody else already processing irqs? */ 6058 if (atomic_xchg(&rdev->ih.lock, 1)) 6059 return IRQ_NONE; 6060 6061 rptr = rdev->ih.rptr; 6062 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 6063 6064 /* Order reading of wptr vs. reading of IH ring data */ 6065 rmb(); 6066 6067 /* display interrupts */ 6068 si_irq_ack(rdev); 6069 6070 while (rptr != wptr) { 6071 /* wptr/rptr are in bytes! */ 6072 ring_index = rptr / 4; 6073 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 6074 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 6075 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; 6076 6077 switch (src_id) { 6078 case 1: /* D1 vblank/vline */ 6079 switch (src_data) { 6080 case 0: /* D1 vblank */ 6081 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) { 6082 if (rdev->irq.crtc_vblank_int[0]) { 6083 drm_handle_vblank(rdev->ddev, 0); 6084 rdev->pm.vblank_sync = true; 6085 wake_up(&rdev->irq.vblank_queue); 6086 } 6087 if (atomic_read(&rdev->irq.pflip[0])) 6088 radeon_crtc_handle_flip(rdev, 0); 6089 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 6090 DRM_DEBUG("IH: D1 vblank\n"); 6091 } 6092 break; 6093 case 1: /* D1 vline */ 6094 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) { 6095 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; 6096 DRM_DEBUG("IH: D1 vline\n"); 6097 } 6098 break; 6099 default: 6100 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6101 break; 6102 } 6103 break; 6104 case 2: /* D2 vblank/vline */ 6105 switch (src_data) { 6106 case 0: /* D2 vblank */ 6107 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 6108 if (rdev->irq.crtc_vblank_int[1]) { 6109 drm_handle_vblank(rdev->ddev, 1); 6110 rdev->pm.vblank_sync = true; 6111 wake_up(&rdev->irq.vblank_queue); 6112 } 6113 if (atomic_read(&rdev->irq.pflip[1])) 6114 radeon_crtc_handle_flip(rdev, 1); 6115 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 6116 DRM_DEBUG("IH: D2 vblank\n"); 6117 } 6118 break; 6119 case 1: /* D2 vline */ 6120 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) { 6121 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 6122 DRM_DEBUG("IH: D2 vline\n"); 6123 } 6124 break; 6125 default: 6126 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6127 break; 6128 } 6129 break; 6130 case 3: /* D3 vblank/vline */ 6131 switch (src_data) { 6132 case 0: /* D3 vblank */ 6133 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 6134 if (rdev->irq.crtc_vblank_int[2]) { 6135 drm_handle_vblank(rdev->ddev, 2); 6136 rdev->pm.vblank_sync = true; 6137 wake_up(&rdev->irq.vblank_queue); 6138 } 6139 if (atomic_read(&rdev->irq.pflip[2])) 6140 radeon_crtc_handle_flip(rdev, 2); 6141 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 6142 DRM_DEBUG("IH: D3 vblank\n"); 6143 } 6144 break; 6145 case 1: /* D3 vline */ 6146 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 6147 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 6148 DRM_DEBUG("IH: D3 vline\n"); 6149 } 6150 break; 6151 default: 6152 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6153 break; 6154 } 6155 break; 6156 case 4: /* D4 vblank/vline */ 6157 switch (src_data) { 6158 case 0: /* D4 vblank */ 6159 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 6160 if (rdev->irq.crtc_vblank_int[3]) { 6161 drm_handle_vblank(rdev->ddev, 3); 6162 rdev->pm.vblank_sync = true; 6163 wake_up(&rdev->irq.vblank_queue); 6164 } 6165 if (atomic_read(&rdev->irq.pflip[3])) 6166 radeon_crtc_handle_flip(rdev, 3); 6167 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 6168 DRM_DEBUG("IH: D4 vblank\n"); 6169 } 6170 break; 6171 case 1: /* D4 vline */ 6172 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 6173 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 6174 DRM_DEBUG("IH: D4 vline\n"); 6175 } 6176 break; 6177 default: 6178 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6179 break; 6180 } 6181 break; 6182 case 5: /* D5 vblank/vline */ 6183 switch (src_data) { 6184 case 0: /* D5 vblank */ 6185 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 6186 if (rdev->irq.crtc_vblank_int[4]) { 6187 drm_handle_vblank(rdev->ddev, 4); 6188 rdev->pm.vblank_sync = true; 6189 wake_up(&rdev->irq.vblank_queue); 6190 } 6191 if (atomic_read(&rdev->irq.pflip[4])) 6192 radeon_crtc_handle_flip(rdev, 4); 6193 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 6194 DRM_DEBUG("IH: D5 vblank\n"); 6195 } 6196 break; 6197 case 1: /* D5 vline */ 6198 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 6199 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 6200 DRM_DEBUG("IH: D5 vline\n"); 6201 } 6202 break; 6203 default: 6204 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6205 break; 6206 } 6207 break; 6208 case 6: /* D6 vblank/vline */ 6209 switch (src_data) { 6210 case 0: /* D6 vblank */ 6211 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 6212 if (rdev->irq.crtc_vblank_int[5]) { 6213 drm_handle_vblank(rdev->ddev, 5); 6214 rdev->pm.vblank_sync = true; 6215 wake_up(&rdev->irq.vblank_queue); 6216 } 6217 if (atomic_read(&rdev->irq.pflip[5])) 6218 radeon_crtc_handle_flip(rdev, 5); 6219 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 6220 DRM_DEBUG("IH: D6 vblank\n"); 6221 } 6222 break; 6223 case 1: /* D6 vline */ 6224 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 6225 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 6226 DRM_DEBUG("IH: D6 vline\n"); 6227 } 6228 break; 6229 default: 6230 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6231 break; 6232 } 6233 break; 6234 case 42: /* HPD hotplug */ 6235 switch (src_data) { 6236 case 0: 6237 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 6238 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; 6239 queue_hotplug = true; 6240 DRM_DEBUG("IH: HPD1\n"); 6241 } 6242 break; 6243 case 1: 6244 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 6245 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; 6246 queue_hotplug = true; 6247 DRM_DEBUG("IH: HPD2\n"); 6248 } 6249 break; 6250 case 2: 6251 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 6252 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 6253 queue_hotplug = true; 6254 DRM_DEBUG("IH: HPD3\n"); 6255 } 6256 break; 6257 case 3: 6258 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 6259 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 6260 queue_hotplug = true; 6261 DRM_DEBUG("IH: HPD4\n"); 6262 } 6263 break; 6264 case 4: 6265 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 6266 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 6267 queue_hotplug = true; 6268 DRM_DEBUG("IH: HPD5\n"); 6269 } 6270 break; 6271 case 5: 6272 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 6273 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 6274 queue_hotplug = true; 6275 DRM_DEBUG("IH: HPD6\n"); 6276 } 6277 break; 6278 default: 6279 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6280 break; 6281 } 6282 break; 6283 case 146: 6284 case 147: 6285 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 6286 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 6287 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 6288 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 6289 addr); 6290 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 6291 status); 6292 si_vm_decode_fault(rdev, status, addr); 6293 /* reset addr and status */ 6294 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 6295 break; 6296 case 176: /* RINGID0 CP_INT */ 6297 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 6298 break; 6299 case 177: /* RINGID1 CP_INT */ 6300 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 6301 break; 6302 case 178: /* RINGID2 CP_INT */ 6303 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 6304 break; 6305 case 181: /* CP EOP event */ 6306 DRM_DEBUG("IH: CP EOP\n"); 6307 switch (ring_id) { 6308 case 0: 6309 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 6310 break; 6311 case 1: 6312 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 6313 break; 6314 case 2: 6315 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 6316 break; 6317 } 6318 break; 6319 case 224: /* DMA trap event */ 6320 DRM_DEBUG("IH: DMA trap\n"); 6321 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 6322 break; 6323 case 230: /* thermal low to high */ 6324 DRM_DEBUG("IH: thermal low to high\n"); 6325 rdev->pm.dpm.thermal.high_to_low = false; 6326 queue_thermal = true; 6327 break; 6328 case 231: /* thermal high to low */ 6329 DRM_DEBUG("IH: thermal high to low\n"); 6330 rdev->pm.dpm.thermal.high_to_low = true; 6331 queue_thermal = true; 6332 break; 6333 case 233: /* GUI IDLE */ 6334 DRM_DEBUG("IH: GUI idle\n"); 6335 break; 6336 case 244: /* DMA trap event */ 6337 DRM_DEBUG("IH: DMA1 trap\n"); 6338 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 6339 break; 6340 default: 6341 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6342 break; 6343 } 6344 6345 /* wptr/rptr are in bytes! */ 6346 rptr += 16; 6347 rptr &= rdev->ih.ptr_mask; 6348 } 6349 if (queue_hotplug) 6350 taskqueue_enqueue(rdev->tq, &rdev->hotplug_work); 6351 if (queue_thermal && rdev->pm.dpm_enabled) 6352 taskqueue_enqueue(rdev->tq, &rdev->pm.dpm.thermal.work); 6353 rdev->ih.rptr = rptr; 6354 WREG32(IH_RB_RPTR, rdev->ih.rptr); 6355 atomic_set(&rdev->ih.lock, 0); 6356 6357 /* make sure wptr hasn't changed while processing */ 6358 wptr = si_get_ih_wptr(rdev); 6359 if (wptr != rptr) 6360 goto restart_ih; 6361 6362 return IRQ_HANDLED; 6363 } 6364 6365 /** 6366 * si_copy_dma - copy pages using the DMA engine 6367 * 6368 * @rdev: radeon_device pointer 6369 * @src_offset: src GPU address 6370 * @dst_offset: dst GPU address 6371 * @num_gpu_pages: number of GPU pages to xfer 6372 * @fence: radeon fence object 6373 * 6374 * Copy GPU paging using the DMA engine (SI). 6375 * Used by the radeon ttm implementation to move pages if 6376 * registered as the asic copy callback. 6377 */ 6378 int si_copy_dma(struct radeon_device *rdev, 6379 uint64_t src_offset, uint64_t dst_offset, 6380 unsigned num_gpu_pages, 6381 struct radeon_fence **fence) 6382 { 6383 struct radeon_semaphore *sem = NULL; 6384 int ring_index = rdev->asic->copy.dma_ring_index; 6385 struct radeon_ring *ring = &rdev->ring[ring_index]; 6386 u32 size_in_bytes, cur_size_in_bytes; 6387 int i, num_loops; 6388 int r = 0; 6389 6390 r = radeon_semaphore_create(rdev, &sem); 6391 if (r) { 6392 DRM_ERROR("radeon: moving bo (%d).\n", r); 6393 return r; 6394 } 6395 6396 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); 6397 num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff); 6398 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11); 6399 if (r) { 6400 DRM_ERROR("radeon: moving bo (%d).\n", r); 6401 radeon_semaphore_free(rdev, &sem, NULL); 6402 return r; 6403 } 6404 6405 if (radeon_fence_need_sync(*fence, ring->idx)) { 6406 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring, 6407 ring->idx); 6408 radeon_fence_note_sync(*fence, ring->idx); 6409 } else { 6410 radeon_semaphore_free(rdev, &sem, NULL); 6411 } 6412 6413 for (i = 0; i < num_loops; i++) { 6414 cur_size_in_bytes = size_in_bytes; 6415 if (cur_size_in_bytes > 0xFFFFF) 6416 cur_size_in_bytes = 0xFFFFF; 6417 size_in_bytes -= cur_size_in_bytes; 6418 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes)); 6419 radeon_ring_write(ring, dst_offset & 0xffffffff); 6420 radeon_ring_write(ring, src_offset & 0xffffffff); 6421 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff); 6422 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff); 6423 src_offset += cur_size_in_bytes; 6424 dst_offset += cur_size_in_bytes; 6425 } 6426 6427 r = radeon_fence_emit(rdev, fence, ring->idx); 6428 if (r) { 6429 radeon_ring_unlock_undo(rdev, ring); 6430 return r; 6431 } 6432 6433 radeon_ring_unlock_commit(rdev, ring); 6434 radeon_semaphore_free(rdev, &sem, *fence); 6435 6436 return r; 6437 } 6438 6439 /* 6440 * startup/shutdown callbacks 6441 */ 6442 static int si_startup(struct radeon_device *rdev) 6443 { 6444 struct radeon_ring *ring; 6445 int r; 6446 6447 /* enable pcie gen2/3 link */ 6448 si_pcie_gen3_enable(rdev); 6449 /* enable aspm */ 6450 si_program_aspm(rdev); 6451 6452 si_mc_program(rdev); 6453 6454 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 6455 !rdev->rlc_fw || !rdev->mc_fw) { 6456 r = si_init_microcode(rdev); 6457 if (r) { 6458 DRM_ERROR("Failed to load firmware!\n"); 6459 return r; 6460 } 6461 } 6462 6463 r = si_mc_load_microcode(rdev); 6464 if (r) { 6465 DRM_ERROR("Failed to load MC firmware!\n"); 6466 return r; 6467 } 6468 6469 r = r600_vram_scratch_init(rdev); 6470 if (r) 6471 return r; 6472 6473 r = si_pcie_gart_enable(rdev); 6474 if (r) 6475 return r; 6476 si_gpu_init(rdev); 6477 6478 /* allocate rlc buffers */ 6479 r = si_rlc_init(rdev); 6480 if (r) { 6481 DRM_ERROR("Failed to init rlc BOs!\n"); 6482 return r; 6483 } 6484 6485 /* allocate wb buffer */ 6486 r = radeon_wb_init(rdev); 6487 if (r) 6488 return r; 6489 6490 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 6491 if (r) { 6492 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 6493 return r; 6494 } 6495 6496 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 6497 if (r) { 6498 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 6499 return r; 6500 } 6501 6502 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 6503 if (r) { 6504 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 6505 return r; 6506 } 6507 6508 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 6509 if (r) { 6510 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 6511 return r; 6512 } 6513 6514 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 6515 if (r) { 6516 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 6517 return r; 6518 } 6519 6520 if (rdev->has_uvd) { 6521 r = rv770_uvd_resume(rdev); 6522 if (!r) { 6523 r = radeon_fence_driver_start_ring(rdev, 6524 R600_RING_TYPE_UVD_INDEX); 6525 if (r) 6526 dev_err(rdev->dev, "UVD fences init error (%d).\n", r); 6527 } 6528 if (r) 6529 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 6530 } 6531 6532 /* Enable IRQ */ 6533 if (!rdev->irq.installed) { 6534 r = radeon_irq_kms_init(rdev); 6535 if (r) 6536 return r; 6537 } 6538 6539 r = si_irq_init(rdev); 6540 if (r) { 6541 DRM_ERROR("radeon: IH init failed (%d).\n", r); 6542 radeon_irq_kms_fini(rdev); 6543 return r; 6544 } 6545 si_irq_set(rdev); 6546 6547 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 6548 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 6549 CP_RB0_RPTR, CP_RB0_WPTR, 6550 0, 0xfffff, RADEON_CP_PACKET2); 6551 if (r) 6552 return r; 6553 6554 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 6555 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, 6556 CP_RB1_RPTR, CP_RB1_WPTR, 6557 0, 0xfffff, RADEON_CP_PACKET2); 6558 if (r) 6559 return r; 6560 6561 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 6562 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, 6563 CP_RB2_RPTR, CP_RB2_WPTR, 6564 0, 0xfffff, RADEON_CP_PACKET2); 6565 if (r) 6566 return r; 6567 6568 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 6569 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 6570 DMA_RB_RPTR + DMA0_REGISTER_OFFSET, 6571 DMA_RB_WPTR + DMA0_REGISTER_OFFSET, 6572 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); 6573 if (r) 6574 return r; 6575 6576 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 6577 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, 6578 DMA_RB_RPTR + DMA1_REGISTER_OFFSET, 6579 DMA_RB_WPTR + DMA1_REGISTER_OFFSET, 6580 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); 6581 if (r) 6582 return r; 6583 6584 r = si_cp_load_microcode(rdev); 6585 if (r) 6586 return r; 6587 r = si_cp_resume(rdev); 6588 if (r) 6589 return r; 6590 6591 r = cayman_dma_resume(rdev); 6592 if (r) 6593 return r; 6594 6595 if (rdev->has_uvd) { 6596 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 6597 if (ring->ring_size) { 6598 r = radeon_ring_init(rdev, ring, ring->ring_size, 6599 R600_WB_UVD_RPTR_OFFSET, 6600 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR, 6601 0, 0xfffff, RADEON_CP_PACKET2); 6602 if (!r) 6603 r = r600_uvd_init(rdev); 6604 if (r) 6605 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); 6606 } 6607 } 6608 6609 r = radeon_ib_pool_init(rdev); 6610 if (r) { 6611 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 6612 return r; 6613 } 6614 6615 r = radeon_vm_manager_init(rdev); 6616 if (r) { 6617 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); 6618 return r; 6619 } 6620 6621 return 0; 6622 } 6623 6624 int si_resume(struct radeon_device *rdev) 6625 { 6626 int r; 6627 6628 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw, 6629 * posting will perform necessary task to bring back GPU into good 6630 * shape. 6631 */ 6632 /* post card */ 6633 atom_asic_init(rdev->mode_info.atom_context); 6634 6635 /* init golden registers */ 6636 si_init_golden_registers(rdev); 6637 6638 rdev->accel_working = true; 6639 r = si_startup(rdev); 6640 if (r) { 6641 DRM_ERROR("si startup failed on resume\n"); 6642 rdev->accel_working = false; 6643 return r; 6644 } 6645 6646 return r; 6647 6648 } 6649 6650 int si_suspend(struct radeon_device *rdev) 6651 { 6652 radeon_vm_manager_fini(rdev); 6653 si_cp_enable(rdev, false); 6654 cayman_dma_stop(rdev); 6655 if (rdev->has_uvd) { 6656 r600_uvd_stop(rdev); 6657 radeon_uvd_suspend(rdev); 6658 } 6659 si_irq_suspend(rdev); 6660 radeon_wb_disable(rdev); 6661 si_pcie_gart_disable(rdev); 6662 return 0; 6663 } 6664 6665 /* Plan is to move initialization in that function and use 6666 * helper function so that radeon_device_init pretty much 6667 * do nothing more than calling asic specific function. This 6668 * should also allow to remove a bunch of callback function 6669 * like vram_info. 6670 */ 6671 int si_init(struct radeon_device *rdev) 6672 { 6673 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 6674 int r; 6675 6676 /* Read BIOS */ 6677 if (!radeon_get_bios(rdev)) { 6678 if (ASIC_IS_AVIVO(rdev)) 6679 return -EINVAL; 6680 } 6681 /* Must be an ATOMBIOS */ 6682 if (!rdev->is_atom_bios) { 6683 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); 6684 return -EINVAL; 6685 } 6686 r = radeon_atombios_init(rdev); 6687 if (r) 6688 return r; 6689 6690 /* Post card if necessary */ 6691 if (!radeon_card_posted(rdev)) { 6692 if (!rdev->bios) { 6693 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 6694 return -EINVAL; 6695 } 6696 DRM_INFO("GPU not posted. posting now...\n"); 6697 atom_asic_init(rdev->mode_info.atom_context); 6698 } 6699 /* init golden registers */ 6700 si_init_golden_registers(rdev); 6701 /* Initialize scratch registers */ 6702 si_scratch_init(rdev); 6703 /* Initialize surface registers */ 6704 radeon_surface_init(rdev); 6705 /* Initialize clocks */ 6706 radeon_get_clock_info(rdev->ddev); 6707 6708 /* Fence driver */ 6709 r = radeon_fence_driver_init(rdev); 6710 if (r) 6711 return r; 6712 6713 /* initialize memory controller */ 6714 r = si_mc_init(rdev); 6715 if (r) 6716 return r; 6717 /* Memory manager */ 6718 r = radeon_bo_init(rdev); 6719 if (r) 6720 return r; 6721 6722 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 6723 ring->ring_obj = NULL; 6724 r600_ring_init(rdev, ring, 1024 * 1024); 6725 6726 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 6727 ring->ring_obj = NULL; 6728 r600_ring_init(rdev, ring, 1024 * 1024); 6729 6730 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 6731 ring->ring_obj = NULL; 6732 r600_ring_init(rdev, ring, 1024 * 1024); 6733 6734 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 6735 ring->ring_obj = NULL; 6736 r600_ring_init(rdev, ring, 64 * 1024); 6737 6738 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 6739 ring->ring_obj = NULL; 6740 r600_ring_init(rdev, ring, 64 * 1024); 6741 6742 if (rdev->has_uvd) { 6743 r = radeon_uvd_init(rdev); 6744 if (!r) { 6745 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 6746 ring->ring_obj = NULL; 6747 r600_ring_init(rdev, ring, 4096); 6748 } 6749 } 6750 6751 rdev->ih.ring_obj = NULL; 6752 r600_ih_ring_init(rdev, 64 * 1024); 6753 6754 r = r600_pcie_gart_init(rdev); 6755 if (r) 6756 return r; 6757 6758 rdev->accel_working = true; 6759 r = si_startup(rdev); 6760 if (r) { 6761 dev_err(rdev->dev, "disabling GPU acceleration\n"); 6762 si_cp_fini(rdev); 6763 cayman_dma_fini(rdev); 6764 si_irq_fini(rdev); 6765 si_rlc_fini(rdev); 6766 radeon_wb_fini(rdev); 6767 radeon_ib_pool_fini(rdev); 6768 radeon_vm_manager_fini(rdev); 6769 radeon_irq_kms_fini(rdev); 6770 si_pcie_gart_fini(rdev); 6771 rdev->accel_working = false; 6772 } 6773 6774 /* Don't start up if the MC ucode is missing. 6775 * The default clocks and voltages before the MC ucode 6776 * is loaded are not suffient for advanced operations. 6777 */ 6778 if (!rdev->mc_fw) { 6779 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 6780 return -EINVAL; 6781 } 6782 6783 return 0; 6784 } 6785 6786 void si_fini(struct radeon_device *rdev) 6787 { 6788 si_cp_fini(rdev); 6789 cayman_dma_fini(rdev); 6790 si_irq_fini(rdev); 6791 si_rlc_fini(rdev); 6792 si_fini_cg(rdev); 6793 si_fini_pg(rdev); 6794 radeon_wb_fini(rdev); 6795 radeon_vm_manager_fini(rdev); 6796 radeon_ib_pool_fini(rdev); 6797 radeon_irq_kms_fini(rdev); 6798 if (rdev->has_uvd) { 6799 r600_uvd_stop(rdev); 6800 radeon_uvd_fini(rdev); 6801 } 6802 si_pcie_gart_fini(rdev); 6803 r600_vram_scratch_fini(rdev); 6804 radeon_gem_fini(rdev); 6805 radeon_fence_driver_fini(rdev); 6806 radeon_bo_fini(rdev); 6807 radeon_atombios_fini(rdev); 6808 si_fini_microcode(rdev); 6809 kfree(rdev->bios); 6810 rdev->bios = NULL; 6811 } 6812 6813 /** 6814 * si_get_gpu_clock_counter - return GPU clock counter snapshot 6815 * 6816 * @rdev: radeon_device pointer 6817 * 6818 * Fetches a GPU clock counter snapshot (SI). 6819 * Returns the 64 bit clock counter snapshot. 6820 */ 6821 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev) 6822 { 6823 uint64_t clock; 6824 6825 spin_lock(&rdev->gpu_clock_mutex); 6826 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1); 6827 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) | 6828 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 6829 spin_unlock(&rdev->gpu_clock_mutex); 6830 return clock; 6831 } 6832 6833 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 6834 { 6835 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0; 6836 int r; 6837 6838 /* bypass vclk and dclk with bclk */ 6839 WREG32_P(CG_UPLL_FUNC_CNTL_2, 6840 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1), 6841 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 6842 6843 /* put PLL in bypass mode */ 6844 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK); 6845 6846 if (!vclk || !dclk) { 6847 /* keep the Bypass mode, put PLL to sleep */ 6848 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 6849 return 0; 6850 } 6851 6852 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000, 6853 16384, 0x03FFFFFF, 0, 128, 5, 6854 &fb_div, &vclk_div, &dclk_div); 6855 if (r) 6856 return r; 6857 6858 /* set RESET_ANTI_MUX to 0 */ 6859 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK); 6860 6861 /* set VCO_MODE to 1 */ 6862 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK); 6863 6864 /* toggle UPLL_SLEEP to 1 then back to 0 */ 6865 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 6866 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK); 6867 6868 /* deassert UPLL_RESET */ 6869 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 6870 6871 mdelay(1); 6872 6873 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 6874 if (r) 6875 return r; 6876 6877 /* assert UPLL_RESET again */ 6878 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK); 6879 6880 /* disable spread spectrum. */ 6881 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK); 6882 6883 /* set feedback divider */ 6884 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK); 6885 6886 /* set ref divider to 0 */ 6887 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK); 6888 6889 if (fb_div < 307200) 6890 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9); 6891 else 6892 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9); 6893 6894 /* set PDIV_A and PDIV_B */ 6895 WREG32_P(CG_UPLL_FUNC_CNTL_2, 6896 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div), 6897 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK)); 6898 6899 /* give the PLL some time to settle */ 6900 mdelay(15); 6901 6902 /* deassert PLL_RESET */ 6903 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 6904 6905 mdelay(15); 6906 6907 /* switch from bypass mode to normal mode */ 6908 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK); 6909 6910 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 6911 if (r) 6912 return r; 6913 6914 /* switch VCLK and DCLK selection */ 6915 WREG32_P(CG_UPLL_FUNC_CNTL_2, 6916 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2), 6917 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 6918 6919 mdelay(100); 6920 6921 return 0; 6922 } 6923 6924 static struct pci_dev dev_to_pcidev(device_t dev) 6925 { 6926 struct pci_dev pdev; 6927 pdev.dev = dev; 6928 return pdev; 6929 } 6930 6931 static void si_pcie_gen3_enable(struct radeon_device *rdev) 6932 { 6933 #if 0 6934 struct pci_dev *root = rdev->dev->bus->self; 6935 #else 6936 device_t root = device_get_parent(rdev->dev); 6937 #endif 6938 int bridge_pos, gpu_pos; 6939 u32 speed_cntl, mask, current_data_rate; 6940 int ret, i; 6941 u16 tmp16; 6942 struct pci_dev root_pdev = dev_to_pcidev(root); 6943 struct pci_dev pdev = dev_to_pcidev(rdev->dev); 6944 6945 if (radeon_pcie_gen2 == 0) 6946 return; 6947 6948 if (rdev->flags & RADEON_IS_IGP) 6949 return; 6950 6951 if (!(rdev->flags & RADEON_IS_PCIE)) 6952 return; 6953 6954 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); 6955 if (ret != 0) 6956 return; 6957 6958 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80))) 6959 return; 6960 6961 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 6962 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >> 6963 LC_CURRENT_DATA_RATE_SHIFT; 6964 if (mask & DRM_PCIE_SPEED_80) { 6965 if (current_data_rate == 2) { 6966 DRM_INFO("PCIE gen 3 link speeds already enabled\n"); 6967 return; 6968 } 6969 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n"); 6970 } else if (mask & DRM_PCIE_SPEED_50) { 6971 if (current_data_rate == 1) { 6972 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 6973 return; 6974 } 6975 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 6976 } 6977 6978 bridge_pos = pci_get_pciecap_ptr(root); 6979 if (!bridge_pos) 6980 return; 6981 6982 gpu_pos = pci_get_pciecap_ptr(rdev->dev); 6983 if (!gpu_pos) 6984 return; 6985 6986 if (mask & DRM_PCIE_SPEED_80) { 6987 /* re-try equalization if gen3 is not already enabled */ 6988 if (current_data_rate != 2) { 6989 u16 bridge_cfg, gpu_cfg; 6990 u16 bridge_cfg2, gpu_cfg2; 6991 u32 max_lw, current_lw, tmp; 6992 6993 pci_read_config_word(&root_pdev, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 6994 pci_read_config_word(&pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 6995 6996 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD; 6997 pci_write_config_word(&root_pdev, bridge_pos + PCI_EXP_LNKCTL, tmp16); 6998 6999 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD; 7000 pci_write_config_word(&pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 7001 7002 tmp = RREG32_PCIE(PCIE_LC_STATUS1); 7003 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT; 7004 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT; 7005 7006 if (current_lw < max_lw) { 7007 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 7008 if (tmp & LC_RENEGOTIATION_SUPPORT) { 7009 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS); 7010 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT); 7011 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW; 7012 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp); 7013 } 7014 } 7015 7016 for (i = 0; i < 10; i++) { 7017 /* check status */ 7018 pci_read_config_word(&pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); 7019 if (tmp16 & PCI_EXP_DEVSTA_TRPND) 7020 break; 7021 7022 pci_read_config_word(&root_pdev, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 7023 pci_read_config_word(&pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 7024 7025 pci_read_config_word(&root_pdev, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2); 7026 pci_read_config_word(&pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); 7027 7028 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 7029 tmp |= LC_SET_QUIESCE; 7030 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 7031 7032 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 7033 tmp |= LC_REDO_EQ; 7034 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 7035 7036 mdelay(100); 7037 7038 /* linkctl */ 7039 pci_read_config_word(&root_pdev, bridge_pos + PCI_EXP_LNKCTL, &tmp16); 7040 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 7041 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD); 7042 pci_write_config_word(&root_pdev, bridge_pos + PCI_EXP_LNKCTL, tmp16); 7043 7044 pci_read_config_word(&pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); 7045 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 7046 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD); 7047 pci_write_config_word(&pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 7048 7049 /* linkctl2 */ 7050 pci_read_config_word(&root_pdev, bridge_pos + PCI_EXP_LNKCTL2, &tmp16); 7051 tmp16 &= ~((1 << 4) | (7 << 9)); 7052 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9))); 7053 pci_write_config_word(&root_pdev, bridge_pos + PCI_EXP_LNKCTL2, tmp16); 7054 7055 pci_read_config_word(&pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 7056 tmp16 &= ~((1 << 4) | (7 << 9)); 7057 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9))); 7058 pci_write_config_word(&pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 7059 7060 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 7061 tmp &= ~LC_SET_QUIESCE; 7062 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 7063 } 7064 } 7065 } 7066 7067 /* set the link speed */ 7068 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE; 7069 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE; 7070 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 7071 7072 pci_read_config_word(&pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 7073 tmp16 &= ~0xf; 7074 if (mask & DRM_PCIE_SPEED_80) 7075 tmp16 |= 3; /* gen3 */ 7076 else if (mask & DRM_PCIE_SPEED_50) 7077 tmp16 |= 2; /* gen2 */ 7078 else 7079 tmp16 |= 1; /* gen1 */ 7080 pci_write_config_word(&pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 7081 7082 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 7083 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE; 7084 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 7085 7086 for (i = 0; i < rdev->usec_timeout; i++) { 7087 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 7088 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0) 7089 break; 7090 udelay(1); 7091 } 7092 } 7093 7094 static void si_program_aspm(struct radeon_device *rdev) 7095 { 7096 u32 data, orig; 7097 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false; 7098 bool disable_clkreq = false; 7099 7100 if (radeon_aspm == 0) 7101 return; 7102 7103 if (!(rdev->flags & RADEON_IS_PCIE)) 7104 return; 7105 7106 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 7107 data &= ~LC_XMIT_N_FTS_MASK; 7108 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN; 7109 if (orig != data) 7110 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data); 7111 7112 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3); 7113 data |= LC_GO_TO_RECOVERY; 7114 if (orig != data) 7115 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data); 7116 7117 orig = data = RREG32_PCIE(PCIE_P_CNTL); 7118 data |= P_IGNORE_EDB_ERR; 7119 if (orig != data) 7120 WREG32_PCIE(PCIE_P_CNTL, data); 7121 7122 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 7123 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 7124 data |= LC_PMI_TO_L1_DIS; 7125 if (!disable_l0s) 7126 data |= LC_L0S_INACTIVITY(7); 7127 7128 if (!disable_l1) { 7129 data |= LC_L1_INACTIVITY(7); 7130 data &= ~LC_PMI_TO_L1_DIS; 7131 if (orig != data) 7132 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 7133 7134 if (!disable_plloff_in_l1) { 7135 bool clk_req_support; 7136 7137 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 7138 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 7139 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 7140 if (orig != data) 7141 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 7142 7143 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 7144 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 7145 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 7146 if (orig != data) 7147 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 7148 7149 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 7150 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 7151 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 7152 if (orig != data) 7153 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 7154 7155 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 7156 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 7157 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 7158 if (orig != data) 7159 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 7160 7161 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) { 7162 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 7163 data &= ~PLL_RAMP_UP_TIME_0_MASK; 7164 if (orig != data) 7165 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 7166 7167 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 7168 data &= ~PLL_RAMP_UP_TIME_1_MASK; 7169 if (orig != data) 7170 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 7171 7172 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2); 7173 data &= ~PLL_RAMP_UP_TIME_2_MASK; 7174 if (orig != data) 7175 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data); 7176 7177 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3); 7178 data &= ~PLL_RAMP_UP_TIME_3_MASK; 7179 if (orig != data) 7180 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data); 7181 7182 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 7183 data &= ~PLL_RAMP_UP_TIME_0_MASK; 7184 if (orig != data) 7185 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 7186 7187 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 7188 data &= ~PLL_RAMP_UP_TIME_1_MASK; 7189 if (orig != data) 7190 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 7191 7192 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2); 7193 data &= ~PLL_RAMP_UP_TIME_2_MASK; 7194 if (orig != data) 7195 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data); 7196 7197 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3); 7198 data &= ~PLL_RAMP_UP_TIME_3_MASK; 7199 if (orig != data) 7200 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data); 7201 } 7202 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 7203 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 7204 data |= LC_DYN_LANES_PWR_STATE(3); 7205 if (orig != data) 7206 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 7207 7208 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL); 7209 data &= ~LS2_EXIT_TIME_MASK; 7210 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN)) 7211 data |= LS2_EXIT_TIME(5); 7212 if (orig != data) 7213 WREG32_PIF_PHY0(PB0_PIF_CNTL, data); 7214 7215 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL); 7216 data &= ~LS2_EXIT_TIME_MASK; 7217 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN)) 7218 data |= LS2_EXIT_TIME(5); 7219 if (orig != data) 7220 WREG32_PIF_PHY1(PB1_PIF_CNTL, data); 7221 7222 if (!disable_clkreq) { 7223 #ifdef MN_TODO 7224 struct pci_dev *root = rdev->pdev->bus->self; 7225 u32 lnkcap; 7226 7227 clk_req_support = false; 7228 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap); 7229 if (lnkcap & PCI_EXP_LNKCAP_CLKPM) 7230 clk_req_support = true; 7231 #else 7232 clk_req_support = false; 7233 #endif 7234 } else { 7235 clk_req_support = false; 7236 } 7237 7238 if (clk_req_support) { 7239 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2); 7240 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23; 7241 if (orig != data) 7242 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data); 7243 7244 orig = data = RREG32(THM_CLK_CNTL); 7245 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK); 7246 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1); 7247 if (orig != data) 7248 WREG32(THM_CLK_CNTL, data); 7249 7250 orig = data = RREG32(MISC_CLK_CNTL); 7251 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK); 7252 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1); 7253 if (orig != data) 7254 WREG32(MISC_CLK_CNTL, data); 7255 7256 orig = data = RREG32(CG_CLKPIN_CNTL); 7257 data &= ~BCLK_AS_XCLK; 7258 if (orig != data) 7259 WREG32(CG_CLKPIN_CNTL, data); 7260 7261 orig = data = RREG32(CG_CLKPIN_CNTL_2); 7262 data &= ~FORCE_BIF_REFCLK_EN; 7263 if (orig != data) 7264 WREG32(CG_CLKPIN_CNTL_2, data); 7265 7266 orig = data = RREG32(MPLL_BYPASSCLK_SEL); 7267 data &= ~MPLL_CLKOUT_SEL_MASK; 7268 data |= MPLL_CLKOUT_SEL(4); 7269 if (orig != data) 7270 WREG32(MPLL_BYPASSCLK_SEL, data); 7271 7272 orig = data = RREG32(SPLL_CNTL_MODE); 7273 data &= ~SPLL_REFCLK_SEL_MASK; 7274 if (orig != data) 7275 WREG32(SPLL_CNTL_MODE, data); 7276 } 7277 } 7278 } else { 7279 if (orig != data) 7280 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 7281 } 7282 7283 orig = data = RREG32_PCIE(PCIE_CNTL2); 7284 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN; 7285 if (orig != data) 7286 WREG32_PCIE(PCIE_CNTL2, data); 7287 7288 if (!disable_l0s) { 7289 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 7290 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) { 7291 data = RREG32_PCIE(PCIE_LC_STATUS1); 7292 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) { 7293 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 7294 data &= ~LC_L0S_INACTIVITY_MASK; 7295 if (orig != data) 7296 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 7297 } 7298 } 7299 } 7300 } 7301