1 /* i915_drv.h -- Private header for the I915 driver -*- linux-c -*- 2 */ 3 /* 4 * 5 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. 6 * All Rights Reserved. 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the 10 * "Software"), to deal in the Software without restriction, including 11 * without limitation the rights to use, copy, modify, merge, publish, 12 * distribute, sub license, and/or sell copies of the Software, and to 13 * permit persons to whom the Software is furnished to do so, subject to 14 * the following conditions: 15 * 16 * The above copyright notice and this permission notice (including the 17 * next paragraph) shall be included in all copies or substantial portions 18 * of the Software. 19 * 20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 23 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 24 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 25 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 26 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 27 * 28 */ 29 30 #ifndef _I915_DRV_H_ 31 #define _I915_DRV_H_ 32 33 #include <uapi/drm/i915_drm.h> 34 #include <uapi/drm/drm_fourcc.h> 35 36 #include <linux/io-mapping.h> 37 #include <linux/i2c.h> 38 #include <linux/i2c-algo-bit.h> 39 #include <linux/backlight.h> 40 #include <linux/hash.h> 41 #include <linux/intel-iommu.h> 42 #include <linux/kref.h> 43 #include <linux/pm_qos.h> 44 #include <linux/reservation.h> 45 #include <linux/shmem_fs.h> 46 47 #include <drm/drmP.h> 48 #include <drm/intel-gtt.h> 49 #include <drm/drm_legacy.h> /* for struct drm_dma_handle */ 50 #include <drm/drm_gem.h> 51 #include <drm/drm_auth.h> 52 #include <drm/drm_cache.h> 53 54 #include "i915_params.h" 55 #include "i915_reg.h" 56 #include "i915_utils.h" 57 58 #include "intel_uncore.h" 59 #include "intel_bios.h" 60 #include "intel_dpll_mgr.h" 61 #include "intel_uc.h" 62 #include "intel_lrc.h" 63 #include "intel_ringbuffer.h" 64 65 #include "i915_gem.h" 66 #include "i915_gem_context.h" 67 #include "i915_gem_fence_reg.h" 68 #include "i915_gem_object.h" 69 #include "i915_gem_gtt.h" 70 #include "i915_gem_render_state.h" 71 #include "i915_gem_request.h" 72 #include "i915_gem_timeline.h" 73 74 #include "i915_vma.h" 75 76 #include "intel_gvt.h" 77 78 /* General customization: 79 */ 80 81 #define DRIVER_NAME "i915" 82 #define DRIVER_DESC "Intel Graphics" 83 #define DRIVER_DATE "20171023" 84 #define DRIVER_TIMESTAMP 1508748913 85 86 /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and 87 * WARN_ON()) for hw state sanity checks to check for unexpected conditions 88 * which may not necessarily be a user visible problem. This will either 89 * WARN() or DRM_ERROR() depending on the verbose_checks moduleparam, to 90 * enable distros and users to tailor their preferred amount of i915 abrt 91 * spam. 92 */ 93 #define I915_STATE_WARN(condition, format...) ({ \ 94 int __ret_warn_on = !!(condition); \ 95 if (unlikely(__ret_warn_on)) \ 96 if (!WARN(i915_modparams.verbose_state_checks, format)) \ 97 DRM_ERROR(format); \ 98 unlikely(__ret_warn_on); \ 99 }) 100 101 #define I915_STATE_WARN_ON(x) \ 102 I915_STATE_WARN((x), "%s", "WARN_ON(" __stringify(x) ")") 103 104 bool __i915_inject_load_failure(const char *func, int line); 105 #define i915_inject_load_failure() \ 106 __i915_inject_load_failure(__func__, __LINE__) 107 108 typedef struct { 109 uint32_t val; 110 } uint_fixed_16_16_t; 111 112 #define FP_16_16_MAX ({ \ 113 uint_fixed_16_16_t fp; \ 114 fp.val = UINT_MAX; \ 115 fp; \ 116 }) 117 118 static inline bool is_fixed16_zero(uint_fixed_16_16_t val) 119 { 120 if (val.val == 0) 121 return true; 122 return false; 123 } 124 125 static inline uint_fixed_16_16_t u32_to_fixed16(uint32_t val) 126 { 127 uint_fixed_16_16_t fp; 128 129 WARN_ON(val > U16_MAX); 130 131 fp.val = val << 16; 132 return fp; 133 } 134 135 static inline uint32_t fixed16_to_u32_round_up(uint_fixed_16_16_t fp) 136 { 137 return DIV_ROUND_UP(fp.val, 1 << 16); 138 } 139 140 static inline uint32_t fixed16_to_u32(uint_fixed_16_16_t fp) 141 { 142 return fp.val >> 16; 143 } 144 145 static inline uint_fixed_16_16_t min_fixed16(uint_fixed_16_16_t min1, 146 uint_fixed_16_16_t min2) 147 { 148 uint_fixed_16_16_t min; 149 150 min.val = min(min1.val, min2.val); 151 return min; 152 } 153 154 static inline uint_fixed_16_16_t max_fixed16(uint_fixed_16_16_t max1, 155 uint_fixed_16_16_t max2) 156 { 157 uint_fixed_16_16_t max; 158 159 max.val = max(max1.val, max2.val); 160 return max; 161 } 162 163 static inline uint_fixed_16_16_t clamp_u64_to_fixed16(uint64_t val) 164 { 165 uint_fixed_16_16_t fp; 166 WARN_ON(val > U32_MAX); 167 fp.val = (uint32_t) val; 168 return fp; 169 } 170 171 static inline uint32_t div_round_up_fixed16(uint_fixed_16_16_t val, 172 uint_fixed_16_16_t d) 173 { 174 return DIV_ROUND_UP(val.val, d.val); 175 } 176 177 static inline uint32_t mul_round_up_u32_fixed16(uint32_t val, 178 uint_fixed_16_16_t mul) 179 { 180 uint64_t intermediate_val; 181 182 intermediate_val = (uint64_t) val * mul.val; 183 intermediate_val = DIV_ROUND_UP_ULL(intermediate_val, 1 << 16); 184 WARN_ON(intermediate_val > U32_MAX); 185 return (uint32_t) intermediate_val; 186 } 187 188 static inline uint_fixed_16_16_t mul_fixed16(uint_fixed_16_16_t val, 189 uint_fixed_16_16_t mul) 190 { 191 uint64_t intermediate_val; 192 193 intermediate_val = (uint64_t) val.val * mul.val; 194 intermediate_val = intermediate_val >> 16; 195 return clamp_u64_to_fixed16(intermediate_val); 196 } 197 198 static inline uint_fixed_16_16_t div_fixed16(uint32_t val, uint32_t d) 199 { 200 uint64_t interm_val; 201 202 interm_val = (uint64_t)val << 16; 203 interm_val = DIV_ROUND_UP_ULL(interm_val, d); 204 return clamp_u64_to_fixed16(interm_val); 205 } 206 207 static inline uint32_t div_round_up_u32_fixed16(uint32_t val, 208 uint_fixed_16_16_t d) 209 { 210 uint64_t interm_val; 211 212 interm_val = (uint64_t)val << 16; 213 interm_val = DIV_ROUND_UP_ULL(interm_val, d.val); 214 WARN_ON(interm_val > U32_MAX); 215 return (uint32_t) interm_val; 216 } 217 218 static inline uint_fixed_16_16_t mul_u32_fixed16(uint32_t val, 219 uint_fixed_16_16_t mul) 220 { 221 uint64_t intermediate_val; 222 223 intermediate_val = (uint64_t) val * mul.val; 224 return clamp_u64_to_fixed16(intermediate_val); 225 } 226 227 static inline uint_fixed_16_16_t add_fixed16(uint_fixed_16_16_t add1, 228 uint_fixed_16_16_t add2) 229 { 230 uint64_t interm_sum; 231 232 interm_sum = (uint64_t) add1.val + add2.val; 233 return clamp_u64_to_fixed16(interm_sum); 234 } 235 236 static inline uint_fixed_16_16_t add_fixed16_u32(uint_fixed_16_16_t add1, 237 uint32_t add2) 238 { 239 uint64_t interm_sum; 240 uint_fixed_16_16_t interm_add2 = u32_to_fixed16(add2); 241 242 interm_sum = (uint64_t) add1.val + interm_add2.val; 243 return clamp_u64_to_fixed16(interm_sum); 244 } 245 246 static inline const char *yesno(bool v) 247 { 248 return v ? "yes" : "no"; 249 } 250 251 static inline const char *onoff(bool v) 252 { 253 return v ? "on" : "off"; 254 } 255 256 static inline const char *enableddisabled(bool v) 257 { 258 return v ? "enabled" : "disabled"; 259 } 260 261 enum i915_pipe { 262 INVALID_PIPE = -1, 263 PIPE_A = 0, 264 PIPE_B, 265 PIPE_C, 266 _PIPE_EDP, 267 I915_MAX_PIPES = _PIPE_EDP 268 }; 269 #define pipe_name(p) ((p) + 'A') 270 271 enum transcoder { 272 TRANSCODER_A = 0, 273 TRANSCODER_B, 274 TRANSCODER_C, 275 TRANSCODER_EDP, 276 TRANSCODER_DSI_A, 277 TRANSCODER_DSI_C, 278 I915_MAX_TRANSCODERS 279 }; 280 281 static inline const char *transcoder_name(enum transcoder transcoder) 282 { 283 switch (transcoder) { 284 case TRANSCODER_A: 285 return "A"; 286 case TRANSCODER_B: 287 return "B"; 288 case TRANSCODER_C: 289 return "C"; 290 case TRANSCODER_EDP: 291 return "EDP"; 292 case TRANSCODER_DSI_A: 293 return "DSI A"; 294 case TRANSCODER_DSI_C: 295 return "DSI C"; 296 default: 297 return "<invalid>"; 298 } 299 } 300 301 static inline bool transcoder_is_dsi(enum transcoder transcoder) 302 { 303 return transcoder == TRANSCODER_DSI_A || transcoder == TRANSCODER_DSI_C; 304 } 305 306 /* 307 * Global legacy plane identifier. Valid only for primary/sprite 308 * planes on pre-g4x, and only for primary planes on g4x+. 309 */ 310 enum plane { 311 PLANE_A, 312 PLANE_B, 313 PLANE_C, 314 }; 315 #define plane_name(p) ((p) + 'A') 316 317 #define sprite_name(p, s) ((p) * INTEL_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A') 318 319 /* 320 * Per-pipe plane identifier. 321 * I915_MAX_PLANES in the enum below is the maximum (across all platforms) 322 * number of planes per CRTC. Not all platforms really have this many planes, 323 * which means some arrays of size I915_MAX_PLANES may have unused entries 324 * between the topmost sprite plane and the cursor plane. 325 * 326 * This is expected to be passed to various register macros 327 * (eg. PLANE_CTL(), PS_PLANE_SEL(), etc.) so adjust with care. 328 */ 329 enum plane_id { 330 PLANE_PRIMARY, 331 PLANE_SPRITE0, 332 PLANE_SPRITE1, 333 PLANE_SPRITE2, 334 PLANE_CURSOR, 335 I915_MAX_PLANES, 336 }; 337 338 #define for_each_plane_id_on_crtc(__crtc, __p) \ 339 for ((__p) = PLANE_PRIMARY; (__p) < I915_MAX_PLANES; (__p)++) \ 340 for_each_if ((__crtc)->plane_ids_mask & BIT(__p)) 341 342 enum port { 343 PORT_NONE = -1, 344 PORT_A = 0, 345 PORT_B, 346 PORT_C, 347 PORT_D, 348 PORT_E, 349 I915_MAX_PORTS 350 }; 351 #define port_name(p) ((p) + 'A') 352 353 #define I915_NUM_PHYS_VLV 2 354 355 enum dpio_channel { 356 DPIO_CH0, 357 DPIO_CH1 358 }; 359 360 enum dpio_phy { 361 DPIO_PHY0, 362 DPIO_PHY1, 363 DPIO_PHY2, 364 }; 365 366 enum intel_display_power_domain { 367 POWER_DOMAIN_PIPE_A, 368 POWER_DOMAIN_PIPE_B, 369 POWER_DOMAIN_PIPE_C, 370 POWER_DOMAIN_PIPE_A_PANEL_FITTER, 371 POWER_DOMAIN_PIPE_B_PANEL_FITTER, 372 POWER_DOMAIN_PIPE_C_PANEL_FITTER, 373 POWER_DOMAIN_TRANSCODER_A, 374 POWER_DOMAIN_TRANSCODER_B, 375 POWER_DOMAIN_TRANSCODER_C, 376 POWER_DOMAIN_TRANSCODER_EDP, 377 POWER_DOMAIN_TRANSCODER_DSI_A, 378 POWER_DOMAIN_TRANSCODER_DSI_C, 379 POWER_DOMAIN_PORT_DDI_A_LANES, 380 POWER_DOMAIN_PORT_DDI_B_LANES, 381 POWER_DOMAIN_PORT_DDI_C_LANES, 382 POWER_DOMAIN_PORT_DDI_D_LANES, 383 POWER_DOMAIN_PORT_DDI_E_LANES, 384 POWER_DOMAIN_PORT_DDI_A_IO, 385 POWER_DOMAIN_PORT_DDI_B_IO, 386 POWER_DOMAIN_PORT_DDI_C_IO, 387 POWER_DOMAIN_PORT_DDI_D_IO, 388 POWER_DOMAIN_PORT_DDI_E_IO, 389 POWER_DOMAIN_PORT_DSI, 390 POWER_DOMAIN_PORT_CRT, 391 POWER_DOMAIN_PORT_OTHER, 392 POWER_DOMAIN_VGA, 393 POWER_DOMAIN_AUDIO, 394 POWER_DOMAIN_PLLS, 395 POWER_DOMAIN_AUX_A, 396 POWER_DOMAIN_AUX_B, 397 POWER_DOMAIN_AUX_C, 398 POWER_DOMAIN_AUX_D, 399 POWER_DOMAIN_GMBUS, 400 POWER_DOMAIN_MODESET, 401 POWER_DOMAIN_INIT, 402 403 POWER_DOMAIN_NUM, 404 }; 405 406 #define POWER_DOMAIN_PIPE(pipe) ((pipe) + POWER_DOMAIN_PIPE_A) 407 #define POWER_DOMAIN_PIPE_PANEL_FITTER(pipe) \ 408 ((pipe) + POWER_DOMAIN_PIPE_A_PANEL_FITTER) 409 #define POWER_DOMAIN_TRANSCODER(tran) \ 410 ((tran) == TRANSCODER_EDP ? POWER_DOMAIN_TRANSCODER_EDP : \ 411 (tran) + POWER_DOMAIN_TRANSCODER_A) 412 413 enum hpd_pin { 414 HPD_NONE = 0, 415 HPD_TV = HPD_NONE, /* TV is known to be unreliable */ 416 HPD_CRT, 417 HPD_SDVO_B, 418 HPD_SDVO_C, 419 HPD_PORT_A, 420 HPD_PORT_B, 421 HPD_PORT_C, 422 HPD_PORT_D, 423 HPD_PORT_E, 424 HPD_NUM_PINS 425 }; 426 427 #define for_each_hpd_pin(__pin) \ 428 for ((__pin) = (HPD_NONE + 1); (__pin) < HPD_NUM_PINS; (__pin)++) 429 430 #define HPD_STORM_DEFAULT_THRESHOLD 5 431 432 struct i915_hotplug { 433 struct work_struct hotplug_work; 434 435 struct { 436 unsigned long last_jiffies; 437 int count; 438 enum { 439 HPD_ENABLED = 0, 440 HPD_DISABLED = 1, 441 HPD_MARK_DISABLED = 2 442 } state; 443 } stats[HPD_NUM_PINS]; 444 u32 event_bits; 445 struct delayed_work reenable_work; 446 447 struct intel_digital_port *irq_port[I915_MAX_PORTS]; 448 u32 long_port_mask; 449 u32 short_port_mask; 450 struct work_struct dig_port_work; 451 452 struct work_struct poll_init_work; 453 bool poll_enabled; 454 455 unsigned int hpd_storm_threshold; 456 457 /* 458 * if we get a HPD irq from DP and a HPD irq from non-DP 459 * the non-DP HPD could block the workqueue on a mode config 460 * mutex getting, that userspace may have taken. However 461 * userspace is waiting on the DP workqueue to run which is 462 * blocked behind the non-DP one. 463 */ 464 struct workqueue_struct *dp_wq; 465 }; 466 467 #define I915_GEM_GPU_DOMAINS \ 468 (I915_GEM_DOMAIN_RENDER | \ 469 I915_GEM_DOMAIN_SAMPLER | \ 470 I915_GEM_DOMAIN_COMMAND | \ 471 I915_GEM_DOMAIN_INSTRUCTION | \ 472 I915_GEM_DOMAIN_VERTEX) 473 474 #define for_each_pipe(__dev_priv, __p) \ 475 for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) 476 #define for_each_pipe_masked(__dev_priv, __p, __mask) \ 477 for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) \ 478 for_each_if ((__mask) & (1 << (__p))) 479 #define for_each_universal_plane(__dev_priv, __pipe, __p) \ 480 for ((__p) = 0; \ 481 (__p) < INTEL_INFO(__dev_priv)->num_sprites[(__pipe)] + 1; \ 482 (__p)++) 483 #define for_each_sprite(__dev_priv, __p, __s) \ 484 for ((__s) = 0; \ 485 (__s) < INTEL_INFO(__dev_priv)->num_sprites[(__p)]; \ 486 (__s)++) 487 488 #define for_each_port_masked(__port, __ports_mask) \ 489 for ((__port) = PORT_A; (__port) < I915_MAX_PORTS; (__port)++) \ 490 for_each_if ((__ports_mask) & (1 << (__port))) 491 492 #define for_each_crtc(dev, crtc) \ 493 list_for_each_entry(crtc, &(dev)->mode_config.crtc_list, head) 494 495 #define for_each_intel_plane(dev, intel_plane) \ 496 list_for_each_entry(intel_plane, \ 497 &(dev)->mode_config.plane_list, \ 498 base.head) 499 500 #define for_each_intel_plane_mask(dev, intel_plane, plane_mask) \ 501 list_for_each_entry(intel_plane, \ 502 &(dev)->mode_config.plane_list, \ 503 base.head) \ 504 for_each_if ((plane_mask) & \ 505 (1 << drm_plane_index(&intel_plane->base))) 506 507 #define for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) \ 508 list_for_each_entry(intel_plane, \ 509 &(dev)->mode_config.plane_list, \ 510 base.head) \ 511 for_each_if ((intel_plane)->pipe == (intel_crtc)->pipe) 512 513 #define for_each_intel_crtc(dev, intel_crtc) \ 514 list_for_each_entry(intel_crtc, \ 515 &(dev)->mode_config.crtc_list, \ 516 base.head) 517 518 #define for_each_intel_crtc_mask(dev, intel_crtc, crtc_mask) \ 519 list_for_each_entry(intel_crtc, \ 520 &(dev)->mode_config.crtc_list, \ 521 base.head) \ 522 for_each_if ((crtc_mask) & (1 << drm_crtc_index(&intel_crtc->base))) 523 524 #define for_each_intel_encoder(dev, intel_encoder) \ 525 list_for_each_entry(intel_encoder, \ 526 &(dev)->mode_config.encoder_list, \ 527 base.head) 528 529 #define for_each_intel_connector_iter(intel_connector, iter) \ 530 while ((intel_connector = to_intel_connector(drm_connector_list_iter_next(iter)))) 531 532 #define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \ 533 list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \ 534 for_each_if ((intel_encoder)->base.crtc == (__crtc)) 535 536 #define for_each_connector_on_encoder(dev, __encoder, intel_connector) \ 537 list_for_each_entry((intel_connector), &(dev)->mode_config.connector_list, base.head) \ 538 for_each_if ((intel_connector)->base.encoder == (__encoder)) 539 540 #define for_each_power_domain(domain, mask) \ 541 for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++) \ 542 for_each_if (BIT_ULL(domain) & (mask)) 543 544 #define for_each_power_well(__dev_priv, __power_well) \ 545 for ((__power_well) = (__dev_priv)->power_domains.power_wells; \ 546 (__power_well) - (__dev_priv)->power_domains.power_wells < \ 547 (__dev_priv)->power_domains.power_well_count; \ 548 (__power_well)++) 549 550 #define for_each_power_well_rev(__dev_priv, __power_well) \ 551 for ((__power_well) = (__dev_priv)->power_domains.power_wells + \ 552 (__dev_priv)->power_domains.power_well_count - 1; \ 553 (__power_well) - (__dev_priv)->power_domains.power_wells >= 0; \ 554 (__power_well)--) 555 556 #define for_each_power_domain_well(__dev_priv, __power_well, __domain_mask) \ 557 for_each_power_well(__dev_priv, __power_well) \ 558 for_each_if ((__power_well)->domains & (__domain_mask)) 559 560 #define for_each_power_domain_well_rev(__dev_priv, __power_well, __domain_mask) \ 561 for_each_power_well_rev(__dev_priv, __power_well) \ 562 for_each_if ((__power_well)->domains & (__domain_mask)) 563 564 #define for_each_intel_plane_in_state(__state, plane, plane_state, __i) \ 565 for ((__i) = 0; \ 566 (__i) < (__state)->base.dev->mode_config.num_total_plane && \ 567 ((plane) = to_intel_plane((__state)->base.planes[__i].ptr), \ 568 (plane_state) = to_intel_plane_state((__state)->base.planes[__i].state), 1); \ 569 (__i)++) \ 570 for_each_if (plane_state) 571 572 #define for_each_new_intel_crtc_in_state(__state, crtc, new_crtc_state, __i) \ 573 for ((__i) = 0; \ 574 (__i) < (__state)->base.dev->mode_config.num_crtc && \ 575 ((crtc) = to_intel_crtc((__state)->base.crtcs[__i].ptr), \ 576 (new_crtc_state) = to_intel_crtc_state((__state)->base.crtcs[__i].new_state), 1); \ 577 (__i)++) \ 578 for_each_if (crtc) 579 580 581 #define for_each_oldnew_intel_plane_in_state(__state, plane, old_plane_state, new_plane_state, __i) \ 582 for ((__i) = 0; \ 583 (__i) < (__state)->base.dev->mode_config.num_total_plane && \ 584 ((plane) = to_intel_plane((__state)->base.planes[__i].ptr), \ 585 (old_plane_state) = to_intel_plane_state((__state)->base.planes[__i].old_state), \ 586 (new_plane_state) = to_intel_plane_state((__state)->base.planes[__i].new_state), 1); \ 587 (__i)++) \ 588 for_each_if (plane) 589 590 struct drm_i915_private; 591 struct i915_mm_struct; 592 struct i915_mmu_object; 593 594 struct drm_i915_file_private { 595 struct drm_i915_private *dev_priv; 596 struct drm_file *file; 597 598 struct { 599 spinlock_t lock; 600 struct list_head request_list; 601 /* 20ms is a fairly arbitrary limit (greater than the average frame time) 602 * chosen to prevent the CPU getting more than a frame ahead of the GPU 603 * (when using lax throttling for the frontbuffer). We also use it to 604 * offer free GPU waitboosts for severely congested workloads. 605 */ 606 #define DRM_I915_THROTTLE_JIFFIES msecs_to_jiffies(20) 607 } mm; 608 struct idr context_idr; 609 610 struct intel_rps_client { 611 atomic_t boosts; 612 } rps_client; 613 614 unsigned int bsd_engine; 615 616 /* Client can have a maximum of 3 contexts banned before 617 * it is denied of creating new contexts. As one context 618 * ban needs 4 consecutive hangs, and more if there is 619 * progress in between, this is a last resort stop gap measure 620 * to limit the badly behaving clients access to gpu. 621 */ 622 #define I915_MAX_CLIENT_CONTEXT_BANS 3 623 atomic_t context_bans; 624 }; 625 626 /* Used by dp and fdi links */ 627 struct intel_link_m_n { 628 uint32_t tu; 629 uint32_t gmch_m; 630 uint32_t gmch_n; 631 uint32_t link_m; 632 uint32_t link_n; 633 }; 634 635 void intel_link_compute_m_n(int bpp, int nlanes, 636 int pixel_clock, int link_clock, 637 struct intel_link_m_n *m_n, 638 bool reduce_m_n); 639 640 /* Interface history: 641 * 642 * 1.1: Original. 643 * 1.2: Add Power Management 644 * 1.3: Add vblank support 645 * 1.4: Fix cmdbuffer path, add heap destroy 646 * 1.5: Add vblank pipe configuration 647 * 1.6: - New ioctl for scheduling buffer swaps on vertical blank 648 * - Support vertical blank on secondary display pipe 649 */ 650 #define DRIVER_MAJOR 1 651 #define DRIVER_MINOR 6 652 #define DRIVER_PATCHLEVEL 0 653 654 struct opregion_header; 655 struct opregion_acpi; 656 struct opregion_swsci; 657 struct opregion_asle; 658 659 struct intel_opregion { 660 struct opregion_header *header; 661 struct opregion_acpi *acpi; 662 struct opregion_swsci *swsci; 663 u32 swsci_gbda_sub_functions; 664 u32 swsci_sbcb_sub_functions; 665 struct opregion_asle *asle; 666 void *rvda; 667 void *vbt_firmware; 668 const void *vbt; 669 u32 vbt_size; 670 u32 *lid_state; 671 struct work_struct asle_work; 672 }; 673 #define OPREGION_SIZE (8*1024) 674 675 struct intel_overlay; 676 struct intel_overlay_error_state; 677 678 struct sdvo_device_mapping { 679 u8 initialized; 680 u8 dvo_port; 681 u8 slave_addr; 682 u8 dvo_wiring; 683 u8 i2c_pin; 684 u8 ddc_pin; 685 }; 686 687 struct intel_connector; 688 struct intel_encoder; 689 struct intel_atomic_state; 690 struct intel_crtc_state; 691 struct intel_initial_plane_config; 692 struct intel_crtc; 693 struct intel_limit; 694 struct dpll; 695 struct intel_cdclk_state; 696 697 struct drm_i915_display_funcs { 698 void (*get_cdclk)(struct drm_i915_private *dev_priv, 699 struct intel_cdclk_state *cdclk_state); 700 void (*set_cdclk)(struct drm_i915_private *dev_priv, 701 const struct intel_cdclk_state *cdclk_state); 702 int (*get_fifo_size)(struct drm_i915_private *dev_priv, int plane); 703 int (*compute_pipe_wm)(struct intel_crtc_state *cstate); 704 int (*compute_intermediate_wm)(struct drm_device *dev, 705 struct intel_crtc *intel_crtc, 706 struct intel_crtc_state *newstate); 707 void (*initial_watermarks)(struct intel_atomic_state *state, 708 struct intel_crtc_state *cstate); 709 void (*atomic_update_watermarks)(struct intel_atomic_state *state, 710 struct intel_crtc_state *cstate); 711 void (*optimize_watermarks)(struct intel_atomic_state *state, 712 struct intel_crtc_state *cstate); 713 int (*compute_global_watermarks)(struct drm_atomic_state *state); 714 void (*update_wm)(struct intel_crtc *crtc); 715 int (*modeset_calc_cdclk)(struct drm_atomic_state *state); 716 /* Returns the active state of the crtc, and if the crtc is active, 717 * fills out the pipe-config with the hw state. */ 718 bool (*get_pipe_config)(struct intel_crtc *, 719 struct intel_crtc_state *); 720 void (*get_initial_plane_config)(struct intel_crtc *, 721 struct intel_initial_plane_config *); 722 int (*crtc_compute_clock)(struct intel_crtc *crtc, 723 struct intel_crtc_state *crtc_state); 724 void (*crtc_enable)(struct intel_crtc_state *pipe_config, 725 struct drm_atomic_state *old_state); 726 void (*crtc_disable)(struct intel_crtc_state *old_crtc_state, 727 struct drm_atomic_state *old_state); 728 void (*update_crtcs)(struct drm_atomic_state *state); 729 void (*audio_codec_enable)(struct drm_connector *connector, 730 struct intel_encoder *encoder, 731 const struct drm_display_mode *adjusted_mode); 732 void (*audio_codec_disable)(struct intel_encoder *encoder); 733 void (*fdi_link_train)(struct intel_crtc *crtc, 734 const struct intel_crtc_state *crtc_state); 735 void (*init_clock_gating)(struct drm_i915_private *dev_priv); 736 void (*hpd_irq_setup)(struct drm_i915_private *dev_priv); 737 /* clock updates for mode set */ 738 /* cursor updates */ 739 /* render clock increase/decrease */ 740 /* display clock increase/decrease */ 741 /* pll clock increase/decrease */ 742 743 void (*load_csc_matrix)(struct drm_crtc_state *crtc_state); 744 void (*load_luts)(struct drm_crtc_state *crtc_state); 745 }; 746 747 #define CSR_VERSION(major, minor) ((major) << 16 | (minor)) 748 #define CSR_VERSION_MAJOR(version) ((version) >> 16) 749 #define CSR_VERSION_MINOR(version) ((version) & 0xffff) 750 751 struct intel_csr { 752 struct work_struct work; 753 const char *fw_path; 754 uint32_t *dmc_payload; 755 uint32_t dmc_fw_size; 756 uint32_t version; 757 uint32_t mmio_count; 758 i915_reg_t mmioaddr[8]; 759 uint32_t mmiodata[8]; 760 uint32_t dc_state; 761 uint32_t allowed_dc_mask; 762 }; 763 764 #define DEV_INFO_FOR_EACH_FLAG(func) \ 765 func(is_mobile); \ 766 func(is_lp); \ 767 func(is_alpha_support); \ 768 /* Keep has_* in alphabetical order */ \ 769 func(has_64bit_reloc); \ 770 func(has_aliasing_ppgtt); \ 771 func(has_csr); \ 772 func(has_ddi); \ 773 func(has_dp_mst); \ 774 func(has_reset_engine); \ 775 func(has_fbc); \ 776 func(has_fpga_dbg); \ 777 func(has_full_ppgtt); \ 778 func(has_full_48bit_ppgtt); \ 779 func(has_gmch_display); \ 780 func(has_guc); \ 781 func(has_guc_ct); \ 782 func(has_hotplug); \ 783 func(has_l3_dpf); \ 784 func(has_llc); \ 785 func(has_logical_ring_contexts); \ 786 func(has_logical_ring_preemption); \ 787 func(has_overlay); \ 788 func(has_pooled_eu); \ 789 func(has_psr); \ 790 func(has_rc6); \ 791 func(has_rc6p); \ 792 func(has_resource_streamer); \ 793 func(has_runtime_pm); \ 794 func(has_snoop); \ 795 func(unfenced_needs_alignment); \ 796 func(cursor_needs_physical); \ 797 func(hws_needs_physical); \ 798 func(overlay_needs_physical); \ 799 func(supports_tv); \ 800 func(has_ipc); 801 802 struct sseu_dev_info { 803 u8 slice_mask; 804 u8 subslice_mask; 805 u8 eu_total; 806 u8 eu_per_subslice; 807 u8 min_eu_in_pool; 808 /* For each slice, which subslice(s) has(have) 7 EUs (bitfield)? */ 809 u8 subslice_7eu[3]; 810 u8 has_slice_pg:1; 811 u8 has_subslice_pg:1; 812 u8 has_eu_pg:1; 813 }; 814 815 static inline unsigned int sseu_subslice_total(const struct sseu_dev_info *sseu) 816 { 817 return hweight8(sseu->slice_mask) * hweight8(sseu->subslice_mask); 818 } 819 820 /* Keep in gen based order, and chronological order within a gen */ 821 enum intel_platform { 822 INTEL_PLATFORM_UNINITIALIZED = 0, 823 INTEL_I830, 824 INTEL_I845G, 825 INTEL_I85X, 826 INTEL_I865G, 827 INTEL_I915G, 828 INTEL_I915GM, 829 INTEL_I945G, 830 INTEL_I945GM, 831 INTEL_G33, 832 INTEL_PINEVIEW, 833 INTEL_I965G, 834 INTEL_I965GM, 835 INTEL_G45, 836 INTEL_GM45, 837 INTEL_IRONLAKE, 838 INTEL_SANDYBRIDGE, 839 INTEL_IVYBRIDGE, 840 INTEL_VALLEYVIEW, 841 INTEL_HASWELL, 842 INTEL_BROADWELL, 843 INTEL_CHERRYVIEW, 844 INTEL_SKYLAKE, 845 INTEL_BROXTON, 846 INTEL_KABYLAKE, 847 INTEL_GEMINILAKE, 848 INTEL_COFFEELAKE, 849 INTEL_CANNONLAKE, 850 INTEL_MAX_PLATFORMS 851 }; 852 853 struct intel_device_info { 854 u16 device_id; 855 u16 gen_mask; 856 857 u8 gen; 858 u8 gt; /* GT number, 0 if undefined */ 859 u8 num_rings; 860 u8 ring_mask; /* Rings supported by the HW */ 861 862 enum intel_platform platform; 863 u32 platform_mask; 864 865 u32 display_mmio_offset; 866 867 u8 num_pipes; 868 u8 num_sprites[I915_MAX_PIPES]; 869 u8 num_scalers[I915_MAX_PIPES]; 870 871 unsigned int page_sizes; /* page sizes supported by the HW */ 872 873 #define DEFINE_FLAG(name) u8 name:1 874 DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG); 875 #undef DEFINE_FLAG 876 u16 ddb_size; /* in blocks */ 877 878 /* Register offsets for the various display pipes and transcoders */ 879 int pipe_offsets[I915_MAX_TRANSCODERS]; 880 int trans_offsets[I915_MAX_TRANSCODERS]; 881 int palette_offsets[I915_MAX_PIPES]; 882 int cursor_offsets[I915_MAX_PIPES]; 883 884 /* Slice/subslice/EU info */ 885 struct sseu_dev_info sseu; 886 887 struct color_luts { 888 u16 degamma_lut_size; 889 u16 gamma_lut_size; 890 } color; 891 }; 892 893 struct intel_display_error_state; 894 895 struct i915_gpu_state { 896 struct kref ref; 897 struct timeval time; 898 struct timeval boottime; 899 struct timeval uptime; 900 901 struct drm_i915_private *i915; 902 903 char error_msg[128]; 904 bool simulated; 905 bool awake; 906 bool wakelock; 907 bool suspended; 908 int iommu; 909 u32 reset_count; 910 u32 suspend_count; 911 struct intel_device_info device_info; 912 struct i915_params params; 913 914 /* Generic register state */ 915 u32 eir; 916 u32 pgtbl_er; 917 u32 ier; 918 u32 gtier[4], ngtier; 919 u32 ccid; 920 u32 derrmr; 921 u32 forcewake; 922 u32 error; /* gen6+ */ 923 u32 err_int; /* gen7 */ 924 u32 fault_data0; /* gen8, gen9 */ 925 u32 fault_data1; /* gen8, gen9 */ 926 u32 done_reg; 927 u32 gac_eco; 928 u32 gam_ecochk; 929 u32 gab_ctl; 930 u32 gfx_mode; 931 932 u32 nfence; 933 u64 fence[I915_MAX_NUM_FENCES]; 934 struct intel_overlay_error_state *overlay; 935 struct intel_display_error_state *display; 936 struct drm_i915_error_object *semaphore; 937 struct drm_i915_error_object *guc_log; 938 939 struct drm_i915_error_engine { 940 int engine_id; 941 /* Software tracked state */ 942 bool waiting; 943 int num_waiters; 944 unsigned long hangcheck_timestamp; 945 bool hangcheck_stalled; 946 enum intel_engine_hangcheck_action hangcheck_action; 947 struct i915_address_space *vm; 948 int num_requests; 949 u32 reset_count; 950 951 /* position of active request inside the ring */ 952 u32 rq_head, rq_post, rq_tail; 953 954 /* our own tracking of ring head and tail */ 955 u32 cpu_ring_head; 956 u32 cpu_ring_tail; 957 958 u32 last_seqno; 959 960 /* Register state */ 961 u32 start; 962 u32 tail; 963 u32 head; 964 u32 ctl; 965 u32 mode; 966 u32 hws; 967 u32 ipeir; 968 u32 ipehr; 969 u32 bbstate; 970 u32 instpm; 971 u32 instps; 972 u32 seqno; 973 u64 bbaddr; 974 u64 acthd; 975 u32 fault_reg; 976 u64 faddr; 977 u32 rc_psmi; /* sleep state */ 978 u32 semaphore_mboxes[I915_NUM_ENGINES - 1]; 979 struct intel_instdone instdone; 980 981 struct drm_i915_error_context { 982 char comm[TASK_COMM_LEN]; 983 pid_t pid; 984 u32 handle; 985 u32 hw_id; 986 int priority; 987 int ban_score; 988 int active; 989 int guilty; 990 } context; 991 992 struct drm_i915_error_object { 993 u64 gtt_offset; 994 u64 gtt_size; 995 int page_count; 996 int unused; 997 u32 *pages[0]; 998 } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page; 999 1000 struct drm_i915_error_object **user_bo; 1001 long user_bo_count; 1002 1003 struct drm_i915_error_object *wa_ctx; 1004 1005 struct drm_i915_error_request { 1006 long jiffies; 1007 pid_t pid; 1008 u32 context; 1009 int priority; 1010 int ban_score; 1011 u32 seqno; 1012 u32 head; 1013 u32 tail; 1014 } *requests, execlist[EXECLIST_MAX_PORTS]; 1015 unsigned int num_ports; 1016 1017 struct drm_i915_error_waiter { 1018 char comm[TASK_COMM_LEN]; 1019 pid_t pid; 1020 u32 seqno; 1021 } *waiters; 1022 1023 struct { 1024 u32 gfx_mode; 1025 union { 1026 u64 pdp[4]; 1027 u32 pp_dir_base; 1028 }; 1029 } vm_info; 1030 } engine[I915_NUM_ENGINES]; 1031 1032 struct drm_i915_error_buffer { 1033 u32 size; 1034 u32 name; 1035 u32 rseqno[I915_NUM_ENGINES], wseqno; 1036 u64 gtt_offset; 1037 u32 read_domains; 1038 u32 write_domain; 1039 s32 fence_reg:I915_MAX_NUM_FENCE_BITS; 1040 u32 tiling:2; 1041 u32 dirty:1; 1042 u32 purgeable:1; 1043 u32 userptr:1; 1044 s32 engine:4; 1045 u32 cache_level:3; 1046 } *active_bo[I915_NUM_ENGINES], *pinned_bo; 1047 u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count; 1048 struct i915_address_space *active_vm[I915_NUM_ENGINES]; 1049 }; 1050 1051 enum i915_cache_level { 1052 I915_CACHE_NONE = 0, 1053 I915_CACHE_LLC, /* also used for snoopable memory on non-LLC */ 1054 I915_CACHE_L3_LLC, /* gen7+, L3 sits between the domain specifc 1055 caches, eg sampler/render caches, and the 1056 large Last-Level-Cache. LLC is coherent with 1057 the CPU, but L3 is only visible to the GPU. */ 1058 I915_CACHE_WT, /* hsw:gt3e WriteThrough for scanouts */ 1059 }; 1060 1061 #define I915_COLOR_UNEVICTABLE (-1) /* a non-vma sharing the address space */ 1062 1063 enum fb_op_origin { 1064 ORIGIN_GTT, 1065 ORIGIN_CPU, 1066 ORIGIN_CS, 1067 ORIGIN_FLIP, 1068 ORIGIN_DIRTYFB, 1069 }; 1070 1071 struct intel_fbc { 1072 /* This is always the inner lock when overlapping with struct_mutex and 1073 * it's the outer lock when overlapping with stolen_lock. */ 1074 struct lock lock; 1075 unsigned threshold; 1076 unsigned int possible_framebuffer_bits; 1077 unsigned int busy_bits; 1078 unsigned int visible_pipes_mask; 1079 struct intel_crtc *crtc; 1080 1081 struct drm_mm_node compressed_fb; 1082 struct drm_mm_node *compressed_llb; 1083 1084 bool false_color; 1085 1086 bool enabled; 1087 bool active; 1088 1089 bool underrun_detected; 1090 struct work_struct underrun_work; 1091 1092 /* 1093 * Due to the atomic rules we can't access some structures without the 1094 * appropriate locking, so we cache information here in order to avoid 1095 * these problems. 1096 */ 1097 struct intel_fbc_state_cache { 1098 struct i915_vma *vma; 1099 1100 struct { 1101 unsigned int mode_flags; 1102 uint32_t hsw_bdw_pixel_rate; 1103 } crtc; 1104 1105 struct { 1106 unsigned int rotation; 1107 int src_w; 1108 int src_h; 1109 bool visible; 1110 /* 1111 * Display surface base address adjustement for 1112 * pageflips. Note that on gen4+ this only adjusts up 1113 * to a tile, offsets within a tile are handled in 1114 * the hw itself (with the TILEOFF register). 1115 */ 1116 int adjusted_x; 1117 int adjusted_y; 1118 1119 int y; 1120 } plane; 1121 1122 struct { 1123 const struct drm_format_info *format; 1124 unsigned int stride; 1125 } fb; 1126 } state_cache; 1127 1128 /* 1129 * This structure contains everything that's relevant to program the 1130 * hardware registers. When we want to figure out if we need to disable 1131 * and re-enable FBC for a new configuration we just check if there's 1132 * something different in the struct. The genx_fbc_activate functions 1133 * are supposed to read from it in order to program the registers. 1134 */ 1135 struct intel_fbc_reg_params { 1136 struct i915_vma *vma; 1137 1138 struct { 1139 enum i915_pipe pipe; 1140 enum plane plane; 1141 unsigned int fence_y_offset; 1142 } crtc; 1143 1144 struct { 1145 const struct drm_format_info *format; 1146 unsigned int stride; 1147 } fb; 1148 1149 int cfb_size; 1150 unsigned int gen9_wa_cfb_stride; 1151 } params; 1152 1153 struct intel_fbc_work { 1154 bool scheduled; 1155 u32 scheduled_vblank; 1156 struct work_struct work; 1157 } work; 1158 1159 const char *no_fbc_reason; 1160 }; 1161 1162 /* 1163 * HIGH_RR is the highest eDP panel refresh rate read from EDID 1164 * LOW_RR is the lowest eDP panel refresh rate found from EDID 1165 * parsing for same resolution. 1166 */ 1167 enum drrs_refresh_rate_type { 1168 DRRS_HIGH_RR, 1169 DRRS_LOW_RR, 1170 DRRS_MAX_RR, /* RR count */ 1171 }; 1172 1173 enum drrs_support_type { 1174 DRRS_NOT_SUPPORTED = 0, 1175 STATIC_DRRS_SUPPORT = 1, 1176 SEAMLESS_DRRS_SUPPORT = 2 1177 }; 1178 1179 struct intel_dp; 1180 struct i915_drrs { 1181 struct lock mutex; 1182 struct delayed_work work; 1183 struct intel_dp *dp; 1184 unsigned busy_frontbuffer_bits; 1185 enum drrs_refresh_rate_type refresh_rate_type; 1186 enum drrs_support_type type; 1187 }; 1188 1189 struct i915_psr { 1190 struct lock lock; 1191 bool sink_support; 1192 bool source_ok; 1193 struct intel_dp *enabled; 1194 bool active; 1195 struct delayed_work work; 1196 unsigned busy_frontbuffer_bits; 1197 bool psr2_support; 1198 bool aux_frame_sync; 1199 bool link_standby; 1200 bool y_cord_support; 1201 bool colorimetry_support; 1202 bool alpm; 1203 1204 void (*enable_source)(struct intel_dp *, 1205 const struct intel_crtc_state *); 1206 void (*disable_source)(struct intel_dp *, 1207 const struct intel_crtc_state *); 1208 void (*enable_sink)(struct intel_dp *); 1209 void (*activate)(struct intel_dp *); 1210 void (*setup_vsc)(struct intel_dp *, const struct intel_crtc_state *); 1211 }; 1212 1213 enum intel_pch { 1214 PCH_NONE = 0, /* No PCH present */ 1215 PCH_IBX, /* Ibexpeak PCH */ 1216 PCH_CPT, /* Cougarpoint/Pantherpoint PCH */ 1217 PCH_LPT, /* Lynxpoint/Wildcatpoint PCH */ 1218 PCH_SPT, /* Sunrisepoint PCH */ 1219 PCH_KBP, /* Kaby Lake PCH */ 1220 PCH_CNP, /* Cannon Lake PCH */ 1221 PCH_NOP, 1222 }; 1223 1224 enum intel_sbi_destination { 1225 SBI_ICLK, 1226 SBI_MPHY, 1227 }; 1228 1229 #define QUIRK_LVDS_SSC_DISABLE (1<<1) 1230 #define QUIRK_INVERT_BRIGHTNESS (1<<2) 1231 #define QUIRK_BACKLIGHT_PRESENT (1<<3) 1232 #define QUIRK_PIN_SWIZZLED_PAGES (1<<5) 1233 #define QUIRK_INCREASE_T12_DELAY (1<<6) 1234 1235 struct intel_fbdev; 1236 struct intel_fbc_work; 1237 1238 struct intel_gmbus { 1239 struct i2c_adapter adapter; 1240 #define GMBUS_FORCE_BIT_RETRY (1U << 31) 1241 u32 force_bit; 1242 u32 reg0; 1243 i915_reg_t gpio_reg; 1244 struct i2c_algo_bit_data bit_algo; 1245 struct drm_i915_private *dev_priv; 1246 }; 1247 1248 struct i915_suspend_saved_registers { 1249 u32 saveDSPARB; 1250 u32 saveFBC_CONTROL; 1251 u32 saveCACHE_MODE_0; 1252 u32 saveMI_ARB_STATE; 1253 u32 saveSWF0[16]; 1254 u32 saveSWF1[16]; 1255 u32 saveSWF3[3]; 1256 uint64_t saveFENCE[I915_MAX_NUM_FENCES]; 1257 u32 savePCH_PORT_HOTPLUG; 1258 u16 saveGCDGMBUS; 1259 }; 1260 1261 struct vlv_s0ix_state { 1262 /* GAM */ 1263 u32 wr_watermark; 1264 u32 gfx_prio_ctrl; 1265 u32 arb_mode; 1266 u32 gfx_pend_tlb0; 1267 u32 gfx_pend_tlb1; 1268 u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM]; 1269 u32 media_max_req_count; 1270 u32 gfx_max_req_count; 1271 u32 render_hwsp; 1272 u32 ecochk; 1273 u32 bsd_hwsp; 1274 u32 blt_hwsp; 1275 u32 tlb_rd_addr; 1276 1277 /* MBC */ 1278 u32 g3dctl; 1279 u32 gsckgctl; 1280 u32 mbctl; 1281 1282 /* GCP */ 1283 u32 ucgctl1; 1284 u32 ucgctl3; 1285 u32 rcgctl1; 1286 u32 rcgctl2; 1287 u32 rstctl; 1288 u32 misccpctl; 1289 1290 /* GPM */ 1291 u32 gfxpause; 1292 u32 rpdeuhwtc; 1293 u32 rpdeuc; 1294 u32 ecobus; 1295 u32 pwrdwnupctl; 1296 u32 rp_down_timeout; 1297 u32 rp_deucsw; 1298 u32 rcubmabdtmr; 1299 u32 rcedata; 1300 u32 spare2gh; 1301 1302 /* Display 1 CZ domain */ 1303 u32 gt_imr; 1304 u32 gt_ier; 1305 u32 pm_imr; 1306 u32 pm_ier; 1307 u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM]; 1308 1309 /* GT SA CZ domain */ 1310 u32 tilectl; 1311 u32 gt_fifoctl; 1312 u32 gtlc_wake_ctrl; 1313 u32 gtlc_survive; 1314 u32 pmwgicz; 1315 1316 /* Display 2 CZ domain */ 1317 u32 gu_ctl0; 1318 u32 gu_ctl1; 1319 u32 pcbr; 1320 u32 clock_gate_dis2; 1321 }; 1322 1323 struct intel_rps_ei { 1324 ktime_t ktime; 1325 u32 render_c0; 1326 u32 media_c0; 1327 }; 1328 1329 struct intel_rps { 1330 /* 1331 * work, interrupts_enabled and pm_iir are protected by 1332 * dev_priv->irq_lock 1333 */ 1334 struct work_struct work; 1335 bool interrupts_enabled; 1336 u32 pm_iir; 1337 1338 /* PM interrupt bits that should never be masked */ 1339 u32 pm_intrmsk_mbz; 1340 1341 /* Frequencies are stored in potentially platform dependent multiples. 1342 * In other words, *_freq needs to be multiplied by X to be interesting. 1343 * Soft limits are those which are used for the dynamic reclocking done 1344 * by the driver (raise frequencies under heavy loads, and lower for 1345 * lighter loads). Hard limits are those imposed by the hardware. 1346 * 1347 * A distinction is made for overclocking, which is never enabled by 1348 * default, and is considered to be above the hard limit if it's 1349 * possible at all. 1350 */ 1351 u8 cur_freq; /* Current frequency (cached, may not == HW) */ 1352 u8 min_freq_softlimit; /* Minimum frequency permitted by the driver */ 1353 u8 max_freq_softlimit; /* Max frequency permitted by the driver */ 1354 u8 max_freq; /* Maximum frequency, RP0 if not overclocking */ 1355 u8 min_freq; /* AKA RPn. Minimum frequency */ 1356 u8 boost_freq; /* Frequency to request when wait boosting */ 1357 u8 idle_freq; /* Frequency to request when we are idle */ 1358 u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */ 1359 u8 rp1_freq; /* "less than" RP0 power/freqency */ 1360 u8 rp0_freq; /* Non-overclocked max frequency. */ 1361 u16 gpll_ref_freq; /* vlv/chv GPLL reference frequency */ 1362 1363 u8 up_threshold; /* Current %busy required to uplock */ 1364 u8 down_threshold; /* Current %busy required to downclock */ 1365 1366 int last_adj; 1367 enum { LOW_POWER, BETWEEN, HIGH_POWER } power; 1368 1369 bool enabled; 1370 atomic_t num_waiters; 1371 atomic_t boosts; 1372 1373 /* manual wa residency calculations */ 1374 struct intel_rps_ei ei; 1375 }; 1376 1377 struct intel_rc6 { 1378 bool enabled; 1379 }; 1380 1381 struct intel_llc_pstate { 1382 bool enabled; 1383 }; 1384 1385 struct intel_gen6_power_mgmt { 1386 struct intel_rps rps; 1387 struct intel_rc6 rc6; 1388 struct intel_llc_pstate llc_pstate; 1389 struct delayed_work autoenable_work; 1390 }; 1391 1392 /* defined intel_pm.c */ 1393 extern spinlock_t mchdev_lock; 1394 1395 struct intel_ilk_power_mgmt { 1396 u8 cur_delay; 1397 u8 min_delay; 1398 u8 max_delay; 1399 u8 fmax; 1400 u8 fstart; 1401 1402 u64 last_count1; 1403 unsigned long last_time1; 1404 unsigned long chipset_power; 1405 u64 last_count2; 1406 u64 last_time2; 1407 unsigned long gfx_power; 1408 u8 corr; 1409 1410 int c_m; 1411 int r_t; 1412 }; 1413 1414 struct drm_i915_private; 1415 struct i915_power_well; 1416 1417 struct i915_power_well_ops { 1418 /* 1419 * Synchronize the well's hw state to match the current sw state, for 1420 * example enable/disable it based on the current refcount. Called 1421 * during driver init and resume time, possibly after first calling 1422 * the enable/disable handlers. 1423 */ 1424 void (*sync_hw)(struct drm_i915_private *dev_priv, 1425 struct i915_power_well *power_well); 1426 /* 1427 * Enable the well and resources that depend on it (for example 1428 * interrupts located on the well). Called after the 0->1 refcount 1429 * transition. 1430 */ 1431 void (*enable)(struct drm_i915_private *dev_priv, 1432 struct i915_power_well *power_well); 1433 /* 1434 * Disable the well and resources that depend on it. Called after 1435 * the 1->0 refcount transition. 1436 */ 1437 void (*disable)(struct drm_i915_private *dev_priv, 1438 struct i915_power_well *power_well); 1439 /* Returns the hw enabled state. */ 1440 bool (*is_enabled)(struct drm_i915_private *dev_priv, 1441 struct i915_power_well *power_well); 1442 }; 1443 1444 /* Power well structure for haswell */ 1445 struct i915_power_well { 1446 const char *name; 1447 bool always_on; 1448 /* power well enable/disable usage count */ 1449 int count; 1450 /* cached hw enabled state */ 1451 bool hw_enabled; 1452 u64 domains; 1453 /* unique identifier for this power well */ 1454 enum i915_power_well_id id; 1455 /* 1456 * Arbitraty data associated with this power well. Platform and power 1457 * well specific. 1458 */ 1459 union { 1460 struct { 1461 enum dpio_phy phy; 1462 } bxt; 1463 struct { 1464 /* Mask of pipes whose IRQ logic is backed by the pw */ 1465 u8 irq_pipe_mask; 1466 /* The pw is backing the VGA functionality */ 1467 bool has_vga:1; 1468 bool has_fuses:1; 1469 } hsw; 1470 }; 1471 const struct i915_power_well_ops *ops; 1472 }; 1473 1474 struct i915_power_domains { 1475 /* 1476 * Power wells needed for initialization at driver init and suspend 1477 * time are on. They are kept on until after the first modeset. 1478 */ 1479 bool init_power_on; 1480 bool initializing; 1481 int power_well_count; 1482 1483 struct lock lock; 1484 int domain_use_count[POWER_DOMAIN_NUM]; 1485 struct i915_power_well *power_wells; 1486 }; 1487 1488 #define MAX_L3_SLICES 2 1489 struct intel_l3_parity { 1490 u32 *remap_info[MAX_L3_SLICES]; 1491 struct work_struct error_work; 1492 int which_slice; 1493 }; 1494 1495 struct i915_gem_mm { 1496 /** Memory allocator for GTT stolen memory */ 1497 struct drm_mm stolen; 1498 /** Protects the usage of the GTT stolen memory allocator. This is 1499 * always the inner lock when overlapping with struct_mutex. */ 1500 struct lock stolen_lock; 1501 1502 /* Protects bound_list/unbound_list and #drm_i915_gem_object.mm.link */ 1503 spinlock_t obj_lock; 1504 1505 /** List of all objects in gtt_space. Used to restore gtt 1506 * mappings on resume */ 1507 struct list_head bound_list; 1508 /** 1509 * List of objects which are not bound to the GTT (thus 1510 * are idle and not used by the GPU). These objects may or may 1511 * not actually have any pages attached. 1512 */ 1513 struct list_head unbound_list; 1514 1515 /** List of all objects in gtt_space, currently mmaped by userspace. 1516 * All objects within this list must also be on bound_list. 1517 */ 1518 struct list_head userfault_list; 1519 1520 /** 1521 * List of objects which are pending destruction. 1522 */ 1523 struct llist_head free_list; 1524 struct work_struct free_work; 1525 spinlock_t free_lock; 1526 1527 /** 1528 * Small stash of WC pages 1529 */ 1530 struct pagevec wc_stash; 1531 1532 /** Usable portion of the GTT for GEM */ 1533 dma_addr_t stolen_base; /* limited to low memory (32-bit) */ 1534 1535 /** 1536 * tmpfs instance used for shmem backed objects 1537 */ 1538 struct vfsmount *gemfs; 1539 1540 /** PPGTT used for aliasing the PPGTT with the GTT */ 1541 struct i915_hw_ppgtt *aliasing_ppgtt; 1542 1543 struct notifier_block oom_notifier; 1544 struct notifier_block vmap_notifier; 1545 struct shrinker shrinker; 1546 1547 /** LRU list of objects with fence regs on them. */ 1548 struct list_head fence_list; 1549 1550 /** 1551 * Workqueue to fault in userptr pages, flushed by the execbuf 1552 * when required but otherwise left to userspace to try again 1553 * on EAGAIN. 1554 */ 1555 struct workqueue_struct *userptr_wq; 1556 1557 u64 unordered_timeline; 1558 1559 /* the indicator for dispatch video commands on two BSD rings */ 1560 atomic_t bsd_engine_dispatch_index; 1561 1562 /** Bit 6 swizzling required for X tiling */ 1563 uint32_t bit_6_swizzle_x; 1564 /** Bit 6 swizzling required for Y tiling */ 1565 uint32_t bit_6_swizzle_y; 1566 1567 /* accounting, useful for userland debugging */ 1568 spinlock_t object_stat_lock; 1569 u64 object_memory; 1570 u32 object_count; 1571 }; 1572 1573 struct drm_i915_error_state_buf { 1574 struct drm_i915_private *i915; 1575 unsigned bytes; 1576 unsigned size; 1577 int err; 1578 u8 *buf; 1579 loff_t start; 1580 loff_t pos; 1581 }; 1582 1583 #define I915_RESET_TIMEOUT (10 * HZ) /* 10s */ 1584 #define I915_FENCE_TIMEOUT (10 * HZ) /* 10s */ 1585 1586 #define I915_ENGINE_DEAD_TIMEOUT (4 * HZ) /* Seqno, head and subunits dead */ 1587 #define I915_SEQNO_DEAD_TIMEOUT (12 * HZ) /* Seqno dead with active head */ 1588 1589 struct i915_gpu_error { 1590 /* For hangcheck timer */ 1591 #define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */ 1592 #define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD) 1593 1594 struct delayed_work hangcheck_work; 1595 1596 /* For reset and error_state handling. */ 1597 spinlock_t lock; 1598 /* Protected by the above dev->gpu_error.lock. */ 1599 struct i915_gpu_state *first_error; 1600 1601 atomic_t pending_fb_pin; 1602 1603 unsigned long missed_irq_rings; 1604 1605 /** 1606 * State variable controlling the reset flow and count 1607 * 1608 * This is a counter which gets incremented when reset is triggered, 1609 * 1610 * Before the reset commences, the I915_RESET_BACKOFF bit is set 1611 * meaning that any waiters holding onto the struct_mutex should 1612 * relinquish the lock immediately in order for the reset to start. 1613 * 1614 * If reset is not completed succesfully, the I915_WEDGE bit is 1615 * set meaning that hardware is terminally sour and there is no 1616 * recovery. All waiters on the reset_queue will be woken when 1617 * that happens. 1618 * 1619 * This counter is used by the wait_seqno code to notice that reset 1620 * event happened and it needs to restart the entire ioctl (since most 1621 * likely the seqno it waited for won't ever signal anytime soon). 1622 * 1623 * This is important for lock-free wait paths, where no contended lock 1624 * naturally enforces the correct ordering between the bail-out of the 1625 * waiter and the gpu reset work code. 1626 */ 1627 unsigned long reset_count; 1628 1629 /** 1630 * flags: Control various stages of the GPU reset 1631 * 1632 * #I915_RESET_BACKOFF - When we start a reset, we want to stop any 1633 * other users acquiring the struct_mutex. To do this we set the 1634 * #I915_RESET_BACKOFF bit in the error flags when we detect a reset 1635 * and then check for that bit before acquiring the struct_mutex (in 1636 * i915_mutex_lock_interruptible()?). I915_RESET_BACKOFF serves a 1637 * secondary role in preventing two concurrent global reset attempts. 1638 * 1639 * #I915_RESET_HANDOFF - To perform the actual GPU reset, we need the 1640 * struct_mutex. We try to acquire the struct_mutex in the reset worker, 1641 * but it may be held by some long running waiter (that we cannot 1642 * interrupt without causing trouble). Once we are ready to do the GPU 1643 * reset, we set the I915_RESET_HANDOFF bit and wakeup any waiters. If 1644 * they already hold the struct_mutex and want to participate they can 1645 * inspect the bit and do the reset directly, otherwise the worker 1646 * waits for the struct_mutex. 1647 * 1648 * #I915_RESET_ENGINE[num_engines] - Since the driver doesn't need to 1649 * acquire the struct_mutex to reset an engine, we need an explicit 1650 * flag to prevent two concurrent reset attempts in the same engine. 1651 * As the number of engines continues to grow, allocate the flags from 1652 * the most significant bits. 1653 * 1654 * #I915_WEDGED - If reset fails and we can no longer use the GPU, 1655 * we set the #I915_WEDGED bit. Prior to command submission, e.g. 1656 * i915_gem_request_alloc(), this bit is checked and the sequence 1657 * aborted (with -EIO reported to userspace) if set. 1658 */ 1659 unsigned long flags; 1660 #define I915_RESET_BACKOFF 0 1661 #define I915_RESET_HANDOFF 1 1662 #define I915_RESET_MODESET 2 1663 #define I915_WEDGED (BITS_PER_LONG - 1) 1664 #define I915_RESET_ENGINE (I915_WEDGED - I915_NUM_ENGINES) 1665 1666 /** Number of times an engine has been reset */ 1667 u32 reset_engine_count[I915_NUM_ENGINES]; 1668 1669 /** 1670 * Waitqueue to signal when a hang is detected. Used to for waiters 1671 * to release the struct_mutex for the reset to procede. 1672 */ 1673 wait_queue_head_t wait_queue; 1674 1675 /** 1676 * Waitqueue to signal when the reset has completed. Used by clients 1677 * that wait for dev_priv->mm.wedged to settle. 1678 */ 1679 wait_queue_head_t reset_queue; 1680 1681 /* For missed irq/seqno simulation. */ 1682 unsigned long test_irq_rings; 1683 }; 1684 1685 enum modeset_restore { 1686 MODESET_ON_LID_OPEN, 1687 MODESET_DONE, 1688 MODESET_SUSPENDED, 1689 }; 1690 1691 #define DP_AUX_A 0x40 1692 #define DP_AUX_B 0x10 1693 #define DP_AUX_C 0x20 1694 #define DP_AUX_D 0x30 1695 1696 #define DDC_PIN_B 0x05 1697 #define DDC_PIN_C 0x04 1698 #define DDC_PIN_D 0x06 1699 1700 struct ddi_vbt_port_info { 1701 /* 1702 * This is an index in the HDMI/DVI DDI buffer translation table. 1703 * The special value HDMI_LEVEL_SHIFT_UNKNOWN means the VBT didn't 1704 * populate this field. 1705 */ 1706 #define HDMI_LEVEL_SHIFT_UNKNOWN 0xff 1707 uint8_t hdmi_level_shift; 1708 1709 uint8_t supports_dvi:1; 1710 uint8_t supports_hdmi:1; 1711 uint8_t supports_dp:1; 1712 uint8_t supports_edp:1; 1713 1714 uint8_t alternate_aux_channel; 1715 uint8_t alternate_ddc_pin; 1716 1717 uint8_t dp_boost_level; 1718 uint8_t hdmi_boost_level; 1719 }; 1720 1721 enum psr_lines_to_wait { 1722 PSR_0_LINES_TO_WAIT = 0, 1723 PSR_1_LINE_TO_WAIT, 1724 PSR_4_LINES_TO_WAIT, 1725 PSR_8_LINES_TO_WAIT 1726 }; 1727 1728 struct intel_vbt_data { 1729 struct drm_display_mode *lfp_lvds_vbt_mode; /* if any */ 1730 struct drm_display_mode *sdvo_lvds_vbt_mode; /* if any */ 1731 1732 /* Feature bits */ 1733 unsigned int int_tv_support:1; 1734 unsigned int lvds_dither:1; 1735 unsigned int lvds_vbt:1; 1736 unsigned int int_crt_support:1; 1737 unsigned int lvds_use_ssc:1; 1738 unsigned int display_clock_mode:1; 1739 unsigned int fdi_rx_polarity_inverted:1; 1740 unsigned int panel_type:4; 1741 int lvds_ssc_freq; 1742 unsigned int bios_lvds_val; /* initial [PCH_]LVDS reg val in VBIOS */ 1743 1744 enum drrs_support_type drrs_type; 1745 1746 struct { 1747 int rate; 1748 int lanes; 1749 int preemphasis; 1750 int vswing; 1751 bool low_vswing; 1752 bool initialized; 1753 bool support; 1754 int bpp; 1755 struct edp_power_seq pps; 1756 } edp; 1757 1758 struct { 1759 bool full_link; 1760 bool require_aux_wakeup; 1761 int idle_frames; 1762 enum psr_lines_to_wait lines_to_wait; 1763 int tp1_wakeup_time; 1764 int tp2_tp3_wakeup_time; 1765 } psr; 1766 1767 struct { 1768 u16 pwm_freq_hz; 1769 bool present; 1770 bool active_low_pwm; 1771 u8 min_brightness; /* min_brightness/255 of max */ 1772 u8 controller; /* brightness controller number */ 1773 enum intel_backlight_type type; 1774 } backlight; 1775 1776 /* MIPI DSI */ 1777 struct { 1778 u16 panel_id; 1779 struct mipi_config *config; 1780 struct mipi_pps_data *pps; 1781 u16 bl_ports; 1782 u16 cabc_ports; 1783 u8 seq_version; 1784 u32 size; 1785 u8 *data; 1786 const u8 *sequence[MIPI_SEQ_MAX]; 1787 } dsi; 1788 1789 int crt_ddc_pin; 1790 1791 int child_dev_num; 1792 struct child_device_config *child_dev; 1793 1794 struct ddi_vbt_port_info ddi_port_info[I915_MAX_PORTS]; 1795 struct sdvo_device_mapping sdvo_mappings[2]; 1796 }; 1797 1798 enum intel_ddb_partitioning { 1799 INTEL_DDB_PART_1_2, 1800 INTEL_DDB_PART_5_6, /* IVB+ */ 1801 }; 1802 1803 struct intel_wm_level { 1804 bool enable; 1805 uint32_t pri_val; 1806 uint32_t spr_val; 1807 uint32_t cur_val; 1808 uint32_t fbc_val; 1809 }; 1810 1811 struct ilk_wm_values { 1812 uint32_t wm_pipe[3]; 1813 uint32_t wm_lp[3]; 1814 uint32_t wm_lp_spr[3]; 1815 uint32_t wm_linetime[3]; 1816 bool enable_fbc_wm; 1817 enum intel_ddb_partitioning partitioning; 1818 }; 1819 1820 struct g4x_pipe_wm { 1821 uint16_t plane[I915_MAX_PLANES]; 1822 uint16_t fbc; 1823 }; 1824 1825 struct g4x_sr_wm { 1826 uint16_t plane; 1827 uint16_t cursor; 1828 uint16_t fbc; 1829 }; 1830 1831 struct vlv_wm_ddl_values { 1832 uint8_t plane[I915_MAX_PLANES]; 1833 }; 1834 1835 struct vlv_wm_values { 1836 struct g4x_pipe_wm pipe[3]; 1837 struct g4x_sr_wm sr; 1838 struct vlv_wm_ddl_values ddl[3]; 1839 uint8_t level; 1840 bool cxsr; 1841 }; 1842 1843 struct g4x_wm_values { 1844 struct g4x_pipe_wm pipe[2]; 1845 struct g4x_sr_wm sr; 1846 struct g4x_sr_wm hpll; 1847 bool cxsr; 1848 bool hpll_en; 1849 bool fbc_en; 1850 }; 1851 1852 struct skl_ddb_entry { 1853 uint16_t start, end; /* in number of blocks, 'end' is exclusive */ 1854 }; 1855 1856 static inline uint16_t skl_ddb_entry_size(const struct skl_ddb_entry *entry) 1857 { 1858 return entry->end - entry->start; 1859 } 1860 1861 static inline bool skl_ddb_entry_equal(const struct skl_ddb_entry *e1, 1862 const struct skl_ddb_entry *e2) 1863 { 1864 if (e1->start == e2->start && e1->end == e2->end) 1865 return true; 1866 1867 return false; 1868 } 1869 1870 struct skl_ddb_allocation { 1871 struct skl_ddb_entry plane[I915_MAX_PIPES][I915_MAX_PLANES]; /* packed/uv */ 1872 struct skl_ddb_entry y_plane[I915_MAX_PIPES][I915_MAX_PLANES]; 1873 }; 1874 1875 struct skl_wm_values { 1876 unsigned dirty_pipes; 1877 struct skl_ddb_allocation ddb; 1878 }; 1879 1880 struct skl_wm_level { 1881 bool plane_en; 1882 uint16_t plane_res_b; 1883 uint8_t plane_res_l; 1884 }; 1885 1886 /* Stores plane specific WM parameters */ 1887 struct skl_wm_params { 1888 bool x_tiled, y_tiled; 1889 bool rc_surface; 1890 uint32_t width; 1891 uint8_t cpp; 1892 uint32_t plane_pixel_rate; 1893 uint32_t y_min_scanlines; 1894 uint32_t plane_bytes_per_line; 1895 uint_fixed_16_16_t plane_blocks_per_line; 1896 uint_fixed_16_16_t y_tile_minimum; 1897 uint32_t linetime_us; 1898 }; 1899 1900 /* 1901 * This struct helps tracking the state needed for runtime PM, which puts the 1902 * device in PCI D3 state. Notice that when this happens, nothing on the 1903 * graphics device works, even register access, so we don't get interrupts nor 1904 * anything else. 1905 * 1906 * Every piece of our code that needs to actually touch the hardware needs to 1907 * either call intel_runtime_pm_get or call intel_display_power_get with the 1908 * appropriate power domain. 1909 * 1910 * Our driver uses the autosuspend delay feature, which means we'll only really 1911 * suspend if we stay with zero refcount for a certain amount of time. The 1912 * default value is currently very conservative (see intel_runtime_pm_enable), but 1913 * it can be changed with the standard runtime PM files from sysfs. 1914 * 1915 * The irqs_disabled variable becomes true exactly after we disable the IRQs and 1916 * goes back to false exactly before we reenable the IRQs. We use this variable 1917 * to check if someone is trying to enable/disable IRQs while they're supposed 1918 * to be disabled. This shouldn't happen and we'll print some error messages in 1919 * case it happens. 1920 * 1921 * For more, read the Documentation/power/runtime_pm.txt. 1922 */ 1923 struct i915_runtime_pm { 1924 atomic_t wakeref_count; 1925 bool suspended; 1926 bool irqs_enabled; 1927 }; 1928 1929 enum intel_pipe_crc_source { 1930 INTEL_PIPE_CRC_SOURCE_NONE, 1931 INTEL_PIPE_CRC_SOURCE_PLANE1, 1932 INTEL_PIPE_CRC_SOURCE_PLANE2, 1933 INTEL_PIPE_CRC_SOURCE_PF, 1934 INTEL_PIPE_CRC_SOURCE_PIPE, 1935 /* TV/DP on pre-gen5/vlv can't use the pipe source. */ 1936 INTEL_PIPE_CRC_SOURCE_TV, 1937 INTEL_PIPE_CRC_SOURCE_DP_B, 1938 INTEL_PIPE_CRC_SOURCE_DP_C, 1939 INTEL_PIPE_CRC_SOURCE_DP_D, 1940 INTEL_PIPE_CRC_SOURCE_AUTO, 1941 INTEL_PIPE_CRC_SOURCE_MAX, 1942 }; 1943 1944 struct intel_pipe_crc_entry { 1945 uint32_t frame; 1946 uint32_t crc[5]; 1947 }; 1948 1949 #define INTEL_PIPE_CRC_ENTRIES_NR 128 1950 struct intel_pipe_crc { 1951 spinlock_t lock; 1952 bool opened; /* exclusive access to the result file */ 1953 struct intel_pipe_crc_entry *entries; 1954 enum intel_pipe_crc_source source; 1955 int head, tail; 1956 wait_queue_head_t wq; 1957 int skipped; 1958 }; 1959 1960 struct i915_frontbuffer_tracking { 1961 spinlock_t lock; 1962 1963 /* 1964 * Tracking bits for delayed frontbuffer flushing du to gpu activity or 1965 * scheduled flips. 1966 */ 1967 unsigned busy_bits; 1968 unsigned flip_bits; 1969 }; 1970 1971 struct i915_wa_reg { 1972 i915_reg_t addr; 1973 u32 value; 1974 /* bitmask representing WA bits */ 1975 u32 mask; 1976 }; 1977 1978 #define I915_MAX_WA_REGS 16 1979 1980 struct i915_workarounds { 1981 struct i915_wa_reg reg[I915_MAX_WA_REGS]; 1982 u32 count; 1983 u32 hw_whitelist_count[I915_NUM_ENGINES]; 1984 }; 1985 1986 struct i915_virtual_gpu { 1987 bool active; 1988 u32 caps; 1989 }; 1990 1991 /* used in computing the new watermarks state */ 1992 struct intel_wm_config { 1993 unsigned int num_pipes_active; 1994 bool sprites_enabled; 1995 bool sprites_scaled; 1996 }; 1997 1998 struct i915_oa_format { 1999 u32 format; 2000 int size; 2001 }; 2002 2003 struct i915_oa_reg { 2004 i915_reg_t addr; 2005 u32 value; 2006 }; 2007 2008 struct i915_oa_config { 2009 char uuid[UUID_STRING_LEN + 1]; 2010 int id; 2011 2012 const struct i915_oa_reg *mux_regs; 2013 u32 mux_regs_len; 2014 const struct i915_oa_reg *b_counter_regs; 2015 u32 b_counter_regs_len; 2016 const struct i915_oa_reg *flex_regs; 2017 u32 flex_regs_len; 2018 2019 struct attribute_group sysfs_metric; 2020 struct attribute *attrs[2]; 2021 struct device_attribute sysfs_metric_id; 2022 2023 atomic_t ref_count; 2024 }; 2025 2026 struct i915_perf_stream; 2027 2028 /** 2029 * struct i915_perf_stream_ops - the OPs to support a specific stream type 2030 */ 2031 struct i915_perf_stream_ops { 2032 /** 2033 * @enable: Enables the collection of HW samples, either in response to 2034 * `I915_PERF_IOCTL_ENABLE` or implicitly called when stream is opened 2035 * without `I915_PERF_FLAG_DISABLED`. 2036 */ 2037 void (*enable)(struct i915_perf_stream *stream); 2038 2039 /** 2040 * @disable: Disables the collection of HW samples, either in response 2041 * to `I915_PERF_IOCTL_DISABLE` or implicitly called before destroying 2042 * the stream. 2043 */ 2044 void (*disable)(struct i915_perf_stream *stream); 2045 2046 /** 2047 * @poll_wait: Call poll_wait, passing a wait queue that will be woken 2048 * once there is something ready to read() for the stream 2049 */ 2050 void (*poll_wait)(struct i915_perf_stream *stream, 2051 struct file *file, 2052 poll_table *wait); 2053 2054 /** 2055 * @wait_unlocked: For handling a blocking read, wait until there is 2056 * something to ready to read() for the stream. E.g. wait on the same 2057 * wait queue that would be passed to poll_wait(). 2058 */ 2059 int (*wait_unlocked)(struct i915_perf_stream *stream); 2060 2061 /** 2062 * @read: Copy buffered metrics as records to userspace 2063 * **buf**: the userspace, destination buffer 2064 * **count**: the number of bytes to copy, requested by userspace 2065 * **offset**: zero at the start of the read, updated as the read 2066 * proceeds, it represents how many bytes have been copied so far and 2067 * the buffer offset for copying the next record. 2068 * 2069 * Copy as many buffered i915 perf samples and records for this stream 2070 * to userspace as will fit in the given buffer. 2071 * 2072 * Only write complete records; returning -%ENOSPC if there isn't room 2073 * for a complete record. 2074 * 2075 * Return any error condition that results in a short read such as 2076 * -%ENOSPC or -%EFAULT, even though these may be squashed before 2077 * returning to userspace. 2078 */ 2079 int (*read)(struct i915_perf_stream *stream, 2080 char __user *buf, 2081 size_t count, 2082 size_t *offset); 2083 2084 /** 2085 * @destroy: Cleanup any stream specific resources. 2086 * 2087 * The stream will always be disabled before this is called. 2088 */ 2089 void (*destroy)(struct i915_perf_stream *stream); 2090 }; 2091 2092 /** 2093 * struct i915_perf_stream - state for a single open stream FD 2094 */ 2095 struct i915_perf_stream { 2096 /** 2097 * @dev_priv: i915 drm device 2098 */ 2099 struct drm_i915_private *dev_priv; 2100 2101 /** 2102 * @link: Links the stream into ``&drm_i915_private->streams`` 2103 */ 2104 struct list_head link; 2105 2106 /** 2107 * @sample_flags: Flags representing the `DRM_I915_PERF_PROP_SAMPLE_*` 2108 * properties given when opening a stream, representing the contents 2109 * of a single sample as read() by userspace. 2110 */ 2111 u32 sample_flags; 2112 2113 /** 2114 * @sample_size: Considering the configured contents of a sample 2115 * combined with the required header size, this is the total size 2116 * of a single sample record. 2117 */ 2118 int sample_size; 2119 2120 /** 2121 * @ctx: %NULL if measuring system-wide across all contexts or a 2122 * specific context that is being monitored. 2123 */ 2124 struct i915_gem_context *ctx; 2125 2126 /** 2127 * @enabled: Whether the stream is currently enabled, considering 2128 * whether the stream was opened in a disabled state and based 2129 * on `I915_PERF_IOCTL_ENABLE` and `I915_PERF_IOCTL_DISABLE` calls. 2130 */ 2131 bool enabled; 2132 2133 /** 2134 * @ops: The callbacks providing the implementation of this specific 2135 * type of configured stream. 2136 */ 2137 const struct i915_perf_stream_ops *ops; 2138 2139 /** 2140 * @oa_config: The OA configuration used by the stream. 2141 */ 2142 struct i915_oa_config *oa_config; 2143 }; 2144 2145 /** 2146 * struct i915_oa_ops - Gen specific implementation of an OA unit stream 2147 */ 2148 struct i915_oa_ops { 2149 /** 2150 * @is_valid_b_counter_reg: Validates register's address for 2151 * programming boolean counters for a particular platform. 2152 */ 2153 bool (*is_valid_b_counter_reg)(struct drm_i915_private *dev_priv, 2154 u32 addr); 2155 2156 /** 2157 * @is_valid_mux_reg: Validates register's address for programming mux 2158 * for a particular platform. 2159 */ 2160 bool (*is_valid_mux_reg)(struct drm_i915_private *dev_priv, u32 addr); 2161 2162 /** 2163 * @is_valid_flex_reg: Validates register's address for programming 2164 * flex EU filtering for a particular platform. 2165 */ 2166 bool (*is_valid_flex_reg)(struct drm_i915_private *dev_priv, u32 addr); 2167 2168 /** 2169 * @init_oa_buffer: Resets the head and tail pointers of the 2170 * circular buffer for periodic OA reports. 2171 * 2172 * Called when first opening a stream for OA metrics, but also may be 2173 * called in response to an OA buffer overflow or other error 2174 * condition. 2175 * 2176 * Note it may be necessary to clear the full OA buffer here as part of 2177 * maintaining the invariable that new reports must be written to 2178 * zeroed memory for us to be able to reliable detect if an expected 2179 * report has not yet landed in memory. (At least on Haswell the OA 2180 * buffer tail pointer is not synchronized with reports being visible 2181 * to the CPU) 2182 */ 2183 void (*init_oa_buffer)(struct drm_i915_private *dev_priv); 2184 2185 /** 2186 * @enable_metric_set: Selects and applies any MUX configuration to set 2187 * up the Boolean and Custom (B/C) counters that are part of the 2188 * counter reports being sampled. May apply system constraints such as 2189 * disabling EU clock gating as required. 2190 */ 2191 int (*enable_metric_set)(struct drm_i915_private *dev_priv, 2192 const struct i915_oa_config *oa_config); 2193 2194 /** 2195 * @disable_metric_set: Remove system constraints associated with using 2196 * the OA unit. 2197 */ 2198 void (*disable_metric_set)(struct drm_i915_private *dev_priv); 2199 2200 /** 2201 * @oa_enable: Enable periodic sampling 2202 */ 2203 void (*oa_enable)(struct drm_i915_private *dev_priv); 2204 2205 /** 2206 * @oa_disable: Disable periodic sampling 2207 */ 2208 void (*oa_disable)(struct drm_i915_private *dev_priv); 2209 2210 /** 2211 * @read: Copy data from the circular OA buffer into a given userspace 2212 * buffer. 2213 */ 2214 int (*read)(struct i915_perf_stream *stream, 2215 char __user *buf, 2216 size_t count, 2217 size_t *offset); 2218 2219 /** 2220 * @oa_hw_tail_read: read the OA tail pointer register 2221 * 2222 * In particular this enables us to share all the fiddly code for 2223 * handling the OA unit tail pointer race that affects multiple 2224 * generations. 2225 */ 2226 u32 (*oa_hw_tail_read)(struct drm_i915_private *dev_priv); 2227 }; 2228 2229 struct intel_cdclk_state { 2230 unsigned int cdclk, vco, ref; 2231 }; 2232 2233 struct drm_i915_private { 2234 struct drm_device drm; 2235 2236 struct kmem_cache *objects; 2237 struct kmem_cache *vmas; 2238 struct kmem_cache *luts; 2239 struct kmem_cache *requests; 2240 struct kmem_cache *dependencies; 2241 struct kmem_cache *priorities; 2242 2243 const struct intel_device_info info; 2244 2245 void __iomem *regs; 2246 2247 struct intel_uncore uncore; 2248 2249 struct i915_virtual_gpu vgpu; 2250 2251 struct intel_gvt *gvt; 2252 2253 struct intel_huc huc; 2254 struct intel_guc guc; 2255 2256 struct intel_csr csr; 2257 2258 struct intel_gmbus gmbus[GMBUS_NUM_PINS]; 2259 2260 /** gmbus_mutex protects against concurrent usage of the single hw gmbus 2261 * controller on different i2c buses. */ 2262 struct lock gmbus_mutex; 2263 2264 /** 2265 * Base address of the gmbus and gpio block. 2266 */ 2267 uint32_t gpio_mmio_base; 2268 2269 /* MMIO base address for MIPI regs */ 2270 uint32_t mipi_mmio_base; 2271 2272 uint32_t psr_mmio_base; 2273 2274 uint32_t pps_mmio_base; 2275 2276 wait_queue_head_t gmbus_wait_queue; 2277 2278 struct pci_dev *bridge_dev; 2279 struct intel_engine_cs *engine[I915_NUM_ENGINES]; 2280 /* Context used internally to idle the GPU and setup initial state */ 2281 struct i915_gem_context *kernel_context; 2282 /* Context only to be used for injecting preemption commands */ 2283 struct i915_gem_context *preempt_context; 2284 struct i915_vma *semaphore; 2285 2286 struct drm_dma_handle *status_page_dmah; 2287 struct resource *mch_res; 2288 int mch_res_rid; 2289 2290 /* protects the irq masks */ 2291 spinlock_t irq_lock; 2292 2293 bool display_irqs_enabled; 2294 2295 /* To control wakeup latency, e.g. for irq-driven dp aux transfers. */ 2296 struct pm_qos_request pm_qos; 2297 2298 /* Sideband mailbox protection */ 2299 struct lock sb_lock; 2300 2301 /** Cached value of IMR to avoid reads in updating the bitfield */ 2302 union { 2303 u32 irq_mask; 2304 u32 de_irq_mask[I915_MAX_PIPES]; 2305 }; 2306 u32 gt_irq_mask; 2307 u32 pm_imr; 2308 u32 pm_ier; 2309 u32 pm_rps_events; 2310 u32 pm_guc_events; 2311 u32 pipestat_irq_mask[I915_MAX_PIPES]; 2312 2313 struct i915_hotplug hotplug; 2314 struct intel_fbc fbc; 2315 struct i915_drrs drrs; 2316 struct intel_opregion opregion; 2317 struct intel_vbt_data vbt; 2318 2319 bool preserve_bios_swizzle; 2320 2321 /* overlay */ 2322 struct intel_overlay *overlay; 2323 2324 /* backlight registers and fields in struct intel_panel */ 2325 struct lock backlight_lock; 2326 2327 /* LVDS info */ 2328 bool no_aux_handshake; 2329 2330 /* protects panel power sequencer state */ 2331 struct lock pps_mutex; 2332 2333 struct drm_i915_fence_reg fence_regs[I915_MAX_NUM_FENCES]; /* assume 965 */ 2334 int num_fence_regs; /* 8 on pre-965, 16 otherwise */ 2335 2336 unsigned int fsb_freq, mem_freq, is_ddr3; 2337 unsigned int skl_preferred_vco_freq; 2338 unsigned int max_cdclk_freq; 2339 2340 unsigned int max_dotclk_freq; 2341 unsigned int rawclk_freq; 2342 unsigned int hpll_freq; 2343 unsigned int czclk_freq; 2344 2345 struct { 2346 /* 2347 * The current logical cdclk state. 2348 * See intel_atomic_state.cdclk.logical 2349 * 2350 * For reading holding any crtc lock is sufficient, 2351 * for writing must hold all of them. 2352 */ 2353 struct intel_cdclk_state logical; 2354 /* 2355 * The current actual cdclk state. 2356 * See intel_atomic_state.cdclk.actual 2357 */ 2358 struct intel_cdclk_state actual; 2359 /* The current hardware cdclk state */ 2360 struct intel_cdclk_state hw; 2361 } cdclk; 2362 2363 /** 2364 * wq - Driver workqueue for GEM. 2365 * 2366 * NOTE: Work items scheduled here are not allowed to grab any modeset 2367 * locks, for otherwise the flushing done in the pageflip code will 2368 * result in deadlocks. 2369 */ 2370 struct workqueue_struct *wq; 2371 2372 /* ordered wq for modesets */ 2373 struct workqueue_struct *modeset_wq; 2374 2375 /* Display functions */ 2376 struct drm_i915_display_funcs display; 2377 2378 /* PCH chipset type */ 2379 enum intel_pch pch_type; 2380 unsigned short pch_id; 2381 2382 unsigned long quirks; 2383 2384 enum modeset_restore modeset_restore; 2385 struct lock modeset_restore_lock; 2386 struct drm_atomic_state *modeset_restore_state; 2387 struct drm_modeset_acquire_ctx reset_ctx; 2388 2389 struct list_head vm_list; /* Global list of all address spaces */ 2390 struct i915_ggtt ggtt; /* VM representing the global address space */ 2391 2392 struct i915_gem_mm mm; 2393 DECLARE_HASHTABLE(mm_structs, 7); 2394 struct lock mm_lock; 2395 2396 struct intel_ppat ppat; 2397 2398 /* Kernel Modesetting */ 2399 2400 struct intel_crtc *plane_to_crtc_mapping[I915_MAX_PIPES]; 2401 struct intel_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES]; 2402 2403 #ifdef CONFIG_DEBUG_FS 2404 struct intel_pipe_crc pipe_crc[I915_MAX_PIPES]; 2405 #endif 2406 2407 /* dpll and cdclk state is protected by connection_mutex */ 2408 int num_shared_dpll; 2409 struct intel_shared_dpll shared_dplls[I915_NUM_PLLS]; 2410 const struct intel_dpll_mgr *dpll_mgr; 2411 2412 /* 2413 * dpll_lock serializes intel_{prepare,enable,disable}_shared_dpll. 2414 * Must be global rather than per dpll, because on some platforms 2415 * plls share registers. 2416 */ 2417 struct lock dpll_lock; 2418 2419 unsigned int active_crtcs; 2420 /* minimum acceptable cdclk for each pipe */ 2421 int min_cdclk[I915_MAX_PIPES]; 2422 2423 int dpio_phy_iosf_port[I915_NUM_PHYS_VLV]; 2424 2425 struct i915_workarounds workarounds; 2426 2427 struct i915_frontbuffer_tracking fb_tracking; 2428 2429 struct intel_atomic_helper { 2430 struct llist_head free_list; 2431 struct work_struct free_work; 2432 } atomic_helper; 2433 2434 u16 orig_clock; 2435 2436 bool mchbar_need_disable; 2437 2438 struct intel_l3_parity l3_parity; 2439 2440 /* Cannot be determined by PCIID. You must always read a register. */ 2441 u32 edram_cap; 2442 2443 /* 2444 * Protects RPS/RC6 register access and PCU communication. 2445 * Must be taken after struct_mutex if nested. Note that 2446 * this lock may be held for long periods of time when 2447 * talking to hw - so only take it when talking to hw! 2448 */ 2449 struct lock pcu_lock; 2450 2451 /* gen6+ GT PM state */ 2452 struct intel_gen6_power_mgmt gt_pm; 2453 2454 /* ilk-only ips/rps state. Everything in here is protected by the global 2455 * mchdev_lock in intel_pm.c */ 2456 struct intel_ilk_power_mgmt ips; 2457 2458 struct i915_power_domains power_domains; 2459 2460 struct i915_psr psr; 2461 2462 struct i915_gpu_error gpu_error; 2463 2464 struct drm_i915_gem_object *vlv_pctx; 2465 2466 /* list of fbdev register on this device */ 2467 struct intel_fbdev *fbdev; 2468 struct work_struct fbdev_suspend_work; 2469 2470 struct drm_property *broadcast_rgb_property; 2471 struct drm_property *force_audio_property; 2472 2473 /* hda/i915 audio component */ 2474 struct i915_audio_component *audio_component; 2475 bool audio_component_registered; 2476 /** 2477 * av_mutex - mutex for audio/video sync 2478 * 2479 */ 2480 struct lock av_mutex; 2481 2482 struct { 2483 struct list_head list; 2484 struct llist_head free_list; 2485 struct work_struct free_work; 2486 2487 /* The hw wants to have a stable context identifier for the 2488 * lifetime of the context (for OA, PASID, faults, etc). 2489 * This is limited in execlists to 21 bits. 2490 */ 2491 struct ida hw_ida; 2492 #define MAX_CONTEXT_HW_ID (1<<21) /* exclusive */ 2493 } contexts; 2494 2495 u32 fdi_rx_config; 2496 2497 /* Shadow for DISPLAY_PHY_CONTROL which can't be safely read */ 2498 u32 chv_phy_control; 2499 /* 2500 * Shadows for CHV DPLL_MD regs to keep the state 2501 * checker somewhat working in the presence hardware 2502 * crappiness (can't read out DPLL_MD for pipes B & C). 2503 */ 2504 u32 chv_dpll_md[I915_MAX_PIPES]; 2505 u32 bxt_phy_grc; 2506 2507 u32 suspend_count; 2508 bool suspended_to_idle; 2509 struct i915_suspend_saved_registers regfile; 2510 struct vlv_s0ix_state vlv_s0ix_state; 2511 2512 enum { 2513 I915_SAGV_UNKNOWN = 0, 2514 I915_SAGV_DISABLED, 2515 I915_SAGV_ENABLED, 2516 I915_SAGV_NOT_CONTROLLED 2517 } sagv_status; 2518 2519 struct { 2520 /* 2521 * Raw watermark latency values: 2522 * in 0.1us units for WM0, 2523 * in 0.5us units for WM1+. 2524 */ 2525 /* primary */ 2526 uint16_t pri_latency[5]; 2527 /* sprite */ 2528 uint16_t spr_latency[5]; 2529 /* cursor */ 2530 uint16_t cur_latency[5]; 2531 /* 2532 * Raw watermark memory latency values 2533 * for SKL for all 8 levels 2534 * in 1us units. 2535 */ 2536 uint16_t skl_latency[8]; 2537 2538 /* current hardware state */ 2539 union { 2540 struct ilk_wm_values hw; 2541 struct skl_wm_values skl_hw; 2542 struct vlv_wm_values vlv; 2543 struct g4x_wm_values g4x; 2544 }; 2545 2546 uint8_t max_level; 2547 2548 /* 2549 * Should be held around atomic WM register writing; also 2550 * protects * intel_crtc->wm.active and 2551 * cstate->wm.need_postvbl_update. 2552 */ 2553 struct lock wm_mutex; 2554 2555 /* 2556 * Set during HW readout of watermarks/DDB. Some platforms 2557 * need to know when we're still using BIOS-provided values 2558 * (which we don't fully trust). 2559 */ 2560 bool distrust_bios_wm; 2561 } wm; 2562 2563 struct i915_runtime_pm runtime_pm; 2564 2565 struct { 2566 bool initialized; 2567 2568 struct kobject *metrics_kobj; 2569 struct ctl_table_header *sysctl_header; 2570 2571 /* 2572 * Lock associated with adding/modifying/removing OA configs 2573 * in dev_priv->perf.metrics_idr. 2574 */ 2575 struct lock metrics_lock; 2576 2577 /* 2578 * List of dynamic configurations, you need to hold 2579 * dev_priv->perf.metrics_lock to access it. 2580 */ 2581 struct idr metrics_idr; 2582 2583 /* 2584 * Lock associated with anything below within this structure 2585 * except exclusive_stream. 2586 */ 2587 struct lock lock; 2588 struct list_head streams; 2589 2590 struct { 2591 /* 2592 * The stream currently using the OA unit. If accessed 2593 * outside a syscall associated to its file 2594 * descriptor, you need to hold 2595 * dev_priv->drm.struct_mutex. 2596 */ 2597 struct i915_perf_stream *exclusive_stream; 2598 2599 u32 specific_ctx_id; 2600 2601 struct hrtimer poll_check_timer; 2602 wait_queue_head_t poll_wq; 2603 bool pollin; 2604 2605 /** 2606 * For rate limiting any notifications of spurious 2607 * invalid OA reports 2608 */ 2609 struct ratelimit_state spurious_report_rs; 2610 2611 bool periodic; 2612 int period_exponent; 2613 int timestamp_frequency; 2614 2615 struct i915_oa_config test_config; 2616 2617 struct { 2618 struct i915_vma *vma; 2619 u8 *vaddr; 2620 u32 last_ctx_id; 2621 int format; 2622 int format_size; 2623 2624 /** 2625 * Locks reads and writes to all head/tail state 2626 * 2627 * Consider: the head and tail pointer state 2628 * needs to be read consistently from a hrtimer 2629 * callback (atomic context) and read() fop 2630 * (user context) with tail pointer updates 2631 * happening in atomic context and head updates 2632 * in user context and the (unlikely) 2633 * possibility of read() errors needing to 2634 * reset all head/tail state. 2635 * 2636 * Note: Contention or performance aren't 2637 * currently a significant concern here 2638 * considering the relatively low frequency of 2639 * hrtimer callbacks (5ms period) and that 2640 * reads typically only happen in response to a 2641 * hrtimer event and likely complete before the 2642 * next callback. 2643 * 2644 * Note: This lock is not held *while* reading 2645 * and copying data to userspace so the value 2646 * of head observed in htrimer callbacks won't 2647 * represent any partial consumption of data. 2648 */ 2649 spinlock_t ptr_lock; 2650 2651 /** 2652 * One 'aging' tail pointer and one 'aged' 2653 * tail pointer ready to used for reading. 2654 * 2655 * Initial values of 0xffffffff are invalid 2656 * and imply that an update is required 2657 * (and should be ignored by an attempted 2658 * read) 2659 */ 2660 struct { 2661 u32 offset; 2662 } tails[2]; 2663 2664 /** 2665 * Index for the aged tail ready to read() 2666 * data up to. 2667 */ 2668 unsigned int aged_tail_idx; 2669 2670 /** 2671 * A monotonic timestamp for when the current 2672 * aging tail pointer was read; used to 2673 * determine when it is old enough to trust. 2674 */ 2675 u64 aging_timestamp; 2676 2677 /** 2678 * Although we can always read back the head 2679 * pointer register, we prefer to avoid 2680 * trusting the HW state, just to avoid any 2681 * risk that some hardware condition could 2682 * somehow bump the head pointer unpredictably 2683 * and cause us to forward the wrong OA buffer 2684 * data to userspace. 2685 */ 2686 u32 head; 2687 } oa_buffer; 2688 2689 u32 gen7_latched_oastatus1; 2690 u32 ctx_oactxctrl_offset; 2691 u32 ctx_flexeu0_offset; 2692 2693 /** 2694 * The RPT_ID/reason field for Gen8+ includes a bit 2695 * to determine if the CTX ID in the report is valid 2696 * but the specific bit differs between Gen 8 and 9 2697 */ 2698 u32 gen8_valid_ctx_bit; 2699 2700 struct i915_oa_ops ops; 2701 const struct i915_oa_format *oa_formats; 2702 } oa; 2703 } perf; 2704 2705 /* Abstract the submission mechanism (legacy ringbuffer or execlists) away */ 2706 struct { 2707 void (*resume)(struct drm_i915_private *); 2708 void (*cleanup_engine)(struct intel_engine_cs *engine); 2709 2710 struct list_head timelines; 2711 struct i915_gem_timeline global_timeline; 2712 u32 active_requests; 2713 2714 /** 2715 * Is the GPU currently considered idle, or busy executing 2716 * userspace requests? Whilst idle, we allow runtime power 2717 * management to power down the hardware and display clocks. 2718 * In order to reduce the effect on performance, there 2719 * is a slight delay before we do so. 2720 */ 2721 bool awake; 2722 2723 /** 2724 * We leave the user IRQ off as much as possible, 2725 * but this means that requests will finish and never 2726 * be retired once the system goes idle. Set a timer to 2727 * fire periodically while the ring is running. When it 2728 * fires, go retire requests. 2729 */ 2730 struct delayed_work retire_work; 2731 2732 /** 2733 * When we detect an idle GPU, we want to turn on 2734 * powersaving features. So once we see that there 2735 * are no more requests outstanding and no more 2736 * arrive within a small period of time, we fire 2737 * off the idle_work. 2738 */ 2739 struct delayed_work idle_work; 2740 2741 ktime_t last_init_time; 2742 } gt; 2743 2744 /* perform PHY state sanity checks? */ 2745 bool chv_phy_assert[2]; 2746 2747 bool ipc_enabled; 2748 2749 /* Used to save the pipe-to-encoder mapping for audio */ 2750 struct intel_encoder *av_enc_map[I915_MAX_PIPES]; 2751 2752 /* necessary resource sharing with HDMI LPE audio driver. */ 2753 struct { 2754 struct platform_device *platdev; 2755 int irq; 2756 } lpe_audio; 2757 2758 /* 2759 * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch 2760 * will be rejected. Instead look for a better place. 2761 */ 2762 }; 2763 2764 static inline struct drm_i915_private *to_i915(struct drm_device *dev) 2765 { 2766 return container_of(dev, struct drm_i915_private, drm); 2767 } 2768 2769 static inline struct drm_i915_private *kdev_to_i915(struct device *kdev) 2770 { 2771 return to_i915(dev_get_drvdata(kdev)); 2772 } 2773 2774 static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc) 2775 { 2776 return container_of(guc, struct drm_i915_private, guc); 2777 } 2778 2779 static inline struct drm_i915_private *huc_to_i915(struct intel_huc *huc) 2780 { 2781 return container_of(huc, struct drm_i915_private, huc); 2782 } 2783 2784 /* Simple iterator over all initialised engines */ 2785 #define for_each_engine(engine__, dev_priv__, id__) \ 2786 for ((id__) = 0; \ 2787 (id__) < I915_NUM_ENGINES; \ 2788 (id__)++) \ 2789 for_each_if ((engine__) = (dev_priv__)->engine[(id__)]) 2790 2791 /* Iterator over subset of engines selected by mask */ 2792 #define for_each_engine_masked(engine__, dev_priv__, mask__, tmp__) \ 2793 for (tmp__ = mask__ & INTEL_INFO(dev_priv__)->ring_mask; \ 2794 tmp__ ? (engine__ = (dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : 0; ) 2795 2796 enum hdmi_force_audio { 2797 HDMI_AUDIO_OFF_DVI = -2, /* no aux data for HDMI-DVI converter */ 2798 HDMI_AUDIO_OFF, /* force turn off HDMI audio */ 2799 HDMI_AUDIO_AUTO, /* trust EDID */ 2800 HDMI_AUDIO_ON, /* force turn on HDMI audio */ 2801 }; 2802 2803 #define I915_GTT_OFFSET_NONE ((u32)-1) 2804 2805 /* 2806 * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is 2807 * considered to be the frontbuffer for the given plane interface-wise. This 2808 * doesn't mean that the hw necessarily already scans it out, but that any 2809 * rendering (by the cpu or gpu) will land in the frontbuffer eventually. 2810 * 2811 * We have one bit per pipe and per scanout plane type. 2812 */ 2813 #define INTEL_MAX_SPRITE_BITS_PER_PIPE 5 2814 #define INTEL_FRONTBUFFER_BITS_PER_PIPE 8 2815 #define INTEL_FRONTBUFFER_PRIMARY(pipe) \ 2816 (1 << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))) 2817 #define INTEL_FRONTBUFFER_CURSOR(pipe) \ 2818 (1 << (1 + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))) 2819 #define INTEL_FRONTBUFFER_SPRITE(pipe, plane) \ 2820 (1 << (2 + plane + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))) 2821 #define INTEL_FRONTBUFFER_OVERLAY(pipe) \ 2822 (1 << (2 + INTEL_MAX_SPRITE_BITS_PER_PIPE + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))) 2823 #define INTEL_FRONTBUFFER_ALL_MASK(pipe) \ 2824 (0xff << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))) 2825 2826 /* 2827 * Optimised SGL iterator for GEM objects 2828 */ 2829 static __always_inline struct sgt_iter { 2830 struct scatterlist *sgp; 2831 union { 2832 unsigned long pfn; 2833 dma_addr_t dma; 2834 }; 2835 unsigned int curr; 2836 unsigned int max; 2837 } __sgt_iter(struct scatterlist *sgl, bool dma) { 2838 struct sgt_iter s = { .sgp = sgl }; 2839 2840 if (s.sgp) { 2841 s.max = s.curr = s.sgp->offset; 2842 s.max += s.sgp->length; 2843 if (dma) 2844 s.dma = sg_dma_address(s.sgp); 2845 else 2846 s.pfn = page_to_pfn(sg_page(s.sgp)); 2847 } 2848 2849 return s; 2850 } 2851 2852 static inline struct scatterlist *____sg_next(struct scatterlist *sg) 2853 { 2854 ++sg; 2855 if (unlikely(sg_is_chain(sg))) 2856 sg = sg_chain_ptr(sg); 2857 return sg; 2858 } 2859 2860 /** 2861 * __sg_next - return the next scatterlist entry in a list 2862 * @sg: The current sg entry 2863 * 2864 * Description: 2865 * If the entry is the last, return NULL; otherwise, step to the next 2866 * element in the array (@sg@+1). If that's a chain pointer, follow it; 2867 * otherwise just return the pointer to the current element. 2868 **/ 2869 static inline struct scatterlist *__sg_next(struct scatterlist *sg) 2870 { 2871 #ifdef CONFIG_DEBUG_SG 2872 BUG_ON(sg->sg_magic != SG_MAGIC); 2873 #endif 2874 return sg_is_last(sg) ? NULL : ____sg_next(sg); 2875 } 2876 2877 /** 2878 * for_each_sgt_dma - iterate over the DMA addresses of the given sg_table 2879 * @__dmap: DMA address (output) 2880 * @__iter: 'struct sgt_iter' (iterator state, internal) 2881 * @__sgt: sg_table to iterate over (input) 2882 */ 2883 #define for_each_sgt_dma(__dmap, __iter, __sgt) \ 2884 for ((__iter) = __sgt_iter((__sgt)->sgl, true); \ 2885 ((__dmap) = (__iter).dma + (__iter).curr); \ 2886 (((__iter).curr += PAGE_SIZE) >= (__iter).max) ? \ 2887 (__iter) = __sgt_iter(__sg_next((__iter).sgp), true), 0 : 0) 2888 2889 /** 2890 * for_each_sgt_page - iterate over the pages of the given sg_table 2891 * @__pp: page pointer (output) 2892 * @__iter: 'struct sgt_iter' (iterator state, internal) 2893 * @__sgt: sg_table to iterate over (input) 2894 */ 2895 #define for_each_sgt_page(__pp, __iter, __sgt) \ 2896 for ((__iter) = __sgt_iter((__sgt)->sgl, false); \ 2897 ((__pp) = (__iter).pfn == 0 ? NULL : \ 2898 pfn_to_page((__iter).pfn + ((__iter).curr >> PAGE_SHIFT))); \ 2899 (((__iter).curr += PAGE_SIZE) >= (__iter).max) ? \ 2900 (__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0 : 0) 2901 2902 static inline unsigned int i915_sg_page_sizes(struct scatterlist *sg) 2903 { 2904 unsigned int page_sizes; 2905 2906 page_sizes = 0; 2907 while (sg) { 2908 GEM_BUG_ON(sg->offset); 2909 GEM_BUG_ON(!IS_ALIGNED(sg->length, PAGE_SIZE)); 2910 page_sizes |= sg->length; 2911 sg = __sg_next(sg); 2912 } 2913 2914 return page_sizes; 2915 } 2916 2917 static inline unsigned int i915_sg_segment_size(void) 2918 { 2919 unsigned int size = swiotlb_max_segment(); 2920 2921 if (size == 0) 2922 return SCATTERLIST_MAX_SEGMENT; 2923 2924 size = rounddown(size, PAGE_SIZE); 2925 /* swiotlb_max_segment_size can return 1 byte when it means one page. */ 2926 if (size < PAGE_SIZE) 2927 size = PAGE_SIZE; 2928 2929 return size; 2930 } 2931 2932 static inline const struct intel_device_info * 2933 intel_info(const struct drm_i915_private *dev_priv) 2934 { 2935 return &dev_priv->info; 2936 } 2937 2938 #define INTEL_INFO(dev_priv) intel_info((dev_priv)) 2939 2940 #define INTEL_GEN(dev_priv) ((dev_priv)->info.gen) 2941 #define INTEL_DEVID(dev_priv) ((dev_priv)->info.device_id) 2942 2943 #define REVID_FOREVER 0xff 2944 #define INTEL_REVID(dev_priv) ((dev_priv)->drm.pdev->revision) 2945 2946 #define GEN_FOREVER (0) 2947 2948 #define INTEL_GEN_MASK(s, e) ( \ 2949 BUILD_BUG_ON_ZERO(!__builtin_constant_p(s)) + \ 2950 BUILD_BUG_ON_ZERO(!__builtin_constant_p(e)) + \ 2951 GENMASK((e) != GEN_FOREVER ? (e) - 1 : BITS_PER_LONG - 1, \ 2952 (s) != GEN_FOREVER ? (s) - 1 : 0) \ 2953 ) 2954 2955 /* 2956 * Returns true if Gen is in inclusive range [Start, End]. 2957 * 2958 * Use GEN_FOREVER for unbound start and or end. 2959 */ 2960 #define IS_GEN(dev_priv, s, e) \ 2961 (!!((dev_priv)->info.gen_mask & INTEL_GEN_MASK((s), (e)))) 2962 2963 /* 2964 * Return true if revision is in range [since,until] inclusive. 2965 * 2966 * Use 0 for open-ended since, and REVID_FOREVER for open-ended until. 2967 */ 2968 #define IS_REVID(p, since, until) \ 2969 (INTEL_REVID(p) >= (since) && INTEL_REVID(p) <= (until)) 2970 2971 #define IS_PLATFORM(dev_priv, p) ((dev_priv)->info.platform_mask & BIT(p)) 2972 2973 #define IS_I830(dev_priv) IS_PLATFORM(dev_priv, INTEL_I830) 2974 #define IS_I845G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I845G) 2975 #define IS_I85X(dev_priv) IS_PLATFORM(dev_priv, INTEL_I85X) 2976 #define IS_I865G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I865G) 2977 #define IS_I915G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I915G) 2978 #define IS_I915GM(dev_priv) IS_PLATFORM(dev_priv, INTEL_I915GM) 2979 #define IS_I945G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I945G) 2980 #define IS_I945GM(dev_priv) IS_PLATFORM(dev_priv, INTEL_I945GM) 2981 #define IS_I965G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I965G) 2982 #define IS_I965GM(dev_priv) IS_PLATFORM(dev_priv, INTEL_I965GM) 2983 #define IS_G45(dev_priv) IS_PLATFORM(dev_priv, INTEL_G45) 2984 #define IS_GM45(dev_priv) IS_PLATFORM(dev_priv, INTEL_GM45) 2985 #define IS_G4X(dev_priv) (IS_G45(dev_priv) || IS_GM45(dev_priv)) 2986 #define IS_PINEVIEW_G(dev_priv) (INTEL_DEVID(dev_priv) == 0xa001) 2987 #define IS_PINEVIEW_M(dev_priv) (INTEL_DEVID(dev_priv) == 0xa011) 2988 #define IS_PINEVIEW(dev_priv) IS_PLATFORM(dev_priv, INTEL_PINEVIEW) 2989 #define IS_G33(dev_priv) IS_PLATFORM(dev_priv, INTEL_G33) 2990 #define IS_IRONLAKE_M(dev_priv) (INTEL_DEVID(dev_priv) == 0x0046) 2991 #define IS_IVYBRIDGE(dev_priv) IS_PLATFORM(dev_priv, INTEL_IVYBRIDGE) 2992 #define IS_IVB_GT1(dev_priv) (IS_IVYBRIDGE(dev_priv) && \ 2993 (dev_priv)->info.gt == 1) 2994 #define IS_VALLEYVIEW(dev_priv) IS_PLATFORM(dev_priv, INTEL_VALLEYVIEW) 2995 #define IS_CHERRYVIEW(dev_priv) IS_PLATFORM(dev_priv, INTEL_CHERRYVIEW) 2996 #define IS_HASWELL(dev_priv) IS_PLATFORM(dev_priv, INTEL_HASWELL) 2997 #define IS_BROADWELL(dev_priv) IS_PLATFORM(dev_priv, INTEL_BROADWELL) 2998 #define IS_SKYLAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_SKYLAKE) 2999 #define IS_BROXTON(dev_priv) IS_PLATFORM(dev_priv, INTEL_BROXTON) 3000 #define IS_KABYLAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_KABYLAKE) 3001 #define IS_GEMINILAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_GEMINILAKE) 3002 #define IS_COFFEELAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_COFFEELAKE) 3003 #define IS_CANNONLAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_CANNONLAKE) 3004 #define IS_MOBILE(dev_priv) ((dev_priv)->info.is_mobile) 3005 #define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \ 3006 (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00) 3007 #define IS_BDW_ULT(dev_priv) (IS_BROADWELL(dev_priv) && \ 3008 ((INTEL_DEVID(dev_priv) & 0xf) == 0x6 || \ 3009 (INTEL_DEVID(dev_priv) & 0xf) == 0xb || \ 3010 (INTEL_DEVID(dev_priv) & 0xf) == 0xe)) 3011 /* ULX machines are also considered ULT. */ 3012 #define IS_BDW_ULX(dev_priv) (IS_BROADWELL(dev_priv) && \ 3013 (INTEL_DEVID(dev_priv) & 0xf) == 0xe) 3014 #define IS_BDW_GT3(dev_priv) (IS_BROADWELL(dev_priv) && \ 3015 (dev_priv)->info.gt == 3) 3016 #define IS_HSW_ULT(dev_priv) (IS_HASWELL(dev_priv) && \ 3017 (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0A00) 3018 #define IS_HSW_GT3(dev_priv) (IS_HASWELL(dev_priv) && \ 3019 (dev_priv)->info.gt == 3) 3020 /* ULX machines are also considered ULT. */ 3021 #define IS_HSW_ULX(dev_priv) (INTEL_DEVID(dev_priv) == 0x0A0E || \ 3022 INTEL_DEVID(dev_priv) == 0x0A1E) 3023 #define IS_SKL_ULT(dev_priv) (INTEL_DEVID(dev_priv) == 0x1906 || \ 3024 INTEL_DEVID(dev_priv) == 0x1913 || \ 3025 INTEL_DEVID(dev_priv) == 0x1916 || \ 3026 INTEL_DEVID(dev_priv) == 0x1921 || \ 3027 INTEL_DEVID(dev_priv) == 0x1926) 3028 #define IS_SKL_ULX(dev_priv) (INTEL_DEVID(dev_priv) == 0x190E || \ 3029 INTEL_DEVID(dev_priv) == 0x1915 || \ 3030 INTEL_DEVID(dev_priv) == 0x191E) 3031 #define IS_KBL_ULT(dev_priv) (INTEL_DEVID(dev_priv) == 0x5906 || \ 3032 INTEL_DEVID(dev_priv) == 0x5913 || \ 3033 INTEL_DEVID(dev_priv) == 0x5916 || \ 3034 INTEL_DEVID(dev_priv) == 0x5921 || \ 3035 INTEL_DEVID(dev_priv) == 0x5926) 3036 #define IS_KBL_ULX(dev_priv) (INTEL_DEVID(dev_priv) == 0x590E || \ 3037 INTEL_DEVID(dev_priv) == 0x5915 || \ 3038 INTEL_DEVID(dev_priv) == 0x591E) 3039 #define IS_SKL_GT2(dev_priv) (IS_SKYLAKE(dev_priv) && \ 3040 (dev_priv)->info.gt == 2) 3041 #define IS_SKL_GT3(dev_priv) (IS_SKYLAKE(dev_priv) && \ 3042 (dev_priv)->info.gt == 3) 3043 #define IS_SKL_GT4(dev_priv) (IS_SKYLAKE(dev_priv) && \ 3044 (dev_priv)->info.gt == 4) 3045 #define IS_KBL_GT2(dev_priv) (IS_KABYLAKE(dev_priv) && \ 3046 (dev_priv)->info.gt == 2) 3047 #define IS_KBL_GT3(dev_priv) (IS_KABYLAKE(dev_priv) && \ 3048 (dev_priv)->info.gt == 3) 3049 #define IS_CFL_ULT(dev_priv) (IS_COFFEELAKE(dev_priv) && \ 3050 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x00A0) 3051 #define IS_CFL_GT2(dev_priv) (IS_COFFEELAKE(dev_priv) && \ 3052 (dev_priv)->info.gt == 2) 3053 3054 #define IS_ALPHA_SUPPORT(intel_info) ((intel_info)->is_alpha_support) 3055 3056 #define SKL_REVID_A0 0x0 3057 #define SKL_REVID_B0 0x1 3058 #define SKL_REVID_C0 0x2 3059 #define SKL_REVID_D0 0x3 3060 #define SKL_REVID_E0 0x4 3061 #define SKL_REVID_F0 0x5 3062 #define SKL_REVID_G0 0x6 3063 #define SKL_REVID_H0 0x7 3064 3065 #define IS_SKL_REVID(p, since, until) (IS_SKYLAKE(p) && IS_REVID(p, since, until)) 3066 3067 #define BXT_REVID_A0 0x0 3068 #define BXT_REVID_A1 0x1 3069 #define BXT_REVID_B0 0x3 3070 #define BXT_REVID_B_LAST 0x8 3071 #define BXT_REVID_C0 0x9 3072 3073 #define IS_BXT_REVID(dev_priv, since, until) \ 3074 (IS_BROXTON(dev_priv) && IS_REVID(dev_priv, since, until)) 3075 3076 #define KBL_REVID_A0 0x0 3077 #define KBL_REVID_B0 0x1 3078 #define KBL_REVID_C0 0x2 3079 #define KBL_REVID_D0 0x3 3080 #define KBL_REVID_E0 0x4 3081 3082 #define IS_KBL_REVID(dev_priv, since, until) \ 3083 (IS_KABYLAKE(dev_priv) && IS_REVID(dev_priv, since, until)) 3084 3085 #define GLK_REVID_A0 0x0 3086 #define GLK_REVID_A1 0x1 3087 3088 #define IS_GLK_REVID(dev_priv, since, until) \ 3089 (IS_GEMINILAKE(dev_priv) && IS_REVID(dev_priv, since, until)) 3090 3091 #define CNL_REVID_A0 0x0 3092 #define CNL_REVID_B0 0x1 3093 #define CNL_REVID_C0 0x2 3094 3095 #define IS_CNL_REVID(p, since, until) \ 3096 (IS_CANNONLAKE(p) && IS_REVID(p, since, until)) 3097 3098 /* 3099 * The genX designation typically refers to the render engine, so render 3100 * capability related checks should use IS_GEN, while display and other checks 3101 * have their own (e.g. HAS_PCH_SPLIT for ILK+ display, IS_foo for particular 3102 * chips, etc.). 3103 */ 3104 #define IS_GEN2(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(1))) 3105 #define IS_GEN3(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(2))) 3106 #define IS_GEN4(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(3))) 3107 #define IS_GEN5(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(4))) 3108 #define IS_GEN6(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(5))) 3109 #define IS_GEN7(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(6))) 3110 #define IS_GEN8(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(7))) 3111 #define IS_GEN9(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(8))) 3112 #define IS_GEN10(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(9))) 3113 3114 #define IS_LP(dev_priv) (INTEL_INFO(dev_priv)->is_lp) 3115 #define IS_GEN9_LP(dev_priv) (IS_GEN9(dev_priv) && IS_LP(dev_priv)) 3116 #define IS_GEN9_BC(dev_priv) (IS_GEN9(dev_priv) && !IS_LP(dev_priv)) 3117 3118 #define ENGINE_MASK(id) BIT(id) 3119 #define RENDER_RING ENGINE_MASK(RCS) 3120 #define BSD_RING ENGINE_MASK(VCS) 3121 #define BLT_RING ENGINE_MASK(BCS) 3122 #define VEBOX_RING ENGINE_MASK(VECS) 3123 #define BSD2_RING ENGINE_MASK(VCS2) 3124 #define ALL_ENGINES (~0) 3125 3126 #define HAS_ENGINE(dev_priv, id) \ 3127 (!!((dev_priv)->info.ring_mask & ENGINE_MASK(id))) 3128 3129 #define HAS_BSD(dev_priv) HAS_ENGINE(dev_priv, VCS) 3130 #define HAS_BSD2(dev_priv) HAS_ENGINE(dev_priv, VCS2) 3131 #define HAS_BLT(dev_priv) HAS_ENGINE(dev_priv, BCS) 3132 #define HAS_VEBOX(dev_priv) HAS_ENGINE(dev_priv, VECS) 3133 3134 #define HAS_LLC(dev_priv) ((dev_priv)->info.has_llc) 3135 #define HAS_SNOOP(dev_priv) ((dev_priv)->info.has_snoop) 3136 #define HAS_EDRAM(dev_priv) (!!((dev_priv)->edram_cap & EDRAM_ENABLED)) 3137 #define HAS_WT(dev_priv) ((IS_HASWELL(dev_priv) || \ 3138 IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv)) 3139 3140 #define HWS_NEEDS_PHYSICAL(dev_priv) ((dev_priv)->info.hws_needs_physical) 3141 3142 #define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \ 3143 ((dev_priv)->info.has_logical_ring_contexts) 3144 #define USES_PPGTT(dev_priv) (i915_modparams.enable_ppgtt) 3145 #define USES_FULL_PPGTT(dev_priv) (i915_modparams.enable_ppgtt >= 2) 3146 #define USES_FULL_48BIT_PPGTT(dev_priv) (i915_modparams.enable_ppgtt == 3) 3147 #define HAS_PAGE_SIZES(dev_priv, sizes) ({ \ 3148 GEM_BUG_ON((sizes) == 0); \ 3149 ((sizes) & ~(dev_priv)->info.page_sizes) == 0; \ 3150 }) 3151 3152 #define HAS_OVERLAY(dev_priv) ((dev_priv)->info.has_overlay) 3153 #define OVERLAY_NEEDS_PHYSICAL(dev_priv) \ 3154 ((dev_priv)->info.overlay_needs_physical) 3155 3156 /* Early gen2 have a totally busted CS tlb and require pinned batches. */ 3157 #define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_I845G(dev_priv)) 3158 3159 /* WaRsDisableCoarsePowerGating:skl,bxt */ 3160 #define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \ 3161 (IS_SKL_GT3(dev_priv) || IS_SKL_GT4(dev_priv)) 3162 3163 /* 3164 * dp aux and gmbus irq on gen4 seems to be able to generate legacy interrupts 3165 * even when in MSI mode. This results in spurious interrupt warnings if the 3166 * legacy irq no. is shared with another device. The kernel then disables that 3167 * interrupt source and so prevents the other device from working properly. 3168 * 3169 * Since we don't enable MSI anymore on gen4, we can always use GMBUS/AUX 3170 * interrupts. 3171 */ 3172 #define HAS_AUX_IRQ(dev_priv) true 3173 #define HAS_GMBUS_IRQ(dev_priv) (INTEL_GEN(dev_priv) >= 4) 3174 3175 /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte 3176 * rows, which changed the alignment requirements and fence programming. 3177 */ 3178 #define HAS_128_BYTE_Y_TILING(dev_priv) (!IS_GEN2(dev_priv) && \ 3179 !(IS_I915G(dev_priv) || \ 3180 IS_I915GM(dev_priv))) 3181 #define SUPPORTS_TV(dev_priv) ((dev_priv)->info.supports_tv) 3182 #define I915_HAS_HOTPLUG(dev_priv) ((dev_priv)->info.has_hotplug) 3183 3184 #define HAS_FW_BLC(dev_priv) (INTEL_GEN(dev_priv) > 2) 3185 #define HAS_FBC(dev_priv) ((dev_priv)->info.has_fbc) 3186 #define HAS_CUR_FBC(dev_priv) (!HAS_GMCH_DISPLAY(dev_priv) && INTEL_INFO(dev_priv)->gen >= 7) 3187 3188 #define HAS_IPS(dev_priv) (IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv)) 3189 3190 #define HAS_DP_MST(dev_priv) ((dev_priv)->info.has_dp_mst) 3191 3192 #define HAS_DDI(dev_priv) ((dev_priv)->info.has_ddi) 3193 #define HAS_FPGA_DBG_UNCLAIMED(dev_priv) ((dev_priv)->info.has_fpga_dbg) 3194 #define HAS_PSR(dev_priv) ((dev_priv)->info.has_psr) 3195 #define HAS_RC6(dev_priv) ((dev_priv)->info.has_rc6) 3196 #define HAS_RC6p(dev_priv) ((dev_priv)->info.has_rc6p) 3197 3198 #define HAS_CSR(dev_priv) ((dev_priv)->info.has_csr) 3199 3200 #define HAS_RUNTIME_PM(dev_priv) ((dev_priv)->info.has_runtime_pm) 3201 #define HAS_64BIT_RELOC(dev_priv) ((dev_priv)->info.has_64bit_reloc) 3202 3203 #define HAS_IPC(dev_priv) ((dev_priv)->info.has_ipc) 3204 3205 /* 3206 * For now, anything with a GuC requires uCode loading, and then supports 3207 * command submission once loaded. But these are logically independent 3208 * properties, so we have separate macros to test them. 3209 */ 3210 #define HAS_GUC(dev_priv) ((dev_priv)->info.has_guc) 3211 #define HAS_GUC_CT(dev_priv) ((dev_priv)->info.has_guc_ct) 3212 #define HAS_GUC_UCODE(dev_priv) (HAS_GUC(dev_priv)) 3213 #define HAS_GUC_SCHED(dev_priv) (HAS_GUC(dev_priv)) 3214 #define HAS_HUC_UCODE(dev_priv) (HAS_GUC(dev_priv)) 3215 3216 #define HAS_RESOURCE_STREAMER(dev_priv) ((dev_priv)->info.has_resource_streamer) 3217 3218 #define HAS_POOLED_EU(dev_priv) ((dev_priv)->info.has_pooled_eu) 3219 3220 #define INTEL_PCH_DEVICE_ID_MASK 0xff80 3221 #define INTEL_PCH_IBX_DEVICE_ID_TYPE 0x3b00 3222 #define INTEL_PCH_CPT_DEVICE_ID_TYPE 0x1c00 3223 #define INTEL_PCH_PPT_DEVICE_ID_TYPE 0x1e00 3224 #define INTEL_PCH_LPT_DEVICE_ID_TYPE 0x8c00 3225 #define INTEL_PCH_LPT_LP_DEVICE_ID_TYPE 0x9c00 3226 #define INTEL_PCH_WPT_DEVICE_ID_TYPE 0x8c80 3227 #define INTEL_PCH_WPT_LP_DEVICE_ID_TYPE 0x9c80 3228 #define INTEL_PCH_SPT_DEVICE_ID_TYPE 0xA100 3229 #define INTEL_PCH_SPT_LP_DEVICE_ID_TYPE 0x9D00 3230 #define INTEL_PCH_KBP_DEVICE_ID_TYPE 0xA280 3231 #define INTEL_PCH_CNP_DEVICE_ID_TYPE 0xA300 3232 #define INTEL_PCH_CNP_LP_DEVICE_ID_TYPE 0x9D80 3233 #define INTEL_PCH_P2X_DEVICE_ID_TYPE 0x7100 3234 #define INTEL_PCH_P3X_DEVICE_ID_TYPE 0x7000 3235 #define INTEL_PCH_QEMU_DEVICE_ID_TYPE 0x2900 /* qemu q35 has 2918 */ 3236 3237 #define INTEL_PCH_TYPE(dev_priv) ((dev_priv)->pch_type) 3238 #define HAS_PCH_CNP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CNP) 3239 #define HAS_PCH_CNP_LP(dev_priv) \ 3240 ((dev_priv)->pch_id == INTEL_PCH_CNP_LP_DEVICE_ID_TYPE) 3241 #define HAS_PCH_KBP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_KBP) 3242 #define HAS_PCH_SPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_SPT) 3243 #define HAS_PCH_LPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_LPT) 3244 #define HAS_PCH_LPT_LP(dev_priv) \ 3245 ((dev_priv)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE || \ 3246 (dev_priv)->pch_id == INTEL_PCH_WPT_LP_DEVICE_ID_TYPE) 3247 #define HAS_PCH_LPT_H(dev_priv) \ 3248 ((dev_priv)->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE || \ 3249 (dev_priv)->pch_id == INTEL_PCH_WPT_DEVICE_ID_TYPE) 3250 #define HAS_PCH_CPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CPT) 3251 #define HAS_PCH_IBX(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_IBX) 3252 #define HAS_PCH_NOP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_NOP) 3253 #define HAS_PCH_SPLIT(dev_priv) (INTEL_PCH_TYPE(dev_priv) != PCH_NONE) 3254 3255 #define HAS_GMCH_DISPLAY(dev_priv) ((dev_priv)->info.has_gmch_display) 3256 3257 #define HAS_LSPCON(dev_priv) (INTEL_GEN(dev_priv) >= 9) 3258 3259 /* DPF == dynamic parity feature */ 3260 #define HAS_L3_DPF(dev_priv) ((dev_priv)->info.has_l3_dpf) 3261 #define NUM_L3_SLICES(dev_priv) (IS_HSW_GT3(dev_priv) ? \ 3262 2 : HAS_L3_DPF(dev_priv)) 3263 3264 #define GT_FREQUENCY_MULTIPLIER 50 3265 #define GEN9_FREQ_SCALER 3 3266 3267 #include "i915_trace.h" 3268 3269 static inline bool intel_vtd_active(void) 3270 { 3271 #ifdef CONFIG_INTEL_IOMMU 3272 if (intel_iommu_gfx_mapped) 3273 return true; 3274 #endif 3275 return false; 3276 } 3277 3278 static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv) 3279 { 3280 return INTEL_GEN(dev_priv) >= 6 && intel_vtd_active(); 3281 } 3282 3283 static inline bool 3284 intel_ggtt_update_needs_vtd_wa(struct drm_i915_private *dev_priv) 3285 { 3286 return IS_BROXTON(dev_priv) && intel_vtd_active(); 3287 } 3288 3289 int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv, 3290 int enable_ppgtt); 3291 3292 bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value); 3293 3294 /* i915_drv.c */ 3295 void __printf(3, 4) 3296 __i915_printk(struct drm_i915_private *dev_priv, const char *level, 3297 const char *fmt, ...); 3298 3299 #define i915_report_error(dev_priv, fmt, ...) \ 3300 __i915_printk(dev_priv, KERN_ERR, fmt, ##__VA_ARGS__) 3301 3302 #ifdef CONFIG_COMPAT 3303 extern long i915_compat_ioctl(struct file *filp, unsigned int cmd, 3304 unsigned long arg); 3305 #else 3306 #define i915_compat_ioctl NULL 3307 #endif 3308 extern const struct dev_pm_ops i915_pm_ops; 3309 3310 extern int i915_driver_load(struct pci_dev *pdev, 3311 const struct pci_device_id *ent); 3312 extern void i915_driver_unload(struct drm_device *dev); 3313 extern int intel_gpu_reset(struct drm_i915_private *dev_priv, u32 engine_mask); 3314 extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv); 3315 3316 #define I915_RESET_QUIET BIT(0) 3317 extern void i915_reset(struct drm_i915_private *i915, unsigned int flags); 3318 extern int i915_reset_engine(struct intel_engine_cs *engine, 3319 unsigned int flags); 3320 3321 extern bool intel_has_reset_engine(struct drm_i915_private *dev_priv); 3322 extern int intel_guc_reset(struct drm_i915_private *dev_priv); 3323 extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine); 3324 extern void intel_hangcheck_init(struct drm_i915_private *dev_priv); 3325 extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); 3326 extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv); 3327 extern unsigned long i915_gfx_val(struct drm_i915_private *dev_priv); 3328 extern void i915_update_gfx_val(struct drm_i915_private *dev_priv); 3329 int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on); 3330 3331 int intel_engines_init_mmio(struct drm_i915_private *dev_priv); 3332 int intel_engines_init(struct drm_i915_private *dev_priv); 3333 3334 /* intel_hotplug.c */ 3335 void intel_hpd_irq_handler(struct drm_i915_private *dev_priv, 3336 u32 pin_mask, u32 long_mask); 3337 void intel_hpd_init(struct drm_i915_private *dev_priv); 3338 void intel_hpd_init_work(struct drm_i915_private *dev_priv); 3339 void intel_hpd_cancel_work(struct drm_i915_private *dev_priv); 3340 enum port intel_hpd_pin_to_port(enum hpd_pin pin); 3341 enum hpd_pin intel_hpd_pin(enum port port); 3342 bool intel_hpd_disable(struct drm_i915_private *dev_priv, enum hpd_pin pin); 3343 void intel_hpd_enable(struct drm_i915_private *dev_priv, enum hpd_pin pin); 3344 3345 /* i915_irq.c */ 3346 static inline void i915_queue_hangcheck(struct drm_i915_private *dev_priv) 3347 { 3348 unsigned long delay; 3349 3350 if (unlikely(!i915_modparams.enable_hangcheck)) 3351 return; 3352 3353 /* Don't continually defer the hangcheck so that it is always run at 3354 * least once after work has been scheduled on any ring. Otherwise, 3355 * we will ignore a hung ring if a second ring is kept busy. 3356 */ 3357 3358 delay = round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES); 3359 queue_delayed_work(system_long_wq, 3360 &dev_priv->gpu_error.hangcheck_work, delay); 3361 } 3362 3363 __printf(3, 4) 3364 void i915_handle_error(struct drm_i915_private *dev_priv, 3365 u32 engine_mask, 3366 const char *fmt, ...); 3367 3368 extern void intel_irq_init(struct drm_i915_private *dev_priv); 3369 extern void intel_irq_fini(struct drm_i915_private *dev_priv); 3370 int intel_irq_install(struct drm_i915_private *dev_priv); 3371 void intel_irq_uninstall(struct drm_i915_private *dev_priv); 3372 3373 static inline bool intel_gvt_active(struct drm_i915_private *dev_priv) 3374 { 3375 return dev_priv->gvt; 3376 } 3377 3378 static inline bool intel_vgpu_active(struct drm_i915_private *dev_priv) 3379 { 3380 return dev_priv->vgpu.active; 3381 } 3382 3383 u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv, 3384 enum i915_pipe pipe); 3385 void 3386 i915_enable_pipestat(struct drm_i915_private *dev_priv, enum i915_pipe pipe, 3387 u32 status_mask); 3388 3389 void 3390 i915_disable_pipestat(struct drm_i915_private *dev_priv, enum i915_pipe pipe, 3391 u32 status_mask); 3392 3393 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv); 3394 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv); 3395 void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv, 3396 uint32_t mask, 3397 uint32_t bits); 3398 void ilk_update_display_irq(struct drm_i915_private *dev_priv, 3399 uint32_t interrupt_mask, 3400 uint32_t enabled_irq_mask); 3401 static inline void 3402 ilk_enable_display_irq(struct drm_i915_private *dev_priv, uint32_t bits) 3403 { 3404 ilk_update_display_irq(dev_priv, bits, bits); 3405 } 3406 static inline void 3407 ilk_disable_display_irq(struct drm_i915_private *dev_priv, uint32_t bits) 3408 { 3409 ilk_update_display_irq(dev_priv, bits, 0); 3410 } 3411 void bdw_update_pipe_irq(struct drm_i915_private *dev_priv, 3412 enum i915_pipe pipe, 3413 uint32_t interrupt_mask, 3414 uint32_t enabled_irq_mask); 3415 static inline void bdw_enable_pipe_irq(struct drm_i915_private *dev_priv, 3416 enum i915_pipe pipe, uint32_t bits) 3417 { 3418 bdw_update_pipe_irq(dev_priv, pipe, bits, bits); 3419 } 3420 static inline void bdw_disable_pipe_irq(struct drm_i915_private *dev_priv, 3421 enum i915_pipe pipe, uint32_t bits) 3422 { 3423 bdw_update_pipe_irq(dev_priv, pipe, bits, 0); 3424 } 3425 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv, 3426 uint32_t interrupt_mask, 3427 uint32_t enabled_irq_mask); 3428 static inline void 3429 ibx_enable_display_interrupt(struct drm_i915_private *dev_priv, uint32_t bits) 3430 { 3431 ibx_display_interrupt_update(dev_priv, bits, bits); 3432 } 3433 static inline void 3434 ibx_disable_display_interrupt(struct drm_i915_private *dev_priv, uint32_t bits) 3435 { 3436 ibx_display_interrupt_update(dev_priv, bits, 0); 3437 } 3438 3439 /* i915_gem.c */ 3440 int i915_gem_create_ioctl(struct drm_device *dev, void *data, 3441 struct drm_file *file_priv); 3442 int i915_gem_pread_ioctl(struct drm_device *dev, void *data, 3443 struct drm_file *file_priv); 3444 int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 3445 struct drm_file *file_priv); 3446 int i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 3447 struct drm_file *file_priv); 3448 int i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 3449 struct drm_file *file_priv); 3450 int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 3451 struct drm_file *file_priv); 3452 int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 3453 struct drm_file *file_priv); 3454 int i915_gem_execbuffer(struct drm_device *dev, void *data, 3455 struct drm_file *file_priv); 3456 int i915_gem_execbuffer2(struct drm_device *dev, void *data, 3457 struct drm_file *file_priv); 3458 int i915_gem_busy_ioctl(struct drm_device *dev, void *data, 3459 struct drm_file *file_priv); 3460 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, 3461 struct drm_file *file); 3462 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, 3463 struct drm_file *file); 3464 int i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 3465 struct drm_file *file_priv); 3466 int i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 3467 struct drm_file *file_priv); 3468 int i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data, 3469 struct drm_file *file_priv); 3470 int i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data, 3471 struct drm_file *file_priv); 3472 int i915_gem_init_userptr(struct drm_i915_private *dev_priv); 3473 void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv); 3474 int i915_gem_userptr_ioctl(struct drm_device *dev, void *data, 3475 struct drm_file *file); 3476 int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 3477 struct drm_file *file_priv); 3478 int i915_gem_wait_ioctl(struct drm_device *dev, void *data, 3479 struct drm_file *file_priv); 3480 void i915_gem_sanitize(struct drm_i915_private *i915); 3481 int i915_gem_load_init(struct drm_i915_private *dev_priv); 3482 void i915_gem_load_cleanup(struct drm_i915_private *dev_priv); 3483 void i915_gem_load_init_fences(struct drm_i915_private *dev_priv); 3484 int i915_gem_freeze(struct drm_i915_private *dev_priv); 3485 int i915_gem_freeze_late(struct drm_i915_private *dev_priv); 3486 3487 void *i915_gem_object_alloc(struct drm_i915_private *dev_priv); 3488 void i915_gem_object_free(struct drm_i915_gem_object *obj); 3489 void i915_gem_object_init(struct drm_i915_gem_object *obj, 3490 const struct drm_i915_gem_object_ops *ops); 3491 struct drm_i915_gem_object * 3492 i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size); 3493 struct drm_i915_gem_object * 3494 i915_gem_object_create_from_data(struct drm_i915_private *dev_priv, 3495 const void *data, size_t size); 3496 void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file); 3497 void i915_gem_free_object(struct drm_gem_object *obj); 3498 3499 static inline void i915_gem_drain_freed_objects(struct drm_i915_private *i915) 3500 { 3501 /* A single pass should suffice to release all the freed objects (along 3502 * most call paths) , but be a little more paranoid in that freeing 3503 * the objects does take a little amount of time, during which the rcu 3504 * callbacks could have added new objects into the freed list, and 3505 * armed the work again. 3506 */ 3507 do { 3508 rcu_barrier(); 3509 } while (flush_work(&i915->mm.free_work)); 3510 } 3511 3512 static inline void i915_gem_drain_workqueue(struct drm_i915_private *i915) 3513 { 3514 /* 3515 * Similar to objects above (see i915_gem_drain_freed-objects), in 3516 * general we have workers that are armed by RCU and then rearm 3517 * themselves in their callbacks. To be paranoid, we need to 3518 * drain the workqueue a second time after waiting for the RCU 3519 * grace period so that we catch work queued via RCU from the first 3520 * pass. As neither drain_workqueue() nor flush_workqueue() report 3521 * a result, we make an assumption that we only don't require more 3522 * than 2 passes to catch all recursive RCU delayed work. 3523 * 3524 */ 3525 int pass = 2; 3526 do { 3527 rcu_barrier(); 3528 drain_workqueue(i915->wq); 3529 } while (--pass); 3530 } 3531 3532 struct i915_vma * __must_check 3533 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, 3534 const struct i915_ggtt_view *view, 3535 u64 size, 3536 u64 alignment, 3537 u64 flags); 3538 3539 int i915_gem_object_unbind(struct drm_i915_gem_object *obj); 3540 void i915_gem_release_mmap(struct drm_i915_gem_object *obj); 3541 3542 void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv); 3543 3544 static inline int __sg_page_count(const struct scatterlist *sg) 3545 { 3546 return sg->length >> PAGE_SHIFT; 3547 } 3548 3549 struct scatterlist * 3550 i915_gem_object_get_sg(struct drm_i915_gem_object *obj, 3551 unsigned int n, unsigned int *offset); 3552 3553 struct page * 3554 i915_gem_object_get_page(struct drm_i915_gem_object *obj, 3555 unsigned int n); 3556 3557 struct page * 3558 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, 3559 unsigned int n); 3560 3561 dma_addr_t 3562 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, 3563 unsigned long n); 3564 3565 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, 3566 struct sg_table *pages, 3567 unsigned int sg_page_sizes); 3568 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj); 3569 3570 static inline int __must_check 3571 i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 3572 { 3573 might_lock(&obj->mm.lock); 3574 3575 if (atomic_inc_not_zero(&obj->mm.pages_pin_count)) 3576 return 0; 3577 3578 return __i915_gem_object_get_pages(obj); 3579 } 3580 3581 static inline bool 3582 i915_gem_object_has_pages(struct drm_i915_gem_object *obj) 3583 { 3584 return !IS_ERR_OR_NULL(READ_ONCE(obj->mm.pages)); 3585 } 3586 3587 static inline void 3588 __i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 3589 { 3590 GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 3591 3592 atomic_inc(&obj->mm.pages_pin_count); 3593 } 3594 3595 static inline bool 3596 i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj) 3597 { 3598 return atomic_read(&obj->mm.pages_pin_count); 3599 } 3600 3601 static inline void 3602 __i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 3603 { 3604 GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 3605 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 3606 3607 atomic_dec(&obj->mm.pages_pin_count); 3608 } 3609 3610 static inline void 3611 i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 3612 { 3613 __i915_gem_object_unpin_pages(obj); 3614 } 3615 3616 enum i915_mm_subclass { /* lockdep subclass for obj->mm.lock */ 3617 I915_MM_NORMAL = 0, 3618 I915_MM_SHRINKER 3619 }; 3620 3621 void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, 3622 enum i915_mm_subclass subclass); 3623 void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj); 3624 3625 enum i915_map_type { 3626 I915_MAP_WB = 0, 3627 I915_MAP_WC, 3628 #define I915_MAP_OVERRIDE BIT(31) 3629 I915_MAP_FORCE_WB = I915_MAP_WB | I915_MAP_OVERRIDE, 3630 I915_MAP_FORCE_WC = I915_MAP_WC | I915_MAP_OVERRIDE, 3631 }; 3632 3633 /** 3634 * i915_gem_object_pin_map - return a contiguous mapping of the entire object 3635 * @obj: the object to map into kernel address space 3636 * @type: the type of mapping, used to select pgprot_t 3637 * 3638 * Calls i915_gem_object_pin_pages() to prevent reaping of the object's 3639 * pages and then returns a contiguous mapping of the backing storage into 3640 * the kernel address space. Based on the @type of mapping, the PTE will be 3641 * set to either WriteBack or WriteCombine (via pgprot_t). 3642 * 3643 * The caller is responsible for calling i915_gem_object_unpin_map() when the 3644 * mapping is no longer required. 3645 * 3646 * Returns the pointer through which to access the mapped object, or an 3647 * ERR_PTR() on error. 3648 */ 3649 void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj, 3650 enum i915_map_type type); 3651 3652 /** 3653 * i915_gem_object_unpin_map - releases an earlier mapping 3654 * @obj: the object to unmap 3655 * 3656 * After pinning the object and mapping its pages, once you are finished 3657 * with your access, call i915_gem_object_unpin_map() to release the pin 3658 * upon the mapping. Once the pin count reaches zero, that mapping may be 3659 * removed. 3660 */ 3661 static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj) 3662 { 3663 i915_gem_object_unpin_pages(obj); 3664 } 3665 3666 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, 3667 unsigned int *needs_clflush); 3668 int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, 3669 unsigned int *needs_clflush); 3670 #define CLFLUSH_BEFORE BIT(0) 3671 #define CLFLUSH_AFTER BIT(1) 3672 #define CLFLUSH_FLAGS (CLFLUSH_BEFORE | CLFLUSH_AFTER) 3673 3674 static inline void 3675 i915_gem_obj_finish_shmem_access(struct drm_i915_gem_object *obj) 3676 { 3677 i915_gem_object_unpin_pages(obj); 3678 } 3679 3680 int __must_check i915_mutex_lock_interruptible(struct drm_device *dev); 3681 void i915_vma_move_to_active(struct i915_vma *vma, 3682 struct drm_i915_gem_request *req, 3683 unsigned int flags); 3684 int i915_gem_dumb_create(struct drm_file *file_priv, 3685 struct drm_device *dev, 3686 struct drm_mode_create_dumb *args); 3687 int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev, 3688 uint32_t handle, uint64_t *offset); 3689 int i915_gem_mmap_gtt_version(void); 3690 3691 void i915_gem_track_fb(struct drm_i915_gem_object *old, 3692 struct drm_i915_gem_object *new, 3693 unsigned frontbuffer_bits); 3694 3695 int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno); 3696 3697 struct drm_i915_gem_request * 3698 i915_gem_find_active_request(struct intel_engine_cs *engine); 3699 3700 void i915_gem_retire_requests(struct drm_i915_private *dev_priv); 3701 3702 static inline bool i915_reset_backoff(struct i915_gpu_error *error) 3703 { 3704 return unlikely(test_bit(I915_RESET_BACKOFF, &error->flags)); 3705 } 3706 3707 static inline bool i915_reset_handoff(struct i915_gpu_error *error) 3708 { 3709 return unlikely(test_bit(I915_RESET_HANDOFF, &error->flags)); 3710 } 3711 3712 static inline bool i915_terminally_wedged(struct i915_gpu_error *error) 3713 { 3714 return unlikely(test_bit(I915_WEDGED, &error->flags)); 3715 } 3716 3717 static inline bool i915_reset_backoff_or_wedged(struct i915_gpu_error *error) 3718 { 3719 return i915_reset_backoff(error) | i915_terminally_wedged(error); 3720 } 3721 3722 static inline u32 i915_reset_count(struct i915_gpu_error *error) 3723 { 3724 return READ_ONCE(error->reset_count); 3725 } 3726 3727 static inline u32 i915_reset_engine_count(struct i915_gpu_error *error, 3728 struct intel_engine_cs *engine) 3729 { 3730 return READ_ONCE(error->reset_engine_count[engine->id]); 3731 } 3732 3733 struct drm_i915_gem_request * 3734 i915_gem_reset_prepare_engine(struct intel_engine_cs *engine); 3735 int i915_gem_reset_prepare(struct drm_i915_private *dev_priv); 3736 void i915_gem_reset(struct drm_i915_private *dev_priv); 3737 void i915_gem_reset_finish_engine(struct intel_engine_cs *engine); 3738 void i915_gem_reset_finish(struct drm_i915_private *dev_priv); 3739 void i915_gem_set_wedged(struct drm_i915_private *dev_priv); 3740 bool i915_gem_unset_wedged(struct drm_i915_private *dev_priv); 3741 void i915_gem_reset_engine(struct intel_engine_cs *engine, 3742 struct drm_i915_gem_request *request); 3743 3744 void i915_gem_init_mmio(struct drm_i915_private *i915); 3745 int __must_check i915_gem_init(struct drm_i915_private *dev_priv); 3746 int __must_check i915_gem_init_hw(struct drm_i915_private *dev_priv); 3747 void i915_gem_init_swizzling(struct drm_i915_private *dev_priv); 3748 void i915_gem_cleanup_engines(struct drm_i915_private *dev_priv); 3749 int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv, 3750 unsigned int flags); 3751 int __must_check i915_gem_suspend(struct drm_i915_private *dev_priv); 3752 void i915_gem_resume(struct drm_i915_private *dev_priv); 3753 int i915_gem_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, vm_page_t *mres); 3754 int i915_gem_object_wait(struct drm_i915_gem_object *obj, 3755 unsigned int flags, 3756 long timeout, 3757 struct intel_rps_client *rps); 3758 int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, 3759 unsigned int flags, 3760 int priority); 3761 #define I915_PRIORITY_DISPLAY I915_PRIORITY_MAX 3762 3763 int __must_check 3764 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write); 3765 int __must_check 3766 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write); 3767 int __must_check 3768 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write); 3769 struct i915_vma * __must_check 3770 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 3771 u32 alignment, 3772 const struct i915_ggtt_view *view); 3773 void i915_gem_object_unpin_from_display_plane(struct i915_vma *vma); 3774 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, 3775 int align); 3776 int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file); 3777 void i915_gem_release(struct drm_device *dev, struct drm_file *file); 3778 3779 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 3780 enum i915_cache_level cache_level); 3781 3782 struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, 3783 struct dma_buf *dma_buf); 3784 3785 struct dma_buf *i915_gem_prime_export(struct drm_device *dev, 3786 struct drm_gem_object *gem_obj, int flags); 3787 3788 static inline struct i915_hw_ppgtt * 3789 i915_vm_to_ppgtt(struct i915_address_space *vm) 3790 { 3791 return container_of(vm, struct i915_hw_ppgtt, base); 3792 } 3793 3794 /* i915_gem_fence_reg.c */ 3795 struct drm_i915_fence_reg * 3796 i915_reserve_fence(struct drm_i915_private *dev_priv); 3797 void i915_unreserve_fence(struct drm_i915_fence_reg *fence); 3798 3799 void i915_gem_revoke_fences(struct drm_i915_private *dev_priv); 3800 void i915_gem_restore_fences(struct drm_i915_private *dev_priv); 3801 3802 void i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv); 3803 void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj, 3804 struct sg_table *pages); 3805 void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj, 3806 struct sg_table *pages); 3807 3808 static inline struct i915_gem_context * 3809 __i915_gem_context_lookup_rcu(struct drm_i915_file_private *file_priv, u32 id) 3810 { 3811 return idr_find(&file_priv->context_idr, id); 3812 } 3813 3814 static inline struct i915_gem_context * 3815 i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id) 3816 { 3817 struct i915_gem_context *ctx; 3818 3819 rcu_read_lock(); 3820 ctx = __i915_gem_context_lookup_rcu(file_priv, id); 3821 if (ctx && !kref_get_unless_zero(&ctx->ref)) 3822 ctx = NULL; 3823 rcu_read_unlock(); 3824 3825 return ctx; 3826 } 3827 3828 static inline struct intel_timeline * 3829 i915_gem_context_lookup_timeline(struct i915_gem_context *ctx, 3830 struct intel_engine_cs *engine) 3831 { 3832 struct i915_address_space *vm; 3833 3834 vm = ctx->ppgtt ? &ctx->ppgtt->base : &ctx->i915->ggtt.base; 3835 return &vm->timeline.engine[engine->id]; 3836 } 3837 3838 int i915_perf_open_ioctl(struct drm_device *dev, void *data, 3839 struct drm_file *file); 3840 int i915_perf_add_config_ioctl(struct drm_device *dev, void *data, 3841 struct drm_file *file); 3842 int i915_perf_remove_config_ioctl(struct drm_device *dev, void *data, 3843 struct drm_file *file); 3844 void i915_oa_init_reg_state(struct intel_engine_cs *engine, 3845 struct i915_gem_context *ctx, 3846 uint32_t *reg_state); 3847 3848 /* i915_gem_evict.c */ 3849 int __must_check i915_gem_evict_something(struct i915_address_space *vm, 3850 u64 min_size, u64 alignment, 3851 unsigned cache_level, 3852 u64 start, u64 end, 3853 unsigned flags); 3854 int __must_check i915_gem_evict_for_node(struct i915_address_space *vm, 3855 struct drm_mm_node *node, 3856 unsigned int flags); 3857 int i915_gem_evict_vm(struct i915_address_space *vm); 3858 3859 /* belongs in i915_gem_gtt.h */ 3860 static inline void i915_gem_chipset_flush(struct drm_i915_private *dev_priv) 3861 { 3862 wmb(); 3863 if (INTEL_GEN(dev_priv) < 6) 3864 intel_gtt_chipset_flush(); 3865 } 3866 3867 /* i915_gem_stolen.c */ 3868 int i915_gem_stolen_insert_node(struct drm_i915_private *dev_priv, 3869 struct drm_mm_node *node, u64 size, 3870 unsigned alignment); 3871 int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv, 3872 struct drm_mm_node *node, u64 size, 3873 unsigned alignment, u64 start, 3874 u64 end); 3875 void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv, 3876 struct drm_mm_node *node); 3877 int i915_gem_init_stolen(struct drm_i915_private *dev_priv); 3878 void i915_gem_cleanup_stolen(struct drm_device *dev); 3879 struct drm_i915_gem_object * 3880 i915_gem_object_create_stolen(struct drm_i915_private *dev_priv, u32 size); 3881 struct drm_i915_gem_object * 3882 i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv, 3883 u32 stolen_offset, 3884 u32 gtt_offset, 3885 u32 size); 3886 3887 /* i915_gem_internal.c */ 3888 struct drm_i915_gem_object * 3889 i915_gem_object_create_internal(struct drm_i915_private *dev_priv, 3890 phys_addr_t size); 3891 3892 /* i915_gem_shrinker.c */ 3893 unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv, 3894 unsigned long target, 3895 unsigned long *nr_scanned, 3896 unsigned flags); 3897 #define I915_SHRINK_PURGEABLE 0x1 3898 #define I915_SHRINK_UNBOUND 0x2 3899 #define I915_SHRINK_BOUND 0x4 3900 #define I915_SHRINK_ACTIVE 0x8 3901 #define I915_SHRINK_VMAPS 0x10 3902 unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv); 3903 void i915_gem_shrinker_init(struct drm_i915_private *dev_priv); 3904 void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv); 3905 3906 3907 /* i915_gem_tiling.c */ 3908 static inline bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 3909 { 3910 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 3911 3912 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 3913 i915_gem_object_is_tiled(obj); 3914 } 3915 3916 u32 i915_gem_fence_size(struct drm_i915_private *dev_priv, u32 size, 3917 unsigned int tiling, unsigned int stride); 3918 u32 i915_gem_fence_alignment(struct drm_i915_private *dev_priv, u32 size, 3919 unsigned int tiling, unsigned int stride); 3920 3921 /* i915_debugfs.c */ 3922 #ifdef CONFIG_DEBUG_FS 3923 int i915_debugfs_register(struct drm_i915_private *dev_priv); 3924 int i915_debugfs_connector_add(struct drm_connector *connector); 3925 void intel_display_crc_init(struct drm_i915_private *dev_priv); 3926 #else 3927 static inline int i915_debugfs_register(struct drm_i915_private *dev_priv) {return 0;} 3928 static inline int i915_debugfs_connector_add(struct drm_connector *connector) 3929 { return 0; } 3930 static inline void intel_display_crc_init(struct drm_i915_private *dev_priv) {} 3931 #endif 3932 3933 /* i915_gpu_error.c */ 3934 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) 3935 3936 __printf(2, 3) 3937 void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...); 3938 int i915_error_state_to_str(struct drm_i915_error_state_buf *estr, 3939 const struct i915_gpu_state *gpu); 3940 int i915_error_state_buf_init(struct drm_i915_error_state_buf *eb, 3941 struct drm_i915_private *i915, 3942 size_t count, loff_t pos); 3943 static inline void i915_error_state_buf_release( 3944 struct drm_i915_error_state_buf *eb) 3945 { 3946 kfree(eb->buf); 3947 } 3948 3949 struct i915_gpu_state *i915_capture_gpu_state(struct drm_i915_private *i915); 3950 void i915_capture_error_state(struct drm_i915_private *dev_priv, 3951 u32 engine_mask, 3952 const char *error_msg); 3953 3954 static inline struct i915_gpu_state * 3955 i915_gpu_state_get(struct i915_gpu_state *gpu) 3956 { 3957 kref_get(&gpu->ref); 3958 return gpu; 3959 } 3960 3961 void __i915_gpu_state_free(struct kref *kref); 3962 static inline void i915_gpu_state_put(struct i915_gpu_state *gpu) 3963 { 3964 if (gpu) 3965 kref_put(&gpu->ref, __i915_gpu_state_free); 3966 } 3967 3968 struct i915_gpu_state *i915_first_error_state(struct drm_i915_private *i915); 3969 void i915_reset_error_state(struct drm_i915_private *i915); 3970 3971 #else 3972 3973 static inline void i915_capture_error_state(struct drm_i915_private *dev_priv, 3974 u32 engine_mask, 3975 const char *error_msg) 3976 { 3977 } 3978 3979 static inline struct i915_gpu_state * 3980 i915_first_error_state(struct drm_i915_private *i915) 3981 { 3982 return NULL; 3983 } 3984 3985 static inline void i915_reset_error_state(struct drm_i915_private *i915) 3986 { 3987 } 3988 3989 #endif 3990 3991 const char *i915_cache_level_str(struct drm_i915_private *i915, int type); 3992 3993 /* i915_cmd_parser.c */ 3994 int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv); 3995 void intel_engine_init_cmd_parser(struct intel_engine_cs *engine); 3996 void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine); 3997 int intel_engine_cmd_parser(struct intel_engine_cs *engine, 3998 struct drm_i915_gem_object *batch_obj, 3999 struct drm_i915_gem_object *shadow_batch_obj, 4000 u32 batch_start_offset, 4001 u32 batch_len, 4002 bool is_master); 4003 4004 /* i915_perf.c */ 4005 extern void i915_perf_init(struct drm_i915_private *dev_priv); 4006 extern void i915_perf_fini(struct drm_i915_private *dev_priv); 4007 extern void i915_perf_register(struct drm_i915_private *dev_priv); 4008 extern void i915_perf_unregister(struct drm_i915_private *dev_priv); 4009 4010 /* i915_suspend.c */ 4011 extern int i915_save_state(struct drm_i915_private *dev_priv); 4012 extern int i915_restore_state(struct drm_i915_private *dev_priv); 4013 4014 /* i915_sysfs.c */ 4015 void i915_setup_sysfs(struct drm_i915_private *dev_priv); 4016 void i915_teardown_sysfs(struct drm_i915_private *dev_priv); 4017 4018 /* intel_lpe_audio.c */ 4019 int intel_lpe_audio_init(struct drm_i915_private *dev_priv); 4020 void intel_lpe_audio_teardown(struct drm_i915_private *dev_priv); 4021 void intel_lpe_audio_irq_handler(struct drm_i915_private *dev_priv); 4022 void intel_lpe_audio_notify(struct drm_i915_private *dev_priv, 4023 enum i915_pipe pipe, enum port port, 4024 const void *eld, int ls_clock, bool dp_output); 4025 4026 /* intel_i2c.c */ 4027 extern int intel_setup_gmbus(struct drm_i915_private *dev_priv); 4028 extern void intel_teardown_gmbus(struct drm_i915_private *dev_priv); 4029 extern bool intel_gmbus_is_valid_pin(struct drm_i915_private *dev_priv, 4030 unsigned int pin); 4031 4032 extern struct i2c_adapter * 4033 intel_gmbus_get_adapter(struct drm_i915_private *dev_priv, unsigned int pin); 4034 extern void intel_gmbus_set_speed(struct i2c_adapter *adapter, int speed); 4035 extern void intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit); 4036 static inline bool intel_gmbus_is_forced_bit(struct i2c_adapter *adapter) 4037 { 4038 return container_of(adapter, struct intel_gmbus, adapter)->force_bit; 4039 } 4040 extern void intel_i2c_reset(struct drm_i915_private *dev_priv); 4041 4042 /* intel_bios.c */ 4043 void intel_bios_init(struct drm_i915_private *dev_priv); 4044 bool intel_bios_is_valid_vbt(const void *buf, size_t size); 4045 bool intel_bios_is_tv_present(struct drm_i915_private *dev_priv); 4046 bool intel_bios_is_lvds_present(struct drm_i915_private *dev_priv, u8 *i2c_pin); 4047 bool intel_bios_is_port_present(struct drm_i915_private *dev_priv, enum port port); 4048 bool intel_bios_is_port_edp(struct drm_i915_private *dev_priv, enum port port); 4049 bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *dev_priv, enum port port); 4050 bool intel_bios_is_dsi_present(struct drm_i915_private *dev_priv, enum port *port); 4051 bool intel_bios_is_port_hpd_inverted(struct drm_i915_private *dev_priv, 4052 enum port port); 4053 bool intel_bios_is_lspcon_present(struct drm_i915_private *dev_priv, 4054 enum port port); 4055 4056 4057 /* intel_opregion.c */ 4058 #ifdef CONFIG_ACPI 4059 extern int intel_opregion_setup(struct drm_i915_private *dev_priv); 4060 extern void intel_opregion_register(struct drm_i915_private *dev_priv); 4061 extern void intel_opregion_unregister(struct drm_i915_private *dev_priv); 4062 extern void intel_opregion_asle_intr(struct drm_i915_private *dev_priv); 4063 extern int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, 4064 bool enable); 4065 extern int intel_opregion_notify_adapter(struct drm_i915_private *dev_priv, 4066 pci_power_t state); 4067 extern int intel_opregion_get_panel_type(struct drm_i915_private *dev_priv); 4068 #else 4069 static inline int intel_opregion_setup(struct drm_i915_private *dev) { return 0; } 4070 static inline void intel_opregion_register(struct drm_i915_private *dev_priv) { } 4071 static inline void intel_opregion_unregister(struct drm_i915_private *dev_priv) { } 4072 static inline void intel_opregion_asle_intr(struct drm_i915_private *dev_priv) 4073 { 4074 } 4075 static inline int 4076 intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, bool enable) 4077 { 4078 return 0; 4079 } 4080 static inline int 4081 intel_opregion_notify_adapter(struct drm_i915_private *dev, pci_power_t state) 4082 { 4083 return 0; 4084 } 4085 static inline int intel_opregion_get_panel_type(struct drm_i915_private *dev) 4086 { 4087 return -ENODEV; 4088 } 4089 #endif 4090 4091 /* intel_acpi.c */ 4092 #ifdef CONFIG_ACPI 4093 extern void intel_register_dsm_handler(void); 4094 extern void intel_unregister_dsm_handler(void); 4095 #else 4096 static inline void intel_register_dsm_handler(void) { return; } 4097 static inline void intel_unregister_dsm_handler(void) { return; } 4098 #endif /* CONFIG_ACPI */ 4099 4100 /* intel_device_info.c */ 4101 static inline struct intel_device_info * 4102 mkwrite_device_info(struct drm_i915_private *dev_priv) 4103 { 4104 return (struct intel_device_info *)&dev_priv->info; 4105 } 4106 4107 const char *intel_platform_name(enum intel_platform platform); 4108 void intel_device_info_runtime_init(struct drm_i915_private *dev_priv); 4109 void intel_device_info_dump(struct drm_i915_private *dev_priv); 4110 4111 /* modesetting */ 4112 extern void intel_modeset_init_hw(struct drm_device *dev); 4113 extern int intel_modeset_init(struct drm_device *dev); 4114 extern void intel_modeset_gem_init(struct drm_device *dev); 4115 extern void intel_modeset_cleanup(struct drm_device *dev); 4116 extern int intel_connector_register(struct drm_connector *); 4117 extern void intel_connector_unregister(struct drm_connector *); 4118 extern int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, 4119 bool state); 4120 extern void intel_display_resume(struct drm_device *dev); 4121 extern void i915_redisable_vga(struct drm_i915_private *dev_priv); 4122 extern void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv); 4123 extern bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val); 4124 extern void intel_init_pch_refclk(struct drm_i915_private *dev_priv); 4125 extern int intel_set_rps(struct drm_i915_private *dev_priv, u8 val); 4126 extern bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, 4127 bool enable); 4128 4129 int i915_reg_read_ioctl(struct drm_device *dev, void *data, 4130 struct drm_file *file); 4131 4132 /* overlay */ 4133 extern struct intel_overlay_error_state * 4134 intel_overlay_capture_error_state(struct drm_i915_private *dev_priv); 4135 extern void intel_overlay_print_error_state(struct drm_i915_error_state_buf *e, 4136 struct intel_overlay_error_state *error); 4137 4138 extern struct intel_display_error_state * 4139 intel_display_capture_error_state(struct drm_i915_private *dev_priv); 4140 extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e, 4141 struct intel_display_error_state *error); 4142 4143 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val); 4144 int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val); 4145 int skl_pcode_request(struct drm_i915_private *dev_priv, u32 mbox, u32 request, 4146 u32 reply_mask, u32 reply, int timeout_base_ms); 4147 4148 /* intel_sideband.c */ 4149 u32 vlv_punit_read(struct drm_i915_private *dev_priv, u32 addr); 4150 int vlv_punit_write(struct drm_i915_private *dev_priv, u32 addr, u32 val); 4151 u32 vlv_nc_read(struct drm_i915_private *dev_priv, u8 addr); 4152 u32 vlv_iosf_sb_read(struct drm_i915_private *dev_priv, u8 port, u32 reg); 4153 void vlv_iosf_sb_write(struct drm_i915_private *dev_priv, u8 port, u32 reg, u32 val); 4154 u32 vlv_cck_read(struct drm_i915_private *dev_priv, u32 reg); 4155 void vlv_cck_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); 4156 u32 vlv_ccu_read(struct drm_i915_private *dev_priv, u32 reg); 4157 void vlv_ccu_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); 4158 u32 vlv_bunit_read(struct drm_i915_private *dev_priv, u32 reg); 4159 void vlv_bunit_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); 4160 u32 vlv_dpio_read(struct drm_i915_private *dev_priv, enum i915_pipe pipe, int reg); 4161 void vlv_dpio_write(struct drm_i915_private *dev_priv, enum i915_pipe pipe, int reg, u32 val); 4162 u32 intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg, 4163 enum intel_sbi_destination destination); 4164 void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value, 4165 enum intel_sbi_destination destination); 4166 u32 vlv_flisdsi_read(struct drm_i915_private *dev_priv, u32 reg); 4167 void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); 4168 4169 /* intel_dpio_phy.c */ 4170 void bxt_port_to_phy_channel(struct drm_i915_private *dev_priv, enum port port, 4171 enum dpio_phy *phy, enum dpio_channel *ch); 4172 void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv, 4173 enum port port, u32 margin, u32 scale, 4174 u32 enable, u32 deemphasis); 4175 void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy); 4176 void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy); 4177 bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, 4178 enum dpio_phy phy); 4179 bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, 4180 enum dpio_phy phy); 4181 uint8_t bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder, 4182 uint8_t lane_count); 4183 void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder, 4184 uint8_t lane_lat_optim_mask); 4185 uint8_t bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder); 4186 4187 void chv_set_phy_signal_level(struct intel_encoder *encoder, 4188 u32 deemph_reg_value, u32 margin_reg_value, 4189 bool uniq_trans_scale); 4190 void chv_data_lane_soft_reset(struct intel_encoder *encoder, 4191 bool reset); 4192 void chv_phy_pre_pll_enable(struct intel_encoder *encoder); 4193 void chv_phy_pre_encoder_enable(struct intel_encoder *encoder); 4194 void chv_phy_release_cl2_override(struct intel_encoder *encoder); 4195 void chv_phy_post_pll_disable(struct intel_encoder *encoder); 4196 4197 void vlv_set_phy_signal_level(struct intel_encoder *encoder, 4198 u32 demph_reg_value, u32 preemph_reg_value, 4199 u32 uniqtranscale_reg_value, u32 tx3_demph); 4200 void vlv_phy_pre_pll_enable(struct intel_encoder *encoder); 4201 void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder); 4202 void vlv_phy_reset_lanes(struct intel_encoder *encoder); 4203 4204 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val); 4205 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val); 4206 u64 intel_rc6_residency_us(struct drm_i915_private *dev_priv, 4207 const i915_reg_t reg); 4208 4209 #define I915_READ8(reg) dev_priv->uncore.funcs.mmio_readb(dev_priv, (reg), true) 4210 #define I915_WRITE8(reg, val) dev_priv->uncore.funcs.mmio_writeb(dev_priv, (reg), (val), true) 4211 4212 #define I915_READ16(reg) dev_priv->uncore.funcs.mmio_readw(dev_priv, (reg), true) 4213 #define I915_WRITE16(reg, val) dev_priv->uncore.funcs.mmio_writew(dev_priv, (reg), (val), true) 4214 #define I915_READ16_NOTRACE(reg) dev_priv->uncore.funcs.mmio_readw(dev_priv, (reg), false) 4215 #define I915_WRITE16_NOTRACE(reg, val) dev_priv->uncore.funcs.mmio_writew(dev_priv, (reg), (val), false) 4216 4217 #define I915_READ(reg) dev_priv->uncore.funcs.mmio_readl(dev_priv, (reg), true) 4218 #define I915_WRITE(reg, val) dev_priv->uncore.funcs.mmio_writel(dev_priv, (reg), (val), true) 4219 #define I915_READ_NOTRACE(reg) dev_priv->uncore.funcs.mmio_readl(dev_priv, (reg), false) 4220 #define I915_WRITE_NOTRACE(reg, val) dev_priv->uncore.funcs.mmio_writel(dev_priv, (reg), (val), false) 4221 4222 /* Be very careful with read/write 64-bit values. On 32-bit machines, they 4223 * will be implemented using 2 32-bit writes in an arbitrary order with 4224 * an arbitrary delay between them. This can cause the hardware to 4225 * act upon the intermediate value, possibly leading to corruption and 4226 * machine death. For this reason we do not support I915_WRITE64, or 4227 * dev_priv->uncore.funcs.mmio_writeq. 4228 * 4229 * When reading a 64-bit value as two 32-bit values, the delay may cause 4230 * the two reads to mismatch, e.g. a timestamp overflowing. Also note that 4231 * occasionally a 64-bit register does not actualy support a full readq 4232 * and must be read using two 32-bit reads. 4233 * 4234 * You have been warned. 4235 */ 4236 #define I915_READ64(reg) dev_priv->uncore.funcs.mmio_readq(dev_priv, (reg), true) 4237 4238 #define I915_READ64_2x32(lower_reg, upper_reg) ({ \ 4239 u32 upper, lower, old_upper, loop = 0; \ 4240 upper = I915_READ(upper_reg); \ 4241 do { \ 4242 old_upper = upper; \ 4243 lower = I915_READ(lower_reg); \ 4244 upper = I915_READ(upper_reg); \ 4245 } while (upper != old_upper && loop++ < 2); \ 4246 (u64)upper << 32 | lower; }) 4247 4248 #define POSTING_READ(reg) (void)I915_READ_NOTRACE(reg) 4249 #define POSTING_READ16(reg) (void)I915_READ16_NOTRACE(reg) 4250 4251 #define __raw_read(x, s) \ 4252 static inline uint##x##_t __raw_i915_read##x(const struct drm_i915_private *dev_priv, \ 4253 i915_reg_t reg) \ 4254 { \ 4255 return read##s(dev_priv->regs + i915_mmio_reg_offset(reg)); \ 4256 } 4257 4258 #define __raw_write(x, s) \ 4259 static inline void __raw_i915_write##x(const struct drm_i915_private *dev_priv, \ 4260 i915_reg_t reg, uint##x##_t val) \ 4261 { \ 4262 write##s(val, dev_priv->regs + i915_mmio_reg_offset(reg)); \ 4263 } 4264 __raw_read(8, b) 4265 __raw_read(16, w) 4266 __raw_read(32, l) 4267 __raw_read(64, q) 4268 4269 __raw_write(8, b) 4270 __raw_write(16, w) 4271 __raw_write(32, l) 4272 __raw_write(64, q) 4273 4274 #undef __raw_read 4275 #undef __raw_write 4276 4277 /* These are untraced mmio-accessors that are only valid to be used inside 4278 * critical sections, such as inside IRQ handlers, where forcewake is explicitly 4279 * controlled. 4280 * 4281 * Think twice, and think again, before using these. 4282 * 4283 * As an example, these accessors can possibly be used between: 4284 * 4285 * spin_lock_irq(&dev_priv->uncore.lock); 4286 * intel_uncore_forcewake_get__locked(); 4287 * 4288 * and 4289 * 4290 * intel_uncore_forcewake_put__locked(); 4291 * spin_unlock_irq(&dev_priv->uncore.lock); 4292 * 4293 * 4294 * Note: some registers may not need forcewake held, so 4295 * intel_uncore_forcewake_{get,put} can be omitted, see 4296 * intel_uncore_forcewake_for_reg(). 4297 * 4298 * Certain architectures will die if the same cacheline is concurrently accessed 4299 * by different clients (e.g. on Ivybridge). Access to registers should 4300 * therefore generally be serialised, by either the dev_priv->uncore.lock or 4301 * a more localised lock guarding all access to that bank of registers. 4302 */ 4303 #define I915_READ_FW(reg__) __raw_i915_read32(dev_priv, (reg__)) 4304 #define I915_WRITE_FW(reg__, val__) __raw_i915_write32(dev_priv, (reg__), (val__)) 4305 #define I915_WRITE64_FW(reg__, val__) __raw_i915_write64(dev_priv, (reg__), (val__)) 4306 #define POSTING_READ_FW(reg__) (void)I915_READ_FW(reg__) 4307 4308 /* "Broadcast RGB" property */ 4309 #define INTEL_BROADCAST_RGB_AUTO 0 4310 #define INTEL_BROADCAST_RGB_FULL 1 4311 #define INTEL_BROADCAST_RGB_LIMITED 2 4312 4313 static inline i915_reg_t i915_vgacntrl_reg(struct drm_i915_private *dev_priv) 4314 { 4315 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 4316 return VLV_VGACNTRL; 4317 else if (INTEL_GEN(dev_priv) >= 5) 4318 return CPU_VGACNTRL; 4319 else 4320 return VGACNTRL; 4321 } 4322 4323 static inline unsigned long msecs_to_jiffies_timeout(const unsigned int m) 4324 { 4325 unsigned long j = msecs_to_jiffies(m); 4326 4327 return min_t(unsigned long, MAX_JIFFY_OFFSET, j + 1); 4328 } 4329 4330 static inline unsigned long nsecs_to_jiffies_timeout(const u64 n) 4331 { 4332 /* nsecs_to_jiffies64() does not guard against overflow */ 4333 if (NSEC_PER_SEC % HZ && 4334 div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ) 4335 return MAX_JIFFY_OFFSET; 4336 4337 return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1); 4338 } 4339 4340 static inline unsigned long 4341 timespec_to_jiffies_timeout(const struct timespec *value) 4342 { 4343 unsigned long j = timespec_to_jiffies(value); 4344 4345 return min_t(unsigned long, MAX_JIFFY_OFFSET, j + 1); 4346 } 4347 4348 /* 4349 * If you need to wait X milliseconds between events A and B, but event B 4350 * doesn't happen exactly after event A, you record the timestamp (jiffies) of 4351 * when event A happened, then just before event B you call this function and 4352 * pass the timestamp as the first argument, and X as the second argument. 4353 */ 4354 static inline void 4355 wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms) 4356 { 4357 unsigned long target_jiffies, tmp_jiffies, remaining_jiffies; 4358 4359 /* 4360 * Don't re-read the value of "jiffies" every time since it may change 4361 * behind our back and break the math. 4362 */ 4363 tmp_jiffies = jiffies; 4364 target_jiffies = timestamp_jiffies + 4365 msecs_to_jiffies_timeout(to_wait_ms); 4366 4367 if (time_after(target_jiffies, tmp_jiffies)) { 4368 remaining_jiffies = target_jiffies - tmp_jiffies; 4369 while (remaining_jiffies) 4370 remaining_jiffies = 4371 schedule_timeout_uninterruptible(remaining_jiffies); 4372 } 4373 } 4374 4375 static inline bool 4376 __i915_request_irq_complete(const struct drm_i915_gem_request *req) 4377 { 4378 struct intel_engine_cs *engine = req->engine; 4379 u32 seqno; 4380 4381 /* Note that the engine may have wrapped around the seqno, and 4382 * so our request->global_seqno will be ahead of the hardware, 4383 * even though it completed the request before wrapping. We catch 4384 * this by kicking all the waiters before resetting the seqno 4385 * in hardware, and also signal the fence. 4386 */ 4387 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &req->fence.flags)) 4388 return true; 4389 4390 /* The request was dequeued before we were awoken. We check after 4391 * inspecting the hw to confirm that this was the same request 4392 * that generated the HWS update. The memory barriers within 4393 * the request execution are sufficient to ensure that a check 4394 * after reading the value from hw matches this request. 4395 */ 4396 seqno = i915_gem_request_global_seqno(req); 4397 if (!seqno) 4398 return false; 4399 4400 /* Before we do the heavier coherent read of the seqno, 4401 * check the value (hopefully) in the CPU cacheline. 4402 */ 4403 if (__i915_gem_request_completed(req, seqno)) 4404 return true; 4405 4406 /* Ensure our read of the seqno is coherent so that we 4407 * do not "miss an interrupt" (i.e. if this is the last 4408 * request and the seqno write from the GPU is not visible 4409 * by the time the interrupt fires, we will see that the 4410 * request is incomplete and go back to sleep awaiting 4411 * another interrupt that will never come.) 4412 * 4413 * Strictly, we only need to do this once after an interrupt, 4414 * but it is easier and safer to do it every time the waiter 4415 * is woken. 4416 */ 4417 if (engine->irq_seqno_barrier && 4418 test_and_clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted)) { 4419 struct intel_breadcrumbs *b = &engine->breadcrumbs; 4420 4421 /* The ordering of irq_posted versus applying the barrier 4422 * is crucial. The clearing of the current irq_posted must 4423 * be visible before we perform the barrier operation, 4424 * such that if a subsequent interrupt arrives, irq_posted 4425 * is reasserted and our task rewoken (which causes us to 4426 * do another __i915_request_irq_complete() immediately 4427 * and reapply the barrier). Conversely, if the clear 4428 * occurs after the barrier, then an interrupt that arrived 4429 * whilst we waited on the barrier would not trigger a 4430 * barrier on the next pass, and the read may not see the 4431 * seqno update. 4432 */ 4433 engine->irq_seqno_barrier(engine); 4434 4435 /* If we consume the irq, but we are no longer the bottom-half, 4436 * the real bottom-half may not have serialised their own 4437 * seqno check with the irq-barrier (i.e. may have inspected 4438 * the seqno before we believe it coherent since they see 4439 * irq_posted == false but we are still running). 4440 */ 4441 spin_lock_irq(&b->irq_lock); 4442 if (b->irq_wait && b->irq_wait->tsk != current) 4443 /* Note that if the bottom-half is changed as we 4444 * are sending the wake-up, the new bottom-half will 4445 * be woken by whomever made the change. We only have 4446 * to worry about when we steal the irq-posted for 4447 * ourself. 4448 */ 4449 wake_up_process(b->irq_wait->tsk); 4450 spin_unlock_irq(&b->irq_lock); 4451 4452 if (__i915_gem_request_completed(req, seqno)) 4453 return true; 4454 } 4455 4456 return false; 4457 } 4458 4459 void i915_memcpy_init_early(struct drm_i915_private *dev_priv); 4460 bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len); 4461 4462 /* The movntdqa instructions used for memcpy-from-wc require 16-byte alignment, 4463 * as well as SSE4.1 support. i915_memcpy_from_wc() will report if it cannot 4464 * perform the operation. To check beforehand, pass in the parameters to 4465 * to i915_can_memcpy_from_wc() - since we only care about the low 4 bits, 4466 * you only need to pass in the minor offsets, page-aligned pointers are 4467 * always valid. 4468 * 4469 * For just checking for SSE4.1, in the foreknowledge that the future use 4470 * will be correctly aligned, just use i915_has_memcpy_from_wc(). 4471 */ 4472 #define i915_can_memcpy_from_wc(dst, src, len) \ 4473 i915_memcpy_from_wc((void *)((unsigned long)(dst) | (unsigned long)(src) | (len)), NULL, 0) 4474 4475 #define i915_has_memcpy_from_wc() \ 4476 i915_memcpy_from_wc(NULL, NULL, 0) 4477 4478 /* i915_mm.c */ 4479 int remap_io_mapping(struct vm_area_struct *vma, 4480 unsigned long addr, unsigned long pfn, unsigned long size, 4481 struct io_mapping *iomap); 4482 4483 static inline int intel_hws_csb_write_index(struct drm_i915_private *i915) 4484 { 4485 if (INTEL_GEN(i915) >= 10) 4486 return CNL_HWS_CSB_WRITE_INDEX; 4487 else 4488 return I915_HWS_CSB_WRITE_INDEX; 4489 } 4490 4491 #endif 4492