1 /* 2 * Copyright © 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 */ 27 28 #include <linux/string.h> 29 #include <linux/bitops.h> 30 #include <drm/drmP.h> 31 #include <drm/i915_drm.h> 32 #include "i915_drv.h" 33 34 /** 35 * DOC: buffer object tiling 36 * 37 * i915_gem_set_tiling_ioctl() and i915_gem_get_tiling_ioctl() is the userspace 38 * interface to declare fence register requirements. 39 * 40 * In principle GEM doesn't care at all about the internal data layout of an 41 * object, and hence it also doesn't care about tiling or swizzling. There's two 42 * exceptions: 43 * 44 * - For X and Y tiling the hardware provides detilers for CPU access, so called 45 * fences. Since there's only a limited amount of them the kernel must manage 46 * these, and therefore userspace must tell the kernel the object tiling if it 47 * wants to use fences for detiling. 48 * - On gen3 and gen4 platforms have a swizzling pattern for tiled objects which 49 * depends upon the physical page frame number. When swapping such objects the 50 * page frame number might change and the kernel must be able to fix this up 51 * and hence now the tiling. Note that on a subset of platforms with 52 * asymmetric memory channel population the swizzling pattern changes in an 53 * unknown way, and for those the kernel simply forbids swapping completely. 54 * 55 * Since neither of this applies for new tiling layouts on modern platforms like 56 * W, Ys and Yf tiling GEM only allows object tiling to be set to X or Y tiled. 57 * Anything else can be handled in userspace entirely without the kernel's 58 * invovlement. 59 */ 60 61 /** 62 * i915_gem_fence_size - required global GTT size for a fence 63 * @i915: i915 device 64 * @size: object size 65 * @tiling: tiling mode 66 * @stride: tiling stride 67 * 68 * Return the required global GTT size for a fence (view of a tiled object), 69 * taking into account potential fence register mapping. 70 */ 71 u32 i915_gem_fence_size(struct drm_i915_private *i915, 72 u32 size, unsigned int tiling, unsigned int stride) 73 { 74 u32 ggtt_size; 75 76 GEM_BUG_ON(!size); 77 78 if (tiling == I915_TILING_NONE) 79 return size; 80 81 GEM_BUG_ON(!stride); 82 83 if (INTEL_GEN(i915) >= 4) { 84 stride *= i915_gem_tile_height(tiling); 85 GEM_BUG_ON(!IS_ALIGNED(stride, I965_FENCE_PAGE)); 86 return roundup(size, stride); 87 } 88 89 /* Previous chips need a power-of-two fence region when tiling */ 90 if (IS_GEN3(i915)) 91 ggtt_size = 1024*1024; 92 else 93 ggtt_size = 512*1024; 94 95 while (ggtt_size < size) 96 ggtt_size <<= 1; 97 98 return ggtt_size; 99 } 100 101 /** 102 * i915_gem_fence_alignment - required global GTT alignment for a fence 103 * @i915: i915 device 104 * @size: object size 105 * @tiling: tiling mode 106 * @stride: tiling stride 107 * 108 * Return the required global GTT alignment for a fence (a view of a tiled 109 * object), taking into account potential fence register mapping. 110 */ 111 u32 i915_gem_fence_alignment(struct drm_i915_private *i915, u32 size, 112 unsigned int tiling, unsigned int stride) 113 { 114 GEM_BUG_ON(!size); 115 116 /* 117 * Minimum alignment is 4k (GTT page size), but might be greater 118 * if a fence register is needed for the object. 119 */ 120 if (tiling == I915_TILING_NONE) 121 return I915_GTT_MIN_ALIGNMENT; 122 123 if (INTEL_GEN(i915) >= 4) 124 return I965_FENCE_PAGE; 125 126 /* 127 * Previous chips need to be aligned to the size of the smallest 128 * fence register that can contain the object. 129 */ 130 return i915_gem_fence_size(i915, size, tiling, stride); 131 } 132 133 /* Check pitch constriants for all chips & tiling formats */ 134 static bool 135 i915_tiling_ok(struct drm_i915_gem_object *obj, 136 unsigned int tiling, unsigned int stride) 137 { 138 struct drm_i915_private *i915 = to_i915(obj->base.dev); 139 unsigned int tile_width; 140 141 /* Linear is always fine */ 142 if (tiling == I915_TILING_NONE) 143 return true; 144 145 if (tiling > I915_TILING_LAST) 146 return false; 147 148 /* check maximum stride & object size */ 149 /* i965+ stores the end address of the gtt mapping in the fence 150 * reg, so dont bother to check the size */ 151 if (INTEL_GEN(i915) >= 7) { 152 if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL) 153 return false; 154 } else if (INTEL_GEN(i915) >= 4) { 155 if (stride / 128 > I965_FENCE_MAX_PITCH_VAL) 156 return false; 157 } else { 158 if (stride > 8192) 159 return false; 160 161 if (!is_power_of_2(stride)) 162 return false; 163 } 164 165 if (IS_GEN2(i915) || 166 (tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(i915))) 167 tile_width = 128; 168 else 169 tile_width = 512; 170 171 if (!stride || !IS_ALIGNED(stride, tile_width)) 172 return false; 173 174 return true; 175 } 176 177 static bool i915_vma_fence_prepare(struct i915_vma *vma, 178 int tiling_mode, unsigned int stride) 179 { 180 struct drm_i915_private *i915 = vma->vm->i915; 181 u32 size, alignment; 182 183 if (!i915_vma_is_map_and_fenceable(vma)) 184 return true; 185 186 size = i915_gem_fence_size(i915, vma->size, tiling_mode, stride); 187 if (vma->node.size < size) 188 return false; 189 190 alignment = i915_gem_fence_alignment(i915, vma->size, tiling_mode, stride); 191 if (!IS_ALIGNED(vma->node.start, alignment)) 192 return false; 193 194 return true; 195 } 196 197 /* Make the current GTT allocation valid for the change in tiling. */ 198 static int 199 i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj, 200 int tiling_mode, unsigned int stride) 201 { 202 struct i915_vma *vma; 203 int ret; 204 205 if (tiling_mode == I915_TILING_NONE) 206 return 0; 207 208 list_for_each_entry(vma, &obj->vma_list, obj_link) { 209 if (!i915_vma_is_ggtt(vma)) 210 break; 211 212 if (i915_vma_fence_prepare(vma, tiling_mode, stride)) 213 continue; 214 215 ret = i915_vma_unbind(vma); 216 if (ret) 217 return ret; 218 } 219 220 return 0; 221 } 222 223 int 224 i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, 225 unsigned int tiling, unsigned int stride) 226 { 227 struct drm_i915_private *i915 = to_i915(obj->base.dev); 228 struct i915_vma *vma; 229 int err; 230 231 /* Make sure we don't cross-contaminate obj->tiling_and_stride */ 232 BUILD_BUG_ON(I915_TILING_LAST & STRIDE_MASK); 233 234 GEM_BUG_ON(!i915_tiling_ok(obj, tiling, stride)); 235 GEM_BUG_ON(!stride ^ (tiling == I915_TILING_NONE)); 236 lockdep_assert_held(&i915->drm.struct_mutex); 237 238 if ((tiling | stride) == obj->tiling_and_stride) 239 return 0; 240 241 if (i915_gem_object_is_framebuffer(obj)) 242 return -EBUSY; 243 244 /* We need to rebind the object if its current allocation 245 * no longer meets the alignment restrictions for its new 246 * tiling mode. Otherwise we can just leave it alone, but 247 * need to ensure that any fence register is updated before 248 * the next fenced (either through the GTT or by the BLT unit 249 * on older GPUs) access. 250 * 251 * After updating the tiling parameters, we then flag whether 252 * we need to update an associated fence register. Note this 253 * has to also include the unfenced register the GPU uses 254 * whilst executing a fenced command for an untiled object. 255 */ 256 257 err = i915_gem_object_fence_prepare(obj, tiling, stride); 258 if (err) 259 return err; 260 261 i915_gem_object_lock(obj); 262 if (i915_gem_object_is_framebuffer(obj)) { 263 i915_gem_object_unlock(obj); 264 return -EBUSY; 265 } 266 267 /* If the memory has unknown (i.e. varying) swizzling, we pin the 268 * pages to prevent them being swapped out and causing corruption 269 * due to the change in swizzling. 270 */ 271 mutex_lock(&obj->mm.lock); 272 if (i915_gem_object_has_pages(obj) && 273 obj->mm.madv == I915_MADV_WILLNEED && 274 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { 275 if (tiling == I915_TILING_NONE) { 276 GEM_BUG_ON(!obj->mm.quirked); 277 __i915_gem_object_unpin_pages(obj); 278 obj->mm.quirked = false; 279 } 280 if (!i915_gem_object_is_tiled(obj)) { 281 GEM_BUG_ON(obj->mm.quirked); 282 __i915_gem_object_pin_pages(obj); 283 obj->mm.quirked = true; 284 } 285 } 286 mutex_unlock(&obj->mm.lock); 287 288 list_for_each_entry(vma, &obj->vma_list, obj_link) { 289 if (!i915_vma_is_ggtt(vma)) 290 break; 291 292 vma->fence_size = 293 i915_gem_fence_size(i915, vma->size, tiling, stride); 294 vma->fence_alignment = 295 i915_gem_fence_alignment(i915, 296 vma->size, tiling, stride); 297 298 if (vma->fence) 299 vma->fence->dirty = true; 300 } 301 302 obj->tiling_and_stride = tiling | stride; 303 i915_gem_object_unlock(obj); 304 305 /* Force the fence to be reacquired for GTT access */ 306 i915_gem_release_mmap(obj); 307 308 /* Try to preallocate memory required to save swizzling on put-pages */ 309 if (i915_gem_object_needs_bit17_swizzle(obj)) { 310 if (!obj->bit_17) { 311 obj->bit_17 = kcalloc(BITS_TO_LONGS(obj->base.size >> PAGE_SHIFT), 312 sizeof(long), GFP_KERNEL); 313 } 314 } else { 315 kfree(obj->bit_17); 316 obj->bit_17 = NULL; 317 } 318 319 return 0; 320 } 321 322 /** 323 * i915_gem_set_tiling_ioctl - IOCTL handler to set tiling mode 324 * @dev: DRM device 325 * @data: data pointer for the ioctl 326 * @file: DRM file for the ioctl call 327 * 328 * Sets the tiling mode of an object, returning the required swizzling of 329 * bit 6 of addresses in the object. 330 * 331 * Called by the user via ioctl. 332 * 333 * Returns: 334 * Zero on success, negative errno on failure. 335 */ 336 int 337 i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data, 338 struct drm_file *file) 339 { 340 struct drm_i915_gem_set_tiling *args = data; 341 struct drm_i915_gem_object *obj; 342 int err; 343 344 obj = i915_gem_object_lookup(file, args->handle); 345 if (!obj) 346 return -ENOENT; 347 348 if (!i915_tiling_ok(obj, args->tiling_mode, args->stride)) { 349 err = -EINVAL; 350 goto err; 351 } 352 353 if (args->tiling_mode == I915_TILING_NONE) { 354 args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; 355 args->stride = 0; 356 } else { 357 if (args->tiling_mode == I915_TILING_X) 358 args->swizzle_mode = to_i915(dev)->mm.bit_6_swizzle_x; 359 else 360 args->swizzle_mode = to_i915(dev)->mm.bit_6_swizzle_y; 361 362 /* Hide bit 17 swizzling from the user. This prevents old Mesa 363 * from aborting the application on sw fallbacks to bit 17, 364 * and we use the pread/pwrite bit17 paths to swizzle for it. 365 * If there was a user that was relying on the swizzle 366 * information for drm_intel_bo_map()ed reads/writes this would 367 * break it, but we don't have any of those. 368 */ 369 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17) 370 args->swizzle_mode = I915_BIT_6_SWIZZLE_9; 371 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) 372 args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; 373 374 /* If we can't handle the swizzling, make it untiled. */ 375 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) { 376 args->tiling_mode = I915_TILING_NONE; 377 args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; 378 args->stride = 0; 379 } 380 } 381 382 err = mutex_lock_interruptible(&dev->struct_mutex); 383 if (err) 384 goto err; 385 386 err = i915_gem_object_set_tiling(obj, args->tiling_mode, args->stride); 387 mutex_unlock(&dev->struct_mutex); 388 389 /* We have to maintain this existing ABI... */ 390 args->stride = i915_gem_object_get_stride(obj); 391 args->tiling_mode = i915_gem_object_get_tiling(obj); 392 393 err: 394 i915_gem_object_put(obj); 395 return err; 396 } 397 398 /** 399 * i915_gem_get_tiling_ioctl - IOCTL handler to get tiling mode 400 * @dev: DRM device 401 * @data: data pointer for the ioctl 402 * @file: DRM file for the ioctl call 403 * 404 * Returns the current tiling mode and required bit 6 swizzling for the object. 405 * 406 * Called by the user via ioctl. 407 * 408 * Returns: 409 * Zero on success, negative errno on failure. 410 */ 411 int 412 i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data, 413 struct drm_file *file) 414 { 415 struct drm_i915_gem_get_tiling *args = data; 416 struct drm_i915_private *dev_priv = to_i915(dev); 417 struct drm_i915_gem_object *obj; 418 int err = -ENOENT; 419 420 rcu_read_lock(); 421 obj = i915_gem_object_lookup_rcu(file, args->handle); 422 if (obj) { 423 args->tiling_mode = 424 READ_ONCE(obj->tiling_and_stride) & TILING_MASK; 425 err = 0; 426 } 427 rcu_read_unlock(); 428 if (unlikely(err)) 429 return err; 430 431 switch (args->tiling_mode) { 432 case I915_TILING_X: 433 args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x; 434 break; 435 case I915_TILING_Y: 436 args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y; 437 break; 438 default: 439 case I915_TILING_NONE: 440 args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; 441 break; 442 } 443 444 /* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */ 445 if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) 446 args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN; 447 else 448 args->phys_swizzle_mode = args->swizzle_mode; 449 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17) 450 args->swizzle_mode = I915_BIT_6_SWIZZLE_9; 451 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) 452 args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; 453 454 return 0; 455 } 456