1 /* 2 * Copyright © 2017 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/sched/mm.h> 26 27 #include "display/intel_frontbuffer.h" 28 #include "gt/intel_gt.h" 29 #include "i915_drv.h" 30 #include "i915_gem_clflush.h" 31 #include "i915_gem_context.h" 32 #include "i915_gem_mman.h" 33 #include "i915_gem_object.h" 34 #include "i915_globals.h" 35 #include "i915_trace.h" 36 37 static struct i915_global_object { 38 struct i915_global base; 39 #ifdef __linux__ 40 struct kmem_cache *slab_objects; 41 #else 42 struct pool slab_objects; 43 #endif 44 } global; 45 46 struct drm_i915_gem_object *i915_gem_object_alloc(void) 47 { 48 #ifdef __linux__ 49 return kmem_cache_zalloc(global.slab_objects, GFP_KERNEL); 50 #else 51 return pool_get(&global.slab_objects, PR_WAITOK | PR_ZERO); 52 #endif 53 } 54 55 void i915_gem_object_free(struct drm_i915_gem_object *obj) 56 { 57 #ifdef __linux__ 58 return kmem_cache_free(global.slab_objects, obj); 59 #else 60 pool_put(&global.slab_objects, obj); 61 #endif 62 } 63 64 void i915_gem_object_init(struct drm_i915_gem_object *obj, 65 const struct drm_i915_gem_object_ops *ops, 66 struct lock_class_key *key) 67 { 68 #ifdef __linux__ 69 __mutex_init(&obj->mm.lock, "obj->mm.lock", key); 70 #else 71 rw_init(&obj->mm.lock, "objmm"); 72 #endif 73 74 mtx_init(&obj->vma.lock, IPL_NONE); 75 INIT_LIST_HEAD(&obj->vma.list); 76 77 INIT_LIST_HEAD(&obj->mm.link); 78 79 INIT_LIST_HEAD(&obj->lut_list); 80 81 mtx_init(&obj->mmo.lock, IPL_NONE); 82 obj->mmo.offsets = RB_ROOT; 83 84 init_rcu_head(&obj->rcu); 85 86 obj->ops = ops; 87 88 obj->mm.madv = I915_MADV_WILLNEED; 89 INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN); 90 rw_init(&obj->mm.get_page.lock, "mmget"); 91 } 92 93 /** 94 * Mark up the object's coherency levels for a given cache_level 95 * @obj: #drm_i915_gem_object 96 * @cache_level: cache level 97 */ 98 void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, 99 unsigned int cache_level) 100 { 101 obj->cache_level = cache_level; 102 103 if (cache_level != I915_CACHE_NONE) 104 obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ | 105 I915_BO_CACHE_COHERENT_FOR_WRITE); 106 else if (HAS_LLC(to_i915(obj->base.dev))) 107 obj->cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ; 108 else 109 obj->cache_coherent = 0; 110 111 obj->cache_dirty = 112 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE); 113 } 114 115 void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) 116 { 117 struct drm_i915_gem_object *obj = to_intel_bo(gem); 118 struct drm_i915_file_private *fpriv = file->driver_priv; 119 struct i915_mmap_offset *mmo, *mn; 120 struct i915_lut_handle *lut, *ln; 121 DRM_LIST_HEAD(close); 122 123 i915_gem_object_lock(obj); 124 list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) { 125 struct i915_gem_context *ctx = lut->ctx; 126 127 if (ctx->file_priv != fpriv) 128 continue; 129 130 i915_gem_context_get(ctx); 131 list_move(&lut->obj_link, &close); 132 } 133 i915_gem_object_unlock(obj); 134 135 spin_lock(&obj->mmo.lock); 136 rbtree_postorder_for_each_entry_safe(mmo, mn, &obj->mmo.offsets, offset) 137 drm_vma_node_revoke(&mmo->vma_node, file->filp); 138 spin_unlock(&obj->mmo.lock); 139 140 list_for_each_entry_safe(lut, ln, &close, obj_link) { 141 struct i915_gem_context *ctx = lut->ctx; 142 struct i915_vma *vma; 143 144 /* 145 * We allow the process to have multiple handles to the same 146 * vma, in the same fd namespace, by virtue of flink/open. 147 */ 148 149 mutex_lock(&ctx->mutex); 150 vma = radix_tree_delete(&ctx->handles_vma, lut->handle); 151 if (vma) { 152 GEM_BUG_ON(vma->obj != obj); 153 GEM_BUG_ON(!atomic_read(&vma->open_count)); 154 if (atomic_dec_and_test(&vma->open_count) && 155 !i915_vma_is_ggtt(vma)) 156 i915_vma_close(vma); 157 } 158 mutex_unlock(&ctx->mutex); 159 160 i915_gem_context_put(lut->ctx); 161 i915_lut_handle_free(lut); 162 i915_gem_object_put(obj); 163 } 164 } 165 166 static void __i915_gem_free_object_rcu(struct rcu_head *head) 167 { 168 struct drm_i915_gem_object *obj = 169 container_of(head, typeof(*obj), rcu); 170 struct drm_i915_private *i915 = to_i915(obj->base.dev); 171 172 #ifdef __OpenBSD__ 173 if (obj->base.uao) 174 uao_detach(obj->base.uao); 175 #endif 176 177 dma_resv_fini(&obj->base._resv); 178 i915_gem_object_free(obj); 179 180 GEM_BUG_ON(!atomic_read(&i915->mm.free_count)); 181 atomic_dec(&i915->mm.free_count); 182 } 183 184 static void __i915_gem_free_objects(struct drm_i915_private *i915, 185 struct llist_node *freed) 186 { 187 struct drm_i915_gem_object *obj, *on; 188 intel_wakeref_t wakeref; 189 190 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 191 llist_for_each_entry_safe(obj, on, freed, freed) { 192 struct i915_mmap_offset *mmo, *mn; 193 194 trace_i915_gem_object_destroy(obj); 195 196 if (!list_empty(&obj->vma.list)) { 197 struct i915_vma *vma; 198 199 /* 200 * Note that the vma keeps an object reference while 201 * it is active, so it *should* not sleep while we 202 * destroy it. Our debug code errs insits it *might*. 203 * For the moment, play along. 204 */ 205 spin_lock(&obj->vma.lock); 206 while ((vma = list_first_entry_or_null(&obj->vma.list, 207 struct i915_vma, 208 obj_link))) { 209 GEM_BUG_ON(vma->obj != obj); 210 spin_unlock(&obj->vma.lock); 211 212 __i915_vma_put(vma); 213 214 spin_lock(&obj->vma.lock); 215 } 216 spin_unlock(&obj->vma.lock); 217 } 218 219 i915_gem_object_release_mmap(obj); 220 221 rbtree_postorder_for_each_entry_safe(mmo, mn, 222 &obj->mmo.offsets, 223 offset) { 224 drm_vma_offset_remove(obj->base.dev->vma_offset_manager, 225 &mmo->vma_node); 226 kfree(mmo); 227 } 228 obj->mmo.offsets = RB_ROOT; 229 230 GEM_BUG_ON(atomic_read(&obj->bind_count)); 231 GEM_BUG_ON(obj->userfault_count); 232 GEM_BUG_ON(!list_empty(&obj->lut_list)); 233 234 atomic_set(&obj->mm.pages_pin_count, 0); 235 __i915_gem_object_put_pages(obj); 236 GEM_BUG_ON(i915_gem_object_has_pages(obj)); 237 bitmap_free(obj->bit_17); 238 239 if (obj->base.import_attach) 240 drm_prime_gem_destroy(&obj->base, NULL); 241 242 drm_gem_free_mmap_offset(&obj->base); 243 244 if (obj->ops->release) 245 obj->ops->release(obj); 246 247 /* But keep the pointer alive for RCU-protected lookups */ 248 call_rcu(&obj->rcu, __i915_gem_free_object_rcu); 249 cond_resched(); 250 } 251 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 252 } 253 254 void i915_gem_flush_free_objects(struct drm_i915_private *i915) 255 { 256 struct llist_node *freed = llist_del_all(&i915->mm.free_list); 257 258 if (unlikely(freed)) 259 __i915_gem_free_objects(i915, freed); 260 } 261 262 static void __i915_gem_free_work(struct work_struct *work) 263 { 264 struct drm_i915_private *i915 = 265 container_of(work, struct drm_i915_private, mm.free_work); 266 267 i915_gem_flush_free_objects(i915); 268 } 269 270 void i915_gem_free_object(struct drm_gem_object *gem_obj) 271 { 272 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 273 struct drm_i915_private *i915 = to_i915(obj->base.dev); 274 275 GEM_BUG_ON(i915_gem_object_is_framebuffer(obj)); 276 277 /* 278 * Before we free the object, make sure any pure RCU-only 279 * read-side critical sections are complete, e.g. 280 * i915_gem_busy_ioctl(). For the corresponding synchronized 281 * lookup see i915_gem_object_lookup_rcu(). 282 */ 283 atomic_inc(&i915->mm.free_count); 284 285 /* 286 * This serializes freeing with the shrinker. Since the free 287 * is delayed, first by RCU then by the workqueue, we want the 288 * shrinker to be able to free pages of unreferenced objects, 289 * or else we may oom whilst there are plenty of deferred 290 * freed objects. 291 */ 292 i915_gem_object_make_unshrinkable(obj); 293 294 /* 295 * Since we require blocking on struct_mutex to unbind the freed 296 * object from the GPU before releasing resources back to the 297 * system, we can not do that directly from the RCU callback (which may 298 * be a softirq context), but must instead then defer that work onto a 299 * kthread. We use the RCU callback rather than move the freed object 300 * directly onto the work queue so that we can mix between using the 301 * worker and performing frees directly from subsequent allocations for 302 * crude but effective memory throttling. 303 */ 304 if (llist_add(&obj->freed, &i915->mm.free_list)) 305 queue_work(i915->wq, &i915->mm.free_work); 306 } 307 308 static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj) 309 { 310 return !(obj->cache_level == I915_CACHE_NONE || 311 obj->cache_level == I915_CACHE_WT); 312 } 313 314 void 315 i915_gem_object_flush_write_domain(struct drm_i915_gem_object *obj, 316 unsigned int flush_domains) 317 { 318 struct i915_vma *vma; 319 320 assert_object_held(obj); 321 322 if (!(obj->write_domain & flush_domains)) 323 return; 324 325 switch (obj->write_domain) { 326 case I915_GEM_DOMAIN_GTT: 327 spin_lock(&obj->vma.lock); 328 for_each_ggtt_vma(vma, obj) { 329 if (i915_vma_unset_ggtt_write(vma)) 330 intel_gt_flush_ggtt_writes(vma->vm->gt); 331 } 332 spin_unlock(&obj->vma.lock); 333 334 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU); 335 break; 336 337 case I915_GEM_DOMAIN_WC: 338 wmb(); 339 break; 340 341 case I915_GEM_DOMAIN_CPU: 342 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC); 343 break; 344 345 case I915_GEM_DOMAIN_RENDER: 346 if (gpu_write_needs_clflush(obj)) 347 obj->cache_dirty = true; 348 break; 349 } 350 351 obj->write_domain = 0; 352 } 353 354 void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj, 355 enum fb_op_origin origin) 356 { 357 struct intel_frontbuffer *front; 358 359 front = __intel_frontbuffer_get(obj); 360 if (front) { 361 intel_frontbuffer_flush(front, origin); 362 intel_frontbuffer_put(front); 363 } 364 } 365 366 void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj, 367 enum fb_op_origin origin) 368 { 369 struct intel_frontbuffer *front; 370 371 front = __intel_frontbuffer_get(obj); 372 if (front) { 373 intel_frontbuffer_invalidate(front, origin); 374 intel_frontbuffer_put(front); 375 } 376 } 377 378 void i915_gem_init__objects(struct drm_i915_private *i915) 379 { 380 INIT_WORK(&i915->mm.free_work, __i915_gem_free_work); 381 } 382 383 static void i915_global_objects_shrink(void) 384 { 385 #ifdef notyet 386 kmem_cache_shrink(global.slab_objects); 387 #endif 388 } 389 390 static void i915_global_objects_exit(void) 391 { 392 #ifdef __linux__ 393 kmem_cache_destroy(global.slab_objects); 394 #else 395 pool_destroy(&global.slab_objects); 396 #endif 397 } 398 399 static struct i915_global_object global = { { 400 .shrink = i915_global_objects_shrink, 401 .exit = i915_global_objects_exit, 402 } }; 403 404 int __init i915_global_objects_init(void) 405 { 406 #ifdef __linux__ 407 global.slab_objects = 408 KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN); 409 if (!global.slab_objects) 410 return -ENOMEM; 411 #else 412 pool_init(&global.slab_objects, sizeof(struct drm_i915_gem_object), 413 0, IPL_NONE, 0, "drmobj", NULL); 414 #endif 415 416 i915_global_register(&global.base); 417 return 0; 418 } 419 420 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 421 #include "selftests/huge_gem_object.c" 422 #include "selftests/huge_pages.c" 423 #include "selftests/i915_gem_object.c" 424 #include "selftests/i915_gem_coherency.c" 425 #endif 426