1 /* 2 * Copyright © 2008-2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/shmem_fs.h> 26 #include <linux/slab.h> 27 #include <linux/swap.h> 28 #include <linux/pci.h> 29 #include <linux/dma-buf.h> 30 #include <linux/vmalloc.h> 31 #include <drm/drmP.h> 32 #include <drm/i915_drm.h> 33 34 #include "i915_drv.h" 35 #include "i915_trace.h" 36 37 #if 0 38 static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) 39 { 40 if (!mutex_is_locked(mutex)) 41 return false; 42 43 #if defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_MUTEX_SPIN_ON_OWNER) 44 return mutex->owner == task; 45 #else 46 /* Since UP may be pre-empted, we cannot assume that we own the lock */ 47 return false; 48 #endif 49 } 50 #endif 51 52 static bool any_vma_pinned(struct drm_i915_gem_object *obj) 53 { 54 struct i915_vma *vma; 55 56 list_for_each_entry(vma, &obj->vma_list, obj_link) 57 if (i915_vma_is_pinned(vma)) 58 return true; 59 60 return false; 61 } 62 63 static bool swap_available(void) 64 { 65 return get_nr_swap_pages() > 0; 66 } 67 68 static bool can_release_pages(struct drm_i915_gem_object *obj) 69 { 70 /* Only shmemfs objects are backed by swap */ 71 if (!obj->base.filp) 72 return false; 73 74 /* Only report true if by unbinding the object and putting its pages 75 * we can actually make forward progress towards freeing physical 76 * pages. 77 * 78 * If the pages are pinned for any other reason than being bound 79 * to the GPU, simply unbinding from the GPU is not going to succeed 80 * in releasing our pin count on the pages themselves. 81 */ 82 if (obj->pages_pin_count > obj->bind_count) 83 return false; 84 85 if (any_vma_pinned(obj)) 86 return false; 87 88 /* We can only return physical pages to the system if we can either 89 * discard the contents (because the user has marked them as being 90 * purgeable) or if we can move their contents out to swap. 91 */ 92 return swap_available() || obj->madv == I915_MADV_DONTNEED; 93 } 94 95 /** 96 * i915_gem_shrink - Shrink buffer object caches 97 * @dev_priv: i915 device 98 * @target: amount of memory to make available, in pages 99 * @flags: control flags for selecting cache types 100 * 101 * This function is the main interface to the shrinker. It will try to release 102 * up to @target pages of main memory backing storage from buffer objects. 103 * Selection of the specific caches can be done with @flags. This is e.g. useful 104 * when purgeable objects should be removed from caches preferentially. 105 * 106 * Note that it's not guaranteed that released amount is actually available as 107 * free system memory - the pages might still be in-used to due to other reasons 108 * (like cpu mmaps) or the mm core has reused them before we could grab them. 109 * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to 110 * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all(). 111 * 112 * Also note that any kind of pinning (both per-vma address space pins and 113 * backing storage pins at the buffer object level) result in the shrinker code 114 * having to skip the object. 115 * 116 * Returns: 117 * The number of pages of backing storage actually released. 118 */ 119 unsigned long 120 i915_gem_shrink(struct drm_i915_private *dev_priv, 121 unsigned long target, unsigned flags) 122 { 123 const struct { 124 struct list_head *list; 125 unsigned int bit; 126 } phases[] = { 127 { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND }, 128 { &dev_priv->mm.bound_list, I915_SHRINK_BOUND }, 129 { NULL, 0 }, 130 }, *phase; 131 unsigned long count = 0; 132 133 trace_i915_gem_shrink(dev_priv, target, flags); 134 i915_gem_retire_requests(dev_priv); 135 136 /* 137 * Unbinding of objects will require HW access; Let us not wake the 138 * device just to recover a little memory. If absolutely necessary, 139 * we will force the wake during oom-notifier. 140 */ 141 if ((flags & I915_SHRINK_BOUND) && 142 !intel_runtime_pm_get_if_in_use(dev_priv)) 143 flags &= ~I915_SHRINK_BOUND; 144 145 /* 146 * As we may completely rewrite the (un)bound list whilst unbinding 147 * (due to retiring requests) we have to strictly process only 148 * one element of the list at the time, and recheck the list 149 * on every iteration. 150 * 151 * In particular, we must hold a reference whilst removing the 152 * object as we may end up waiting for and/or retiring the objects. 153 * This might release the final reference (held by the active list) 154 * and result in the object being freed from under us. This is 155 * similar to the precautions the eviction code must take whilst 156 * removing objects. 157 * 158 * Also note that although these lists do not hold a reference to 159 * the object we can safely grab one here: The final object 160 * unreferencing and the bound_list are both protected by the 161 * dev->struct_mutex and so we won't ever be able to observe an 162 * object on the bound_list with a reference count equals 0. 163 */ 164 for (phase = phases; phase->list; phase++) { 165 struct list_head still_in_list; 166 struct drm_i915_gem_object *obj; 167 168 if ((flags & phase->bit) == 0) 169 continue; 170 171 INIT_LIST_HEAD(&still_in_list); 172 while (count < target && 173 (obj = list_first_entry_or_null(phase->list, 174 typeof(*obj), 175 global_list))) { 176 list_move_tail(&obj->global_list, &still_in_list); 177 178 if (flags & I915_SHRINK_PURGEABLE && 179 obj->madv != I915_MADV_DONTNEED) 180 continue; 181 182 if (flags & I915_SHRINK_VMAPS && 183 !is_vmalloc_addr(obj->mapping)) 184 continue; 185 186 if (!(flags & I915_SHRINK_ACTIVE) && 187 (i915_gem_object_is_active(obj) || 188 obj->framebuffer_references)) 189 continue; 190 191 if (!can_release_pages(obj)) 192 continue; 193 194 i915_gem_object_get(obj); 195 196 /* For the unbound phase, this should be a no-op! */ 197 i915_gem_object_unbind(obj); 198 if (i915_gem_object_put_pages(obj) == 0) 199 count += obj->base.size >> PAGE_SHIFT; 200 201 i915_gem_object_put(obj); 202 } 203 list_splice(&still_in_list, phase->list); 204 } 205 206 if (flags & I915_SHRINK_BOUND) 207 intel_runtime_pm_put(dev_priv); 208 209 i915_gem_retire_requests(dev_priv); 210 /* expedite the RCU grace period to free some request slabs */ 211 synchronize_rcu_expedited(); 212 213 return count; 214 } 215 216 /** 217 * i915_gem_shrink_all - Shrink buffer object caches completely 218 * @dev_priv: i915 device 219 * 220 * This is a simple wraper around i915_gem_shrink() to aggressively shrink all 221 * caches completely. It also first waits for and retires all outstanding 222 * requests to also be able to release backing storage for active objects. 223 * 224 * This should only be used in code to intentionally quiescent the gpu or as a 225 * last-ditch effort when memory seems to have run out. 226 * 227 * Returns: 228 * The number of pages of backing storage actually released. 229 */ 230 unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv) 231 { 232 unsigned long freed; 233 234 freed = i915_gem_shrink(dev_priv, -1UL, 235 I915_SHRINK_BOUND | 236 I915_SHRINK_UNBOUND | 237 I915_SHRINK_ACTIVE); 238 rcu_barrier(); /* wait until our RCU delayed slab frees are completed */ 239 240 return freed; 241 } 242 243 #if 0 244 static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock) 245 { 246 if (!mutex_trylock(&dev->struct_mutex)) { 247 if (!mutex_is_locked_by(&dev->struct_mutex, current)) 248 return false; 249 250 *unlock = false; 251 } else 252 *unlock = true; 253 254 return true; 255 } 256 #endif 257 258 static unsigned long 259 i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) 260 { 261 #if 0 262 struct drm_i915_private *dev_priv = 263 container_of(shrinker, struct drm_i915_private, mm.shrinker); 264 struct drm_device *dev = &dev_priv->drm; 265 struct drm_i915_gem_object *obj; 266 unsigned long count; 267 bool unlock; 268 269 if (!i915_gem_shrinker_lock(dev, &unlock)) 270 return 0; 271 272 i915_gem_retire_requests(dev_priv); 273 274 count = 0; 275 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) 276 if (can_release_pages(obj)) 277 count += obj->base.size >> PAGE_SHIFT; 278 279 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 280 if (!i915_gem_object_is_active(obj) && can_release_pages(obj)) 281 count += obj->base.size >> PAGE_SHIFT; 282 } 283 284 if (unlock) 285 mutex_unlock(&dev->struct_mutex); 286 287 return count; 288 #endif 289 return 0; 290 } 291 292 static unsigned long 293 i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) 294 { 295 #if 0 296 struct drm_i915_private *dev_priv = 297 container_of(shrinker, struct drm_i915_private, mm.shrinker); 298 struct drm_device *dev = &dev_priv->drm; 299 unsigned long freed; 300 bool unlock; 301 302 if (!i915_gem_shrinker_lock(dev, &unlock)) 303 return SHRINK_STOP; 304 305 freed = i915_gem_shrink(dev_priv, 306 sc->nr_to_scan, 307 I915_SHRINK_BOUND | 308 I915_SHRINK_UNBOUND | 309 I915_SHRINK_PURGEABLE); 310 if (freed < sc->nr_to_scan) 311 freed += i915_gem_shrink(dev_priv, 312 sc->nr_to_scan - freed, 313 I915_SHRINK_BOUND | 314 I915_SHRINK_UNBOUND); 315 if (unlock) 316 mutex_unlock(&dev->struct_mutex); 317 318 return freed; 319 #endif 320 return 0; 321 } 322 323 #if 0 324 struct shrinker_lock_uninterruptible { 325 bool was_interruptible; 326 bool unlock; 327 }; 328 329 static bool 330 i915_gem_shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv, 331 struct shrinker_lock_uninterruptible *slu, 332 int timeout_ms) 333 { 334 unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms); 335 336 do { 337 if (i915_gem_wait_for_idle(dev_priv, 0) == 0 && 338 i915_gem_shrinker_lock(&dev_priv->drm, &slu->unlock)) 339 break; 340 341 schedule_timeout_killable(1); 342 if (fatal_signal_pending(current)) 343 return false; 344 345 if (time_after(jiffies, timeout)) { 346 pr_err("Unable to lock GPU to purge memory.\n"); 347 return false; 348 } 349 } while (1); 350 351 slu->was_interruptible = dev_priv->mm.interruptible; 352 dev_priv->mm.interruptible = false; 353 return true; 354 } 355 356 static void 357 i915_gem_shrinker_unlock_uninterruptible(struct drm_i915_private *dev_priv, 358 struct shrinker_lock_uninterruptible *slu) 359 { 360 dev_priv->mm.interruptible = slu->was_interruptible; 361 if (slu->unlock) 362 mutex_unlock(&dev_priv->drm.struct_mutex); 363 } 364 365 static int 366 i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) 367 { 368 struct drm_i915_private *dev_priv = 369 container_of(nb, struct drm_i915_private, mm.oom_notifier); 370 struct shrinker_lock_uninterruptible slu; 371 struct drm_i915_gem_object *obj; 372 unsigned long unevictable, bound, unbound, freed_pages; 373 374 if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000)) 375 return NOTIFY_DONE; 376 377 intel_runtime_pm_get(dev_priv); 378 freed_pages = i915_gem_shrink_all(dev_priv); 379 intel_runtime_pm_put(dev_priv); 380 381 /* Because we may be allocating inside our own driver, we cannot 382 * assert that there are no objects with pinned pages that are not 383 * being pointed to by hardware. 384 */ 385 unbound = bound = unevictable = 0; 386 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { 387 if (!can_release_pages(obj)) 388 unevictable += obj->base.size >> PAGE_SHIFT; 389 else 390 unbound += obj->base.size >> PAGE_SHIFT; 391 } 392 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 393 if (!can_release_pages(obj)) 394 unevictable += obj->base.size >> PAGE_SHIFT; 395 else 396 bound += obj->base.size >> PAGE_SHIFT; 397 } 398 399 i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu); 400 401 if (freed_pages || unbound || bound) 402 pr_info("Purging GPU memory, %lu pages freed, " 403 "%lu pages still pinned.\n", 404 freed_pages, unevictable); 405 if (unbound || bound) 406 pr_err("%lu and %lu pages still available in the " 407 "bound and unbound GPU page lists.\n", 408 bound, unbound); 409 410 *(unsigned long *)ptr += freed_pages; 411 return NOTIFY_DONE; 412 } 413 414 static int 415 i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr) 416 { 417 struct drm_i915_private *dev_priv = 418 container_of(nb, struct drm_i915_private, mm.vmap_notifier); 419 struct shrinker_lock_uninterruptible slu; 420 struct i915_vma *vma, *next; 421 unsigned long freed_pages = 0; 422 int ret; 423 424 if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000)) 425 return NOTIFY_DONE; 426 427 /* Force everything onto the inactive lists */ 428 ret = i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED); 429 if (ret) 430 goto out; 431 432 intel_runtime_pm_get(dev_priv); 433 freed_pages += i915_gem_shrink(dev_priv, -1UL, 434 I915_SHRINK_BOUND | 435 I915_SHRINK_UNBOUND | 436 I915_SHRINK_ACTIVE | 437 I915_SHRINK_VMAPS); 438 intel_runtime_pm_put(dev_priv); 439 440 /* We also want to clear any cached iomaps as they wrap vmap */ 441 list_for_each_entry_safe(vma, next, 442 &dev_priv->ggtt.base.inactive_list, vm_link) { 443 unsigned long count = vma->node.size >> PAGE_SHIFT; 444 if (vma->iomap && i915_vma_unbind(vma) == 0) 445 freed_pages += count; 446 } 447 448 out: 449 i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu); 450 451 *(unsigned long *)ptr += freed_pages; 452 return NOTIFY_DONE; 453 } 454 #endif 455 456 /** 457 * i915_gem_shrinker_init - Initialize i915 shrinker 458 * @dev_priv: i915 device 459 * 460 * This function registers and sets up the i915 shrinker and OOM handler. 461 */ 462 void i915_gem_shrinker_init(struct drm_i915_private *dev_priv) 463 { 464 dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan; 465 dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count; 466 dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS; 467 #if 0 468 WARN_ON(register_shrinker(&dev_priv->mm.shrinker)); 469 470 dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom; 471 WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier)); 472 473 dev_priv->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap; 474 WARN_ON(register_vmap_purge_notifier(&dev_priv->mm.vmap_notifier)); 475 #endif 476 } 477 478 /** 479 * i915_gem_shrinker_cleanup - Clean up i915 shrinker 480 * @dev_priv: i915 device 481 * 482 * This function unregisters the i915 shrinker and OOM handler. 483 */ 484 void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv) 485 { 486 #if 0 487 WARN_ON(unregister_vmap_purge_notifier(&dev_priv->mm.vmap_notifier)); 488 WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier)); 489 unregister_shrinker(&dev_priv->mm.shrinker); 490 #endif 491 } 492