1 /* 2 * Copyright © 2008-2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/oom.h> 26 #include <linux/shmem_fs.h> 27 #include <linux/slab.h> 28 #include <linux/swap.h> 29 #include <linux/pci.h> 30 #include <linux/dma-buf.h> 31 #include <linux/vmalloc.h> 32 #include <drm/drmP.h> 33 #include <drm/i915_drm.h> 34 35 #include "i915_drv.h" 36 #include "i915_trace.h" 37 38 static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock) 39 { 40 switch (mutex_trylock_recursive(&dev->struct_mutex)) { 41 case MUTEX_TRYLOCK_FAILED: 42 return false; 43 44 case MUTEX_TRYLOCK_SUCCESS: 45 *unlock = true; 46 return true; 47 48 case MUTEX_TRYLOCK_RECURSIVE: 49 *unlock = false; 50 return true; 51 } 52 53 BUG(); 54 } 55 56 57 static void i915_gem_shrinker_unlock(struct drm_device *dev, bool unlock) 58 { 59 if (!unlock) 60 return; 61 62 mutex_unlock(&dev->struct_mutex); 63 } 64 65 static bool any_vma_pinned(struct drm_i915_gem_object *obj) 66 { 67 struct i915_vma *vma; 68 69 list_for_each_entry(vma, &obj->vma_list, obj_link) 70 if (i915_vma_is_pinned(vma)) 71 return true; 72 73 return false; 74 } 75 76 static bool swap_available(void) 77 { 78 return get_nr_swap_pages() > 0; 79 } 80 81 static bool can_release_pages(struct drm_i915_gem_object *obj) 82 { 83 if (!obj->mm.pages) 84 return false; 85 86 /* Consider only shrinkable ojects. */ 87 if (!i915_gem_object_is_shrinkable(obj)) 88 return false; 89 90 /* Only report true if by unbinding the object and putting its pages 91 * we can actually make forward progress towards freeing physical 92 * pages. 93 * 94 * If the pages are pinned for any other reason than being bound 95 * to the GPU, simply unbinding from the GPU is not going to succeed 96 * in releasing our pin count on the pages themselves. 97 */ 98 if (atomic_read(&obj->mm.pages_pin_count) > obj->bind_count) 99 return false; 100 101 if (any_vma_pinned(obj)) 102 return false; 103 104 /* We can only return physical pages to the system if we can either 105 * discard the contents (because the user has marked them as being 106 * purgeable) or if we can move their contents out to swap. 107 */ 108 return swap_available() || obj->mm.madv == I915_MADV_DONTNEED; 109 } 110 111 static bool unsafe_drop_pages(struct drm_i915_gem_object *obj) 112 { 113 if (i915_gem_object_unbind(obj) == 0) 114 __i915_gem_object_put_pages(obj, I915_MM_SHRINKER); 115 return !READ_ONCE(obj->mm.pages); 116 } 117 118 /** 119 * i915_gem_shrink - Shrink buffer object caches 120 * @dev_priv: i915 device 121 * @target: amount of memory to make available, in pages 122 * @flags: control flags for selecting cache types 123 * 124 * This function is the main interface to the shrinker. It will try to release 125 * up to @target pages of main memory backing storage from buffer objects. 126 * Selection of the specific caches can be done with @flags. This is e.g. useful 127 * when purgeable objects should be removed from caches preferentially. 128 * 129 * Note that it's not guaranteed that released amount is actually available as 130 * free system memory - the pages might still be in-used to due to other reasons 131 * (like cpu mmaps) or the mm core has reused them before we could grab them. 132 * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to 133 * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all(). 134 * 135 * Also note that any kind of pinning (both per-vma address space pins and 136 * backing storage pins at the buffer object level) result in the shrinker code 137 * having to skip the object. 138 * 139 * Returns: 140 * The number of pages of backing storage actually released. 141 */ 142 unsigned long 143 i915_gem_shrink(struct drm_i915_private *dev_priv, 144 unsigned long target, unsigned flags) 145 { 146 const struct { 147 struct list_head *list; 148 unsigned int bit; 149 } phases[] = { 150 { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND }, 151 { &dev_priv->mm.bound_list, I915_SHRINK_BOUND }, 152 { NULL, 0 }, 153 }, *phase; 154 unsigned long count = 0; 155 bool unlock; 156 157 if (!i915_gem_shrinker_lock(&dev_priv->drm, &unlock)) 158 return 0; 159 160 trace_i915_gem_shrink(dev_priv, target, flags); 161 i915_gem_retire_requests(dev_priv); 162 163 /* 164 * Unbinding of objects will require HW access; Let us not wake the 165 * device just to recover a little memory. If absolutely necessary, 166 * we will force the wake during oom-notifier. 167 */ 168 if ((flags & I915_SHRINK_BOUND) && 169 !intel_runtime_pm_get_if_in_use(dev_priv)) 170 flags &= ~I915_SHRINK_BOUND; 171 172 /* 173 * As we may completely rewrite the (un)bound list whilst unbinding 174 * (due to retiring requests) we have to strictly process only 175 * one element of the list at the time, and recheck the list 176 * on every iteration. 177 * 178 * In particular, we must hold a reference whilst removing the 179 * object as we may end up waiting for and/or retiring the objects. 180 * This might release the final reference (held by the active list) 181 * and result in the object being freed from under us. This is 182 * similar to the precautions the eviction code must take whilst 183 * removing objects. 184 * 185 * Also note that although these lists do not hold a reference to 186 * the object we can safely grab one here: The final object 187 * unreferencing and the bound_list are both protected by the 188 * dev->struct_mutex and so we won't ever be able to observe an 189 * object on the bound_list with a reference count equals 0. 190 */ 191 for (phase = phases; phase->list; phase++) { 192 struct list_head still_in_list; 193 struct drm_i915_gem_object *obj; 194 195 if ((flags & phase->bit) == 0) 196 continue; 197 198 INIT_LIST_HEAD(&still_in_list); 199 while (count < target && 200 (obj = list_first_entry_or_null(phase->list, 201 typeof(*obj), 202 global_link))) { 203 list_move_tail(&obj->global_link, &still_in_list); 204 if (!obj->mm.pages) { 205 list_del_init(&obj->global_link); 206 continue; 207 } 208 209 if (flags & I915_SHRINK_PURGEABLE && 210 obj->mm.madv != I915_MADV_DONTNEED) 211 continue; 212 213 if (flags & I915_SHRINK_VMAPS && 214 !is_vmalloc_addr(obj->mm.mapping)) 215 continue; 216 217 if (!(flags & I915_SHRINK_ACTIVE) && 218 (i915_gem_object_is_active(obj) || 219 i915_gem_object_is_framebuffer(obj))) 220 continue; 221 222 if (!can_release_pages(obj)) 223 continue; 224 225 if (unsafe_drop_pages(obj)) { 226 /* May arrive from get_pages on another bo */ 227 mutex_lock_nested(&obj->mm.lock, 228 I915_MM_SHRINKER); 229 if (!obj->mm.pages) { 230 __i915_gem_object_invalidate(obj); 231 list_del_init(&obj->global_link); 232 count += obj->base.size >> PAGE_SHIFT; 233 } 234 mutex_unlock(&obj->mm.lock); 235 } 236 } 237 list_splice_tail(&still_in_list, phase->list); 238 } 239 240 if (flags & I915_SHRINK_BOUND) 241 intel_runtime_pm_put(dev_priv); 242 243 i915_gem_retire_requests(dev_priv); 244 245 i915_gem_shrinker_unlock(&dev_priv->drm, unlock); 246 247 return count; 248 } 249 250 /** 251 * i915_gem_shrink_all - Shrink buffer object caches completely 252 * @dev_priv: i915 device 253 * 254 * This is a simple wraper around i915_gem_shrink() to aggressively shrink all 255 * caches completely. It also first waits for and retires all outstanding 256 * requests to also be able to release backing storage for active objects. 257 * 258 * This should only be used in code to intentionally quiescent the gpu or as a 259 * last-ditch effort when memory seems to have run out. 260 * 261 * Returns: 262 * The number of pages of backing storage actually released. 263 */ 264 unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv) 265 { 266 unsigned long freed; 267 268 intel_runtime_pm_get(dev_priv); 269 freed = i915_gem_shrink(dev_priv, -1UL, 270 I915_SHRINK_BOUND | 271 I915_SHRINK_UNBOUND | 272 I915_SHRINK_ACTIVE); 273 intel_runtime_pm_put(dev_priv); 274 275 return freed; 276 } 277 278 static unsigned long 279 i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) 280 { 281 #if 0 282 struct drm_i915_private *dev_priv = 283 container_of(shrinker, struct drm_i915_private, mm.shrinker); 284 struct drm_device *dev = &dev_priv->drm; 285 struct drm_i915_gem_object *obj; 286 unsigned long count; 287 bool unlock; 288 289 if (!i915_gem_shrinker_lock(dev, &unlock)) 290 return 0; 291 292 i915_gem_retire_requests(dev_priv); 293 294 count = 0; 295 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) 296 if (can_release_pages(obj)) 297 count += obj->base.size >> PAGE_SHIFT; 298 299 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { 300 if (!i915_gem_object_is_active(obj) && can_release_pages(obj)) 301 count += obj->base.size >> PAGE_SHIFT; 302 } 303 304 i915_gem_shrinker_unlock(dev, unlock); 305 306 return count; 307 #endif 308 return 0; 309 } 310 311 static unsigned long 312 i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) 313 { 314 #if 0 315 struct drm_i915_private *dev_priv = 316 container_of(shrinker, struct drm_i915_private, mm.shrinker); 317 struct drm_device *dev = &dev_priv->drm; 318 unsigned long freed; 319 bool unlock; 320 321 if (!i915_gem_shrinker_lock(dev, &unlock)) 322 return SHRINK_STOP; 323 324 freed = i915_gem_shrink(dev_priv, 325 sc->nr_to_scan, 326 I915_SHRINK_BOUND | 327 I915_SHRINK_UNBOUND | 328 I915_SHRINK_PURGEABLE); 329 if (freed < sc->nr_to_scan) 330 freed += i915_gem_shrink(dev_priv, 331 sc->nr_to_scan - freed, 332 I915_SHRINK_BOUND | 333 I915_SHRINK_UNBOUND); 334 335 i915_gem_shrinker_unlock(dev, unlock); 336 337 return freed; 338 #endif 339 return 0; 340 } 341 342 #if 0 343 struct shrinker_lock_uninterruptible { 344 bool was_interruptible; 345 bool unlock; 346 }; 347 348 static bool 349 i915_gem_shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv, 350 struct shrinker_lock_uninterruptible *slu, 351 int timeout_ms) 352 { 353 unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms); 354 355 do { 356 if (i915_gem_wait_for_idle(dev_priv, 0) == 0 && 357 i915_gem_shrinker_lock(&dev_priv->drm, &slu->unlock)) 358 break; 359 360 schedule_timeout_killable(1); 361 if (fatal_signal_pending(current)) 362 return false; 363 364 if (time_after(jiffies, timeout)) { 365 pr_err("Unable to lock GPU to purge memory.\n"); 366 return false; 367 } 368 } while (1); 369 370 slu->was_interruptible = dev_priv->mm.interruptible; 371 dev_priv->mm.interruptible = false; 372 return true; 373 } 374 375 static void 376 i915_gem_shrinker_unlock_uninterruptible(struct drm_i915_private *dev_priv, 377 struct shrinker_lock_uninterruptible *slu) 378 { 379 dev_priv->mm.interruptible = slu->was_interruptible; 380 i915_gem_shrinker_unlock(&dev_priv->drm, slu->unlock); 381 } 382 383 static int 384 i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) 385 { 386 struct drm_i915_private *dev_priv = 387 container_of(nb, struct drm_i915_private, mm.oom_notifier); 388 struct shrinker_lock_uninterruptible slu; 389 struct drm_i915_gem_object *obj; 390 unsigned long unevictable, bound, unbound, freed_pages; 391 392 if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000)) 393 return NOTIFY_DONE; 394 395 freed_pages = i915_gem_shrink_all(dev_priv); 396 397 /* Because we may be allocating inside our own driver, we cannot 398 * assert that there are no objects with pinned pages that are not 399 * being pointed to by hardware. 400 */ 401 unbound = bound = unevictable = 0; 402 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) { 403 if (!obj->mm.pages) 404 continue; 405 406 if (!can_release_pages(obj)) 407 unevictable += obj->base.size >> PAGE_SHIFT; 408 else 409 unbound += obj->base.size >> PAGE_SHIFT; 410 } 411 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { 412 if (!obj->mm.pages) 413 continue; 414 415 if (!can_release_pages(obj)) 416 unevictable += obj->base.size >> PAGE_SHIFT; 417 else 418 bound += obj->base.size >> PAGE_SHIFT; 419 } 420 421 i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu); 422 423 if (freed_pages || unbound || bound) 424 pr_info("Purging GPU memory, %lu pages freed, " 425 "%lu pages still pinned.\n", 426 freed_pages, unevictable); 427 if (unbound || bound) 428 pr_err("%lu and %lu pages still available in the " 429 "bound and unbound GPU page lists.\n", 430 bound, unbound); 431 432 *(unsigned long *)ptr += freed_pages; 433 return NOTIFY_DONE; 434 } 435 436 static int 437 i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr) 438 { 439 struct drm_i915_private *dev_priv = 440 container_of(nb, struct drm_i915_private, mm.vmap_notifier); 441 struct shrinker_lock_uninterruptible slu; 442 struct i915_vma *vma, *next; 443 unsigned long freed_pages = 0; 444 int ret; 445 446 if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000)) 447 return NOTIFY_DONE; 448 449 /* Force everything onto the inactive lists */ 450 ret = i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED); 451 if (ret) 452 goto out; 453 454 intel_runtime_pm_get(dev_priv); 455 freed_pages += i915_gem_shrink(dev_priv, -1UL, 456 I915_SHRINK_BOUND | 457 I915_SHRINK_UNBOUND | 458 I915_SHRINK_ACTIVE | 459 I915_SHRINK_VMAPS); 460 intel_runtime_pm_put(dev_priv); 461 462 /* We also want to clear any cached iomaps as they wrap vmap */ 463 list_for_each_entry_safe(vma, next, 464 &dev_priv->ggtt.base.inactive_list, vm_link) { 465 unsigned long count = vma->node.size >> PAGE_SHIFT; 466 if (vma->iomap && i915_vma_unbind(vma) == 0) 467 freed_pages += count; 468 } 469 470 out: 471 i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu); 472 473 *(unsigned long *)ptr += freed_pages; 474 return NOTIFY_DONE; 475 } 476 #endif 477 478 /** 479 * i915_gem_shrinker_init - Initialize i915 shrinker 480 * @dev_priv: i915 device 481 * 482 * This function registers and sets up the i915 shrinker and OOM handler. 483 */ 484 void i915_gem_shrinker_init(struct drm_i915_private *dev_priv) 485 { 486 dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan; 487 dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count; 488 dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS; 489 #if 0 490 WARN_ON(register_shrinker(&dev_priv->mm.shrinker)); 491 492 dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom; 493 WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier)); 494 495 dev_priv->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap; 496 WARN_ON(register_vmap_purge_notifier(&dev_priv->mm.vmap_notifier)); 497 #endif 498 } 499 500 /** 501 * i915_gem_shrinker_cleanup - Clean up i915 shrinker 502 * @dev_priv: i915 device 503 * 504 * This function unregisters the i915 shrinker and OOM handler. 505 */ 506 void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv) 507 { 508 #if 0 509 WARN_ON(unregister_vmap_purge_notifier(&dev_priv->mm.vmap_notifier)); 510 WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier)); 511 unregister_shrinker(&dev_priv->mm.shrinker); 512 #endif 513 } 514