1 /* 2 * Copyright © 2008-2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/shmem_fs.h> 26 #include <linux/swap.h> 27 #include <linux/pci.h> 28 #include <linux/vmalloc.h> 29 #include <drm/drmP.h> 30 #include <drm/i915_drm.h> 31 32 #include "i915_drv.h" 33 #include "i915_trace.h" 34 35 #if 0 36 static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) 37 { 38 if (!mutex_is_locked(mutex)) 39 return false; 40 41 #if defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_MUTEX_SPIN_ON_OWNER) 42 return mutex->owner == task; 43 #else 44 /* Since UP may be pre-empted, we cannot assume that we own the lock */ 45 return false; 46 #endif 47 } 48 #endif 49 50 static int num_vma_bound(struct drm_i915_gem_object *obj) 51 { 52 struct i915_vma *vma; 53 int count = 0; 54 55 list_for_each_entry(vma, &obj->vma_list, obj_link) { 56 if (drm_mm_node_allocated(&vma->node)) 57 count++; 58 if (vma->pin_count) 59 count++; 60 } 61 62 return count; 63 } 64 65 static bool swap_available(void) 66 { 67 return get_nr_swap_pages() > 0; 68 } 69 70 static bool can_release_pages(struct drm_i915_gem_object *obj) 71 { 72 /* Only shmemfs objects are backed by swap */ 73 if (!obj->base.vm_obj) 74 return false; 75 76 /* Only report true if by unbinding the object and putting its pages 77 * we can actually make forward progress towards freeing physical 78 * pages. 79 * 80 * If the pages are pinned for any other reason than being bound 81 * to the GPU, simply unbinding from the GPU is not going to succeed 82 * in releasing our pin count on the pages themselves. 83 */ 84 if (obj->pages_pin_count != num_vma_bound(obj)) 85 return false; 86 87 /* We can only return physical pages to the system if we can either 88 * discard the contents (because the user has marked them as being 89 * purgeable) or if we can move their contents out to swap. 90 */ 91 return swap_available() || obj->madv == I915_MADV_DONTNEED; 92 } 93 94 /** 95 * i915_gem_shrink - Shrink buffer object caches 96 * @dev_priv: i915 device 97 * @target: amount of memory to make available, in pages 98 * @flags: control flags for selecting cache types 99 * 100 * This function is the main interface to the shrinker. It will try to release 101 * up to @target pages of main memory backing storage from buffer objects. 102 * Selection of the specific caches can be done with @flags. This is e.g. useful 103 * when purgeable objects should be removed from caches preferentially. 104 * 105 * Note that it's not guaranteed that released amount is actually available as 106 * free system memory - the pages might still be in-used to due to other reasons 107 * (like cpu mmaps) or the mm core has reused them before we could grab them. 108 * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to 109 * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all(). 110 * 111 * Also note that any kind of pinning (both per-vma address space pins and 112 * backing storage pins at the buffer object level) result in the shrinker code 113 * having to skip the object. 114 * 115 * Returns: 116 * The number of pages of backing storage actually released. 117 */ 118 unsigned long 119 i915_gem_shrink(struct drm_i915_private *dev_priv, 120 unsigned long target, unsigned flags) 121 { 122 const struct { 123 struct list_head *list; 124 unsigned int bit; 125 } phases[] = { 126 { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND }, 127 { &dev_priv->mm.bound_list, I915_SHRINK_BOUND }, 128 { NULL, 0 }, 129 }, *phase; 130 unsigned long count = 0; 131 132 trace_i915_gem_shrink(dev_priv, target, flags); 133 i915_gem_retire_requests(dev_priv->dev); 134 135 /* 136 * As we may completely rewrite the (un)bound list whilst unbinding 137 * (due to retiring requests) we have to strictly process only 138 * one element of the list at the time, and recheck the list 139 * on every iteration. 140 * 141 * In particular, we must hold a reference whilst removing the 142 * object as we may end up waiting for and/or retiring the objects. 143 * This might release the final reference (held by the active list) 144 * and result in the object being freed from under us. This is 145 * similar to the precautions the eviction code must take whilst 146 * removing objects. 147 * 148 * Also note that although these lists do not hold a reference to 149 * the object we can safely grab one here: The final object 150 * unreferencing and the bound_list are both protected by the 151 * dev->struct_mutex and so we won't ever be able to observe an 152 * object on the bound_list with a reference count equals 0. 153 */ 154 for (phase = phases; phase->list; phase++) { 155 struct list_head still_in_list; 156 157 if ((flags & phase->bit) == 0) 158 continue; 159 160 INIT_LIST_HEAD(&still_in_list); 161 while (count < target && !list_empty(phase->list)) { 162 struct drm_i915_gem_object *obj; 163 struct i915_vma *vma, *v; 164 165 obj = list_first_entry(phase->list, 166 typeof(*obj), global_list); 167 list_move_tail(&obj->global_list, &still_in_list); 168 169 if (flags & I915_SHRINK_PURGEABLE && 170 obj->madv != I915_MADV_DONTNEED) 171 continue; 172 173 if (flags & I915_SHRINK_VMAPS && 174 !is_vmalloc_addr(obj->mapping)) 175 continue; 176 177 if ((flags & I915_SHRINK_ACTIVE) == 0 && obj->active) 178 continue; 179 180 if (!can_release_pages(obj)) 181 continue; 182 183 drm_gem_object_reference(&obj->base); 184 185 /* For the unbound phase, this should be a no-op! */ 186 list_for_each_entry_safe(vma, v, 187 &obj->vma_list, obj_link) 188 if (i915_vma_unbind(vma)) 189 break; 190 191 if (i915_gem_object_put_pages(obj) == 0) 192 count += obj->base.size >> PAGE_SHIFT; 193 194 drm_gem_object_unreference(&obj->base); 195 } 196 list_splice(&still_in_list, phase->list); 197 } 198 199 i915_gem_retire_requests(dev_priv->dev); 200 201 return count; 202 } 203 204 /** 205 * i915_gem_shrink_all - Shrink buffer object caches completely 206 * @dev_priv: i915 device 207 * 208 * This is a simple wraper around i915_gem_shrink() to aggressively shrink all 209 * caches completely. It also first waits for and retires all outstanding 210 * requests to also be able to release backing storage for active objects. 211 * 212 * This should only be used in code to intentionally quiescent the gpu or as a 213 * last-ditch effort when memory seems to have run out. 214 * 215 * Returns: 216 * The number of pages of backing storage actually released. 217 */ 218 unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv) 219 { 220 return i915_gem_shrink(dev_priv, -1UL, 221 I915_SHRINK_BOUND | 222 I915_SHRINK_UNBOUND | 223 I915_SHRINK_ACTIVE); 224 } 225 226 #if 0 227 static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock) 228 { 229 if (!mutex_trylock(&dev->struct_mutex)) { 230 if (!mutex_is_locked_by(&dev->struct_mutex, current)) 231 return false; 232 233 if (to_i915(dev)->mm.shrinker_no_lock_stealing) 234 return false; 235 236 *unlock = false; 237 } else 238 *unlock = true; 239 240 return true; 241 } 242 243 static unsigned long 244 i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) 245 { 246 struct drm_i915_private *dev_priv = 247 container_of(shrinker, struct drm_i915_private, mm.shrinker); 248 struct drm_device *dev = dev_priv->dev; 249 struct drm_i915_gem_object *obj; 250 unsigned long count; 251 bool unlock; 252 253 if (!i915_gem_shrinker_lock(dev, &unlock)) 254 return 0; 255 256 count = 0; 257 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) 258 if (can_release_pages(obj)) 259 count += obj->base.size >> PAGE_SHIFT; 260 261 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 262 if (!obj->active && can_release_pages(obj)) 263 count += obj->base.size >> PAGE_SHIFT; 264 } 265 266 if (unlock) 267 mutex_unlock(&dev->struct_mutex); 268 269 return count; 270 } 271 272 static unsigned long 273 i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) 274 { 275 struct drm_i915_private *dev_priv = 276 container_of(shrinker, struct drm_i915_private, mm.shrinker); 277 struct drm_device *dev = dev_priv->dev; 278 unsigned long freed; 279 bool unlock; 280 281 if (!i915_gem_shrinker_lock(dev, &unlock)) 282 return SHRINK_STOP; 283 284 freed = i915_gem_shrink(dev_priv, 285 sc->nr_to_scan, 286 I915_SHRINK_BOUND | 287 I915_SHRINK_UNBOUND | 288 I915_SHRINK_PURGEABLE); 289 if (freed < sc->nr_to_scan) 290 freed += i915_gem_shrink(dev_priv, 291 sc->nr_to_scan - freed, 292 I915_SHRINK_BOUND | 293 I915_SHRINK_UNBOUND); 294 if (unlock) 295 mutex_unlock(&dev->struct_mutex); 296 297 return freed; 298 } 299 300 struct shrinker_lock_uninterruptible { 301 bool was_interruptible; 302 bool unlock; 303 }; 304 305 static bool 306 i915_gem_shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv, 307 struct shrinker_lock_uninterruptible *slu, 308 int timeout_ms) 309 { 310 unsigned long timeout = msecs_to_jiffies(timeout_ms) + 1; 311 312 while (!i915_gem_shrinker_lock(dev_priv->dev, &slu->unlock)) { 313 schedule_timeout_killable(1); 314 if (fatal_signal_pending(current)) 315 return false; 316 if (--timeout == 0) { 317 pr_err("Unable to lock GPU to purge memory.\n"); 318 return false; 319 } 320 } 321 322 slu->was_interruptible = dev_priv->mm.interruptible; 323 dev_priv->mm.interruptible = false; 324 return true; 325 } 326 327 static void 328 i915_gem_shrinker_unlock_uninterruptible(struct drm_i915_private *dev_priv, 329 struct shrinker_lock_uninterruptible *slu) 330 { 331 dev_priv->mm.interruptible = slu->was_interruptible; 332 if (slu->unlock) 333 mutex_unlock(&dev_priv->dev->struct_mutex); 334 } 335 336 static int 337 i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) 338 { 339 struct drm_i915_private *dev_priv = 340 container_of(nb, struct drm_i915_private, mm.oom_notifier); 341 struct shrinker_lock_uninterruptible slu; 342 struct drm_i915_gem_object *obj; 343 unsigned long unevictable, bound, unbound, freed_pages; 344 345 if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000)) 346 return NOTIFY_DONE; 347 348 freed_pages = i915_gem_shrink_all(dev_priv); 349 350 /* Because we may be allocating inside our own driver, we cannot 351 * assert that there are no objects with pinned pages that are not 352 * being pointed to by hardware. 353 */ 354 unbound = bound = unevictable = 0; 355 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { 356 if (!can_release_pages(obj)) 357 unevictable += obj->base.size >> PAGE_SHIFT; 358 else 359 unbound += obj->base.size >> PAGE_SHIFT; 360 } 361 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 362 if (!can_release_pages(obj)) 363 unevictable += obj->base.size >> PAGE_SHIFT; 364 else 365 bound += obj->base.size >> PAGE_SHIFT; 366 } 367 368 i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu); 369 370 if (freed_pages || unbound || bound) 371 pr_info("Purging GPU memory, %lu pages freed, " 372 "%lu pages still pinned.\n", 373 freed_pages, unevictable); 374 if (unbound || bound) 375 pr_err("%lu and %lu pages still available in the " 376 "bound and unbound GPU page lists.\n", 377 bound, unbound); 378 379 *(unsigned long *)ptr += freed_pages; 380 return NOTIFY_DONE; 381 } 382 383 static int 384 i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr) 385 { 386 struct drm_i915_private *dev_priv = 387 container_of(nb, struct drm_i915_private, mm.vmap_notifier); 388 struct shrinker_lock_uninterruptible slu; 389 unsigned long freed_pages; 390 391 if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000)) 392 return NOTIFY_DONE; 393 394 freed_pages = i915_gem_shrink(dev_priv, -1UL, 395 I915_SHRINK_BOUND | 396 I915_SHRINK_UNBOUND | 397 I915_SHRINK_ACTIVE | 398 I915_SHRINK_VMAPS); 399 400 i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu); 401 402 *(unsigned long *)ptr += freed_pages; 403 return NOTIFY_DONE; 404 } 405 #endif 406 407 /** 408 * i915_gem_shrinker_init - Initialize i915 shrinker 409 * @dev_priv: i915 device 410 * 411 * This function registers and sets up the i915 shrinker and OOM handler. 412 */ 413 void i915_gem_shrinker_init(struct drm_i915_private *dev_priv) 414 { 415 #if 0 416 dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan; 417 dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count; 418 dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS; 419 WARN_ON(register_shrinker(&dev_priv->mm.shrinker)); 420 421 dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom; 422 WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier)); 423 424 dev_priv->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap; 425 WARN_ON(register_vmap_purge_notifier(&dev_priv->mm.vmap_notifier)); 426 #endif 427 } 428 429 /** 430 * i915_gem_shrinker_cleanup - Clean up i915 shrinker 431 * @dev_priv: i915 device 432 * 433 * This function unregisters the i915 shrinker and OOM handler. 434 */ 435 void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv) 436 { 437 #if 0 438 WARN_ON(unregister_vmap_purge_notifier(&dev_priv->mm.vmap_notifier)); 439 WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier)); 440 unregister_shrinker(&dev_priv->mm.shrinker); 441 #endif 442 } 443