1 /* 2 * Copyright © 2008-2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/shmem_fs.h> 26 #include <linux/swap.h> 27 #include <linux/pci.h> 28 #include <linux/vmalloc.h> 29 #include <drm/drmP.h> 30 #include <drm/i915_drm.h> 31 32 #include "i915_drv.h" 33 #include "i915_trace.h" 34 35 #if 0 36 static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) 37 { 38 if (!mutex_is_locked(mutex)) 39 return false; 40 41 #if defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_MUTEX_SPIN_ON_OWNER) 42 return mutex->owner == task; 43 #else 44 /* Since UP may be pre-empted, we cannot assume that we own the lock */ 45 return false; 46 #endif 47 } 48 #endif 49 50 static int num_vma_bound(struct drm_i915_gem_object *obj) 51 { 52 struct i915_vma *vma; 53 int count = 0; 54 55 list_for_each_entry(vma, &obj->vma_list, obj_link) { 56 if (drm_mm_node_allocated(&vma->node)) 57 count++; 58 if (vma->pin_count) 59 count++; 60 } 61 62 return count; 63 } 64 65 static bool swap_available(void) 66 { 67 return get_nr_swap_pages() > 0; 68 } 69 70 static bool can_release_pages(struct drm_i915_gem_object *obj) 71 { 72 /* Only shmemfs objects are backed by swap */ 73 if (!obj->base.filp) 74 return false; 75 76 /* Only report true if by unbinding the object and putting its pages 77 * we can actually make forward progress towards freeing physical 78 * pages. 79 * 80 * If the pages are pinned for any other reason than being bound 81 * to the GPU, simply unbinding from the GPU is not going to succeed 82 * in releasing our pin count on the pages themselves. 83 */ 84 if (obj->pages_pin_count != num_vma_bound(obj)) 85 return false; 86 87 /* We can only return physical pages to the system if we can either 88 * discard the contents (because the user has marked them as being 89 * purgeable) or if we can move their contents out to swap. 90 */ 91 return swap_available() || obj->madv == I915_MADV_DONTNEED; 92 } 93 94 /** 95 * i915_gem_shrink - Shrink buffer object caches 96 * @dev_priv: i915 device 97 * @target: amount of memory to make available, in pages 98 * @flags: control flags for selecting cache types 99 * 100 * This function is the main interface to the shrinker. It will try to release 101 * up to @target pages of main memory backing storage from buffer objects. 102 * Selection of the specific caches can be done with @flags. This is e.g. useful 103 * when purgeable objects should be removed from caches preferentially. 104 * 105 * Note that it's not guaranteed that released amount is actually available as 106 * free system memory - the pages might still be in-used to due to other reasons 107 * (like cpu mmaps) or the mm core has reused them before we could grab them. 108 * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to 109 * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all(). 110 * 111 * Also note that any kind of pinning (both per-vma address space pins and 112 * backing storage pins at the buffer object level) result in the shrinker code 113 * having to skip the object. 114 * 115 * Returns: 116 * The number of pages of backing storage actually released. 117 */ 118 unsigned long 119 i915_gem_shrink(struct drm_i915_private *dev_priv, 120 unsigned long target, unsigned flags) 121 { 122 const struct { 123 struct list_head *list; 124 unsigned int bit; 125 } phases[] = { 126 { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND }, 127 { &dev_priv->mm.bound_list, I915_SHRINK_BOUND }, 128 { NULL, 0 }, 129 }, *phase; 130 unsigned long count = 0; 131 132 trace_i915_gem_shrink(dev_priv, target, flags); 133 i915_gem_retire_requests(dev_priv->dev); 134 135 /* 136 * As we may completely rewrite the (un)bound list whilst unbinding 137 * (due to retiring requests) we have to strictly process only 138 * one element of the list at the time, and recheck the list 139 * on every iteration. 140 * 141 * In particular, we must hold a reference whilst removing the 142 * object as we may end up waiting for and/or retiring the objects. 143 * This might release the final reference (held by the active list) 144 * and result in the object being freed from under us. This is 145 * similar to the precautions the eviction code must take whilst 146 * removing objects. 147 * 148 * Also note that although these lists do not hold a reference to 149 * the object we can safely grab one here: The final object 150 * unreferencing and the bound_list are both protected by the 151 * dev->struct_mutex and so we won't ever be able to observe an 152 * object on the bound_list with a reference count equals 0. 153 */ 154 for (phase = phases; phase->list; phase++) { 155 struct list_head still_in_list; 156 157 if ((flags & phase->bit) == 0) 158 continue; 159 160 INIT_LIST_HEAD(&still_in_list); 161 while (count < target && !list_empty(phase->list)) { 162 struct drm_i915_gem_object *obj; 163 struct i915_vma *vma, *v; 164 165 obj = list_first_entry(phase->list, 166 typeof(*obj), global_list); 167 list_move_tail(&obj->global_list, &still_in_list); 168 169 if (flags & I915_SHRINK_PURGEABLE && 170 obj->madv != I915_MADV_DONTNEED) 171 continue; 172 173 if (flags & I915_SHRINK_VMAPS && 174 !is_vmalloc_addr(obj->mapping)) 175 continue; 176 177 if ((flags & I915_SHRINK_ACTIVE) == 0 && obj->active) 178 continue; 179 180 if (!can_release_pages(obj)) 181 continue; 182 183 drm_gem_object_reference(&obj->base); 184 185 /* For the unbound phase, this should be a no-op! */ 186 list_for_each_entry_safe(vma, v, 187 &obj->vma_list, obj_link) 188 if (i915_vma_unbind(vma)) 189 break; 190 191 if (i915_gem_object_put_pages(obj) == 0) 192 count += obj->base.size >> PAGE_SHIFT; 193 194 drm_gem_object_unreference(&obj->base); 195 } 196 list_splice(&still_in_list, phase->list); 197 } 198 199 i915_gem_retire_requests(dev_priv->dev); 200 201 return count; 202 } 203 204 /** 205 * i915_gem_shrink_all - Shrink buffer object caches completely 206 * @dev_priv: i915 device 207 * 208 * This is a simple wraper around i915_gem_shrink() to aggressively shrink all 209 * caches completely. It also first waits for and retires all outstanding 210 * requests to also be able to release backing storage for active objects. 211 * 212 * This should only be used in code to intentionally quiescent the gpu or as a 213 * last-ditch effort when memory seems to have run out. 214 * 215 * Returns: 216 * The number of pages of backing storage actually released. 217 */ 218 unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv) 219 { 220 return i915_gem_shrink(dev_priv, -1UL, 221 I915_SHRINK_BOUND | 222 I915_SHRINK_UNBOUND | 223 I915_SHRINK_ACTIVE); 224 } 225 226 #if 0 227 static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock) 228 { 229 if (!mutex_trylock(&dev->struct_mutex)) { 230 if (!mutex_is_locked_by(&dev->struct_mutex, current)) 231 return false; 232 233 if (to_i915(dev)->mm.shrinker_no_lock_stealing) 234 return false; 235 236 *unlock = false; 237 } else 238 *unlock = true; 239 240 return true; 241 } 242 #endif 243 244 static unsigned long 245 i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) 246 { 247 #if 0 248 struct drm_i915_private *dev_priv = 249 container_of(shrinker, struct drm_i915_private, mm.shrinker); 250 struct drm_device *dev = dev_priv->dev; 251 struct drm_i915_gem_object *obj; 252 unsigned long count; 253 bool unlock; 254 255 if (!i915_gem_shrinker_lock(dev, &unlock)) 256 return 0; 257 258 count = 0; 259 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) 260 if (can_release_pages(obj)) 261 count += obj->base.size >> PAGE_SHIFT; 262 263 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 264 if (!obj->active && can_release_pages(obj)) 265 count += obj->base.size >> PAGE_SHIFT; 266 } 267 268 if (unlock) 269 mutex_unlock(&dev->struct_mutex); 270 271 return count; 272 #endif 273 return 0; 274 } 275 276 static unsigned long 277 i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) 278 { 279 #if 0 280 struct drm_i915_private *dev_priv = 281 container_of(shrinker, struct drm_i915_private, mm.shrinker); 282 struct drm_device *dev = dev_priv->dev; 283 unsigned long freed; 284 bool unlock; 285 286 if (!i915_gem_shrinker_lock(dev, &unlock)) 287 return SHRINK_STOP; 288 289 freed = i915_gem_shrink(dev_priv, 290 sc->nr_to_scan, 291 I915_SHRINK_BOUND | 292 I915_SHRINK_UNBOUND | 293 I915_SHRINK_PURGEABLE); 294 if (freed < sc->nr_to_scan) 295 freed += i915_gem_shrink(dev_priv, 296 sc->nr_to_scan - freed, 297 I915_SHRINK_BOUND | 298 I915_SHRINK_UNBOUND); 299 if (unlock) 300 mutex_unlock(&dev->struct_mutex); 301 302 return freed; 303 #endif 304 return 0; 305 } 306 307 #if 0 308 struct shrinker_lock_uninterruptible { 309 bool was_interruptible; 310 bool unlock; 311 }; 312 313 static bool 314 i915_gem_shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv, 315 struct shrinker_lock_uninterruptible *slu, 316 int timeout_ms) 317 { 318 unsigned long timeout = msecs_to_jiffies(timeout_ms) + 1; 319 320 while (!i915_gem_shrinker_lock(dev_priv->dev, &slu->unlock)) { 321 schedule_timeout_killable(1); 322 if (fatal_signal_pending(current)) 323 return false; 324 if (--timeout == 0) { 325 pr_err("Unable to lock GPU to purge memory.\n"); 326 return false; 327 } 328 } 329 330 slu->was_interruptible = dev_priv->mm.interruptible; 331 dev_priv->mm.interruptible = false; 332 return true; 333 } 334 335 static void 336 i915_gem_shrinker_unlock_uninterruptible(struct drm_i915_private *dev_priv, 337 struct shrinker_lock_uninterruptible *slu) 338 { 339 dev_priv->mm.interruptible = slu->was_interruptible; 340 if (slu->unlock) 341 mutex_unlock(&dev_priv->dev->struct_mutex); 342 } 343 344 static int 345 i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) 346 { 347 struct drm_i915_private *dev_priv = 348 container_of(nb, struct drm_i915_private, mm.oom_notifier); 349 struct shrinker_lock_uninterruptible slu; 350 struct drm_i915_gem_object *obj; 351 unsigned long unevictable, bound, unbound, freed_pages; 352 353 if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000)) 354 return NOTIFY_DONE; 355 356 freed_pages = i915_gem_shrink_all(dev_priv); 357 358 /* Because we may be allocating inside our own driver, we cannot 359 * assert that there are no objects with pinned pages that are not 360 * being pointed to by hardware. 361 */ 362 unbound = bound = unevictable = 0; 363 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { 364 if (!can_release_pages(obj)) 365 unevictable += obj->base.size >> PAGE_SHIFT; 366 else 367 unbound += obj->base.size >> PAGE_SHIFT; 368 } 369 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 370 if (!can_release_pages(obj)) 371 unevictable += obj->base.size >> PAGE_SHIFT; 372 else 373 bound += obj->base.size >> PAGE_SHIFT; 374 } 375 376 i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu); 377 378 if (freed_pages || unbound || bound) 379 pr_info("Purging GPU memory, %lu pages freed, " 380 "%lu pages still pinned.\n", 381 freed_pages, unevictable); 382 if (unbound || bound) 383 pr_err("%lu and %lu pages still available in the " 384 "bound and unbound GPU page lists.\n", 385 bound, unbound); 386 387 *(unsigned long *)ptr += freed_pages; 388 return NOTIFY_DONE; 389 } 390 391 static int 392 i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr) 393 { 394 struct drm_i915_private *dev_priv = 395 container_of(nb, struct drm_i915_private, mm.vmap_notifier); 396 struct shrinker_lock_uninterruptible slu; 397 unsigned long freed_pages; 398 399 if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000)) 400 return NOTIFY_DONE; 401 402 freed_pages = i915_gem_shrink(dev_priv, -1UL, 403 I915_SHRINK_BOUND | 404 I915_SHRINK_UNBOUND | 405 I915_SHRINK_ACTIVE | 406 I915_SHRINK_VMAPS); 407 408 i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu); 409 410 *(unsigned long *)ptr += freed_pages; 411 return NOTIFY_DONE; 412 } 413 #endif 414 415 /** 416 * i915_gem_shrinker_init - Initialize i915 shrinker 417 * @dev_priv: i915 device 418 * 419 * This function registers and sets up the i915 shrinker and OOM handler. 420 */ 421 void i915_gem_shrinker_init(struct drm_i915_private *dev_priv) 422 { 423 dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan; 424 dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count; 425 dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS; 426 #if 0 427 WARN_ON(register_shrinker(&dev_priv->mm.shrinker)); 428 429 dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom; 430 WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier)); 431 432 dev_priv->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap; 433 WARN_ON(register_vmap_purge_notifier(&dev_priv->mm.vmap_notifier)); 434 #endif 435 } 436 437 /** 438 * i915_gem_shrinker_cleanup - Clean up i915 shrinker 439 * @dev_priv: i915 device 440 * 441 * This function unregisters the i915 shrinker and OOM handler. 442 */ 443 void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv) 444 { 445 #if 0 446 WARN_ON(unregister_vmap_purge_notifier(&dev_priv->mm.vmap_notifier)); 447 WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier)); 448 unregister_shrinker(&dev_priv->mm.shrinker); 449 #endif 450 } 451