1 /* 2 * Copyright © 2008-2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/shmem_fs.h> 26 #include <linux/swap.h> 27 #include <linux/pci.h> 28 #include <drm/drmP.h> 29 #include <drm/i915_drm.h> 30 31 #include "i915_drv.h" 32 #include "i915_trace.h" 33 34 #if 0 35 static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) 36 { 37 if (!mutex_is_locked(mutex)) 38 return false; 39 40 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES) 41 return mutex->owner == task; 42 #else 43 /* Since UP may be pre-empted, we cannot assume that we own the lock */ 44 return false; 45 #endif 46 } 47 #endif 48 49 /** 50 * i915_gem_shrink - Shrink buffer object caches 51 * @dev_priv: i915 device 52 * @target: amount of memory to make available, in pages 53 * @flags: control flags for selecting cache types 54 * 55 * This function is the main interface to the shrinker. It will try to release 56 * up to @target pages of main memory backing storage from buffer objects. 57 * Selection of the specific caches can be done with @flags. This is e.g. useful 58 * when purgeable objects should be removed from caches preferentially. 59 * 60 * Note that it's not guaranteed that released amount is actually available as 61 * free system memory - the pages might still be in-used to due to other reasons 62 * (like cpu mmaps) or the mm core has reused them before we could grab them. 63 * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to 64 * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all(). 65 * 66 * Also note that any kind of pinning (both per-vma address space pins and 67 * backing storage pins at the buffer object level) result in the shrinker code 68 * having to skip the object. 69 * 70 * Returns: 71 * The number of pages of backing storage actually released. 72 */ 73 unsigned long 74 i915_gem_shrink(struct drm_i915_private *dev_priv, 75 long target, unsigned flags) 76 { 77 const struct { 78 struct list_head *list; 79 unsigned int bit; 80 } phases[] = { 81 { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND }, 82 { &dev_priv->mm.bound_list, I915_SHRINK_BOUND }, 83 { NULL, 0 }, 84 }, *phase; 85 unsigned long count = 0; 86 87 /* 88 * As we may completely rewrite the (un)bound list whilst unbinding 89 * (due to retiring requests) we have to strictly process only 90 * one element of the list at the time, and recheck the list 91 * on every iteration. 92 * 93 * In particular, we must hold a reference whilst removing the 94 * object as we may end up waiting for and/or retiring the objects. 95 * This might release the final reference (held by the active list) 96 * and result in the object being freed from under us. This is 97 * similar to the precautions the eviction code must take whilst 98 * removing objects. 99 * 100 * Also note that although these lists do not hold a reference to 101 * the object we can safely grab one here: The final object 102 * unreferencing and the bound_list are both protected by the 103 * dev->struct_mutex and so we won't ever be able to observe an 104 * object on the bound_list with a reference count equals 0. 105 */ 106 for (phase = phases; phase->list; phase++) { 107 struct list_head still_in_list; 108 109 if ((flags & phase->bit) == 0) 110 continue; 111 112 INIT_LIST_HEAD(&still_in_list); 113 while (count < target && !list_empty(phase->list)) { 114 struct drm_i915_gem_object *obj; 115 struct i915_vma *vma, *v; 116 117 obj = list_first_entry(phase->list, 118 typeof(*obj), global_list); 119 list_move_tail(&obj->global_list, &still_in_list); 120 121 if (flags & I915_SHRINK_PURGEABLE && 122 obj->madv != I915_MADV_DONTNEED) 123 continue; 124 125 drm_gem_object_reference(&obj->base); 126 127 /* For the unbound phase, this should be a no-op! */ 128 list_for_each_entry_safe(vma, v, 129 &obj->vma_list, vma_link) 130 if (i915_vma_unbind(vma)) 131 break; 132 133 if (i915_gem_object_put_pages(obj) == 0) 134 count += obj->base.size >> PAGE_SHIFT; 135 136 drm_gem_object_unreference(&obj->base); 137 } 138 list_splice(&still_in_list, phase->list); 139 } 140 141 return count; 142 } 143 144 /** 145 * i915_gem_shrink - Shrink buffer object caches completely 146 * @dev_priv: i915 device 147 * 148 * This is a simple wraper around i915_gem_shrink() to aggressively shrink all 149 * caches completely. It also first waits for and retires all outstanding 150 * requests to also be able to release backing storage for active objects. 151 * 152 * This should only be used in code to intentionally quiescent the gpu or as a 153 * last-ditch effort when memory seems to have run out. 154 * 155 * Returns: 156 * The number of pages of backing storage actually released. 157 */ 158 unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv) 159 { 160 i915_gem_evict_everything(dev_priv->dev); 161 return i915_gem_shrink(dev_priv, LONG_MAX, 162 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND); 163 } 164 165 #if 0 166 static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock) 167 { 168 if (!mutex_trylock(&dev->struct_mutex)) { 169 if (!mutex_is_locked_by(&dev->struct_mutex, current)) 170 return false; 171 172 if (to_i915(dev)->mm.shrinker_no_lock_stealing) 173 return false; 174 175 *unlock = false; 176 } else 177 *unlock = true; 178 179 return true; 180 } 181 182 static int num_vma_bound(struct drm_i915_gem_object *obj) 183 { 184 struct i915_vma *vma; 185 int count = 0; 186 187 list_for_each_entry(vma, &obj->vma_list, vma_link) 188 if (drm_mm_node_allocated(&vma->node)) 189 count++; 190 191 return count; 192 } 193 194 static unsigned long 195 i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) 196 { 197 struct drm_i915_private *dev_priv = 198 container_of(shrinker, struct drm_i915_private, mm.shrinker); 199 struct drm_device *dev = dev_priv->dev; 200 struct drm_i915_gem_object *obj; 201 unsigned long count; 202 bool unlock; 203 204 if (!i915_gem_shrinker_lock(dev, &unlock)) 205 return 0; 206 207 count = 0; 208 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) 209 if (obj->pages_pin_count == 0) 210 count += obj->base.size >> PAGE_SHIFT; 211 212 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 213 if (!i915_gem_obj_is_pinned(obj) && 214 obj->pages_pin_count == num_vma_bound(obj)) 215 count += obj->base.size >> PAGE_SHIFT; 216 } 217 218 if (unlock) 219 mutex_unlock(&dev->struct_mutex); 220 221 return count; 222 } 223 224 static unsigned long 225 i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) 226 { 227 struct drm_i915_private *dev_priv = 228 container_of(shrinker, struct drm_i915_private, mm.shrinker); 229 struct drm_device *dev = dev_priv->dev; 230 unsigned long freed; 231 bool unlock; 232 233 if (!i915_gem_shrinker_lock(dev, &unlock)) 234 return SHRINK_STOP; 235 236 freed = i915_gem_shrink(dev_priv, 237 sc->nr_to_scan, 238 I915_SHRINK_BOUND | 239 I915_SHRINK_UNBOUND | 240 I915_SHRINK_PURGEABLE); 241 if (freed < sc->nr_to_scan) 242 freed += i915_gem_shrink(dev_priv, 243 sc->nr_to_scan - freed, 244 I915_SHRINK_BOUND | 245 I915_SHRINK_UNBOUND); 246 if (unlock) 247 mutex_unlock(&dev->struct_mutex); 248 249 return freed; 250 } 251 252 static int 253 i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) 254 { 255 struct drm_i915_private *dev_priv = 256 container_of(nb, struct drm_i915_private, mm.oom_notifier); 257 struct drm_device *dev = dev_priv->dev; 258 struct drm_i915_gem_object *obj; 259 unsigned long timeout = msecs_to_jiffies(5000) + 1; 260 unsigned long pinned, bound, unbound, freed_pages; 261 bool was_interruptible; 262 bool unlock; 263 264 while (!i915_gem_shrinker_lock(dev, &unlock) && --timeout) { 265 schedule_timeout_killable(1); 266 if (fatal_signal_pending(current)) 267 return NOTIFY_DONE; 268 } 269 if (timeout == 0) { 270 pr_err("Unable to purge GPU memory due lock contention.\n"); 271 return NOTIFY_DONE; 272 } 273 274 was_interruptible = dev_priv->mm.interruptible; 275 dev_priv->mm.interruptible = false; 276 277 freed_pages = i915_gem_shrink_all(dev_priv); 278 279 dev_priv->mm.interruptible = was_interruptible; 280 281 /* Because we may be allocating inside our own driver, we cannot 282 * assert that there are no objects with pinned pages that are not 283 * being pointed to by hardware. 284 */ 285 unbound = bound = pinned = 0; 286 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { 287 if (!obj->base.filp) /* not backed by a freeable object */ 288 continue; 289 290 if (obj->pages_pin_count) 291 pinned += obj->base.size; 292 else 293 unbound += obj->base.size; 294 } 295 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 296 if (!obj->base.filp) 297 continue; 298 299 if (obj->pages_pin_count) 300 pinned += obj->base.size; 301 else 302 bound += obj->base.size; 303 } 304 305 if (unlock) 306 mutex_unlock(&dev->struct_mutex); 307 308 if (freed_pages || unbound || bound) 309 pr_info("Purging GPU memory, %lu bytes freed, %lu bytes still pinned.\n", 310 freed_pages << PAGE_SHIFT, pinned); 311 if (unbound || bound) 312 pr_err("%lu and %lu bytes still available in the " 313 "bound and unbound GPU page lists.\n", 314 bound, unbound); 315 316 *(unsigned long *)ptr += freed_pages; 317 return NOTIFY_DONE; 318 } 319 #endif 320 321 /** 322 * i915_gem_shrinker_init - Initialize i915 shrinker 323 * @dev_priv: i915 device 324 * 325 * This function registers and sets up the i915 shrinker and OOM handler. 326 */ 327 void i915_gem_shrinker_init(struct drm_i915_private *dev_priv) 328 { 329 #if 0 330 dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan; 331 dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count; 332 dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS; 333 register_shrinker(&dev_priv->mm.shrinker); 334 335 dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom; 336 register_oom_notifier(&dev_priv->mm.oom_notifier); 337 #endif 338 } 339