1 /* 2 * Copyright © 2008-2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/shmem_fs.h> 26 #include <linux/swap.h> 27 #include <linux/pci.h> 28 #include <drm/drmP.h> 29 #include <drm/i915_drm.h> 30 31 #include "i915_drv.h" 32 #include "i915_trace.h" 33 34 #if 0 35 static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) 36 { 37 if (!mutex_is_locked(mutex)) 38 return false; 39 40 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES) 41 return mutex->owner == task; 42 #else 43 /* Since UP may be pre-empted, we cannot assume that we own the lock */ 44 return false; 45 #endif 46 } 47 #endif 48 49 /** 50 * i915_gem_shrink - Shrink buffer object caches 51 * @dev_priv: i915 device 52 * @target: amount of memory to make available, in pages 53 * @flags: control flags for selecting cache types 54 * 55 * This function is the main interface to the shrinker. It will try to release 56 * up to @target pages of main memory backing storage from buffer objects. 57 * Selection of the specific caches can be done with @flags. This is e.g. useful 58 * when purgeable objects should be removed from caches preferentially. 59 * 60 * Note that it's not guaranteed that released amount is actually available as 61 * free system memory - the pages might still be in-used to due to other reasons 62 * (like cpu mmaps) or the mm core has reused them before we could grab them. 63 * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to 64 * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all(). 65 * 66 * Also note that any kind of pinning (both per-vma address space pins and 67 * backing storage pins at the buffer object level) result in the shrinker code 68 * having to skip the object. 69 * 70 * Returns: 71 * The number of pages of backing storage actually released. 72 */ 73 unsigned long 74 i915_gem_shrink(struct drm_i915_private *dev_priv, 75 long target, unsigned flags) 76 { 77 const struct { 78 struct list_head *list; 79 unsigned int bit; 80 } phases[] = { 81 { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND }, 82 { &dev_priv->mm.bound_list, I915_SHRINK_BOUND }, 83 { NULL, 0 }, 84 }, *phase; 85 unsigned long count = 0; 86 87 /* 88 * As we may completely rewrite the (un)bound list whilst unbinding 89 * (due to retiring requests) we have to strictly process only 90 * one element of the list at the time, and recheck the list 91 * on every iteration. 92 * 93 * In particular, we must hold a reference whilst removing the 94 * object as we may end up waiting for and/or retiring the objects. 95 * This might release the final reference (held by the active list) 96 * and result in the object being freed from under us. This is 97 * similar to the precautions the eviction code must take whilst 98 * removing objects. 99 * 100 * Also note that although these lists do not hold a reference to 101 * the object we can safely grab one here: The final object 102 * unreferencing and the bound_list are both protected by the 103 * dev->struct_mutex and so we won't ever be able to observe an 104 * object on the bound_list with a reference count equals 0. 105 */ 106 for (phase = phases; phase->list; phase++) { 107 struct list_head still_in_list; 108 109 if ((flags & phase->bit) == 0) 110 continue; 111 112 INIT_LIST_HEAD(&still_in_list); 113 while (count < target && !list_empty(phase->list)) { 114 struct drm_i915_gem_object *obj; 115 struct i915_vma *vma, *v; 116 117 obj = list_first_entry(phase->list, 118 typeof(*obj), global_list); 119 list_move_tail(&obj->global_list, &still_in_list); 120 121 if (flags & I915_SHRINK_PURGEABLE && 122 obj->madv != I915_MADV_DONTNEED) 123 continue; 124 125 drm_gem_object_reference(&obj->base); 126 127 /* For the unbound phase, this should be a no-op! */ 128 list_for_each_entry_safe(vma, v, 129 &obj->vma_list, vma_link) 130 if (i915_vma_unbind(vma)) 131 break; 132 133 if (i915_gem_object_put_pages(obj) == 0) 134 count += obj->base.size >> PAGE_SHIFT; 135 136 drm_gem_object_unreference(&obj->base); 137 } 138 list_splice(&still_in_list, phase->list); 139 } 140 141 return count; 142 } 143 144 /** 145 * i915_gem_shrink - Shrink buffer object caches completely 146 * @dev_priv: i915 device 147 * 148 * This is a simple wraper around i915_gem_shrink() to aggressively shrink all 149 * caches completely. It also first waits for and retires all outstanding 150 * requests to also be able to release backing storage for active objects. 151 * 152 * This should only be used in code to intentionally quiescent the gpu or as a 153 * last-ditch effort when memory seems to have run out. 154 * 155 * Returns: 156 * The number of pages of backing storage actually released. 157 */ 158 unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv) 159 { 160 i915_gem_evict_everything(dev_priv->dev); 161 return i915_gem_shrink(dev_priv, LONG_MAX, 162 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND); 163 } 164 165 #if 0 166 static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock) 167 { 168 if (!mutex_trylock(&dev->struct_mutex)) { 169 if (!mutex_is_locked_by(&dev->struct_mutex, current)) 170 return false; 171 172 if (to_i915(dev)->mm.shrinker_no_lock_stealing) 173 return false; 174 175 *unlock = false; 176 } else 177 *unlock = true; 178 179 return true; 180 } 181 182 static int num_vma_bound(struct drm_i915_gem_object *obj) 183 { 184 struct i915_vma *vma; 185 int count = 0; 186 187 list_for_each_entry(vma, &obj->vma_list, vma_link) { 188 if (drm_mm_node_allocated(&vma->node)) 189 count++; 190 if (vma->pin_count) 191 count++; 192 } 193 194 return count; 195 } 196 197 static unsigned long 198 i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) 199 { 200 struct drm_i915_private *dev_priv = 201 container_of(shrinker, struct drm_i915_private, mm.shrinker); 202 struct drm_device *dev = dev_priv->dev; 203 struct drm_i915_gem_object *obj; 204 unsigned long count; 205 bool unlock; 206 207 if (!i915_gem_shrinker_lock(dev, &unlock)) 208 return 0; 209 210 count = 0; 211 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) 212 if (obj->pages_pin_count == 0) 213 count += obj->base.size >> PAGE_SHIFT; 214 215 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 216 if (obj->pages_pin_count == num_vma_bound(obj)) 217 count += obj->base.size >> PAGE_SHIFT; 218 } 219 220 if (unlock) 221 mutex_unlock(&dev->struct_mutex); 222 223 return count; 224 } 225 226 static unsigned long 227 i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) 228 { 229 struct drm_i915_private *dev_priv = 230 container_of(shrinker, struct drm_i915_private, mm.shrinker); 231 struct drm_device *dev = dev_priv->dev; 232 unsigned long freed; 233 bool unlock; 234 235 if (!i915_gem_shrinker_lock(dev, &unlock)) 236 return SHRINK_STOP; 237 238 freed = i915_gem_shrink(dev_priv, 239 sc->nr_to_scan, 240 I915_SHRINK_BOUND | 241 I915_SHRINK_UNBOUND | 242 I915_SHRINK_PURGEABLE); 243 if (freed < sc->nr_to_scan) 244 freed += i915_gem_shrink(dev_priv, 245 sc->nr_to_scan - freed, 246 I915_SHRINK_BOUND | 247 I915_SHRINK_UNBOUND); 248 if (unlock) 249 mutex_unlock(&dev->struct_mutex); 250 251 return freed; 252 } 253 254 static int 255 i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) 256 { 257 struct drm_i915_private *dev_priv = 258 container_of(nb, struct drm_i915_private, mm.oom_notifier); 259 struct drm_device *dev = dev_priv->dev; 260 struct drm_i915_gem_object *obj; 261 unsigned long timeout = msecs_to_jiffies(5000) + 1; 262 unsigned long pinned, bound, unbound, freed_pages; 263 bool was_interruptible; 264 bool unlock; 265 266 while (!i915_gem_shrinker_lock(dev, &unlock) && --timeout) { 267 schedule_timeout_killable(1); 268 if (fatal_signal_pending(current)) 269 return NOTIFY_DONE; 270 } 271 if (timeout == 0) { 272 pr_err("Unable to purge GPU memory due lock contention.\n"); 273 return NOTIFY_DONE; 274 } 275 276 was_interruptible = dev_priv->mm.interruptible; 277 dev_priv->mm.interruptible = false; 278 279 freed_pages = i915_gem_shrink_all(dev_priv); 280 281 dev_priv->mm.interruptible = was_interruptible; 282 283 /* Because we may be allocating inside our own driver, we cannot 284 * assert that there are no objects with pinned pages that are not 285 * being pointed to by hardware. 286 */ 287 unbound = bound = pinned = 0; 288 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { 289 if (!obj->base.filp) /* not backed by a freeable object */ 290 continue; 291 292 if (obj->pages_pin_count) 293 pinned += obj->base.size; 294 else 295 unbound += obj->base.size; 296 } 297 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 298 if (!obj->base.filp) 299 continue; 300 301 if (obj->pages_pin_count) 302 pinned += obj->base.size; 303 else 304 bound += obj->base.size; 305 } 306 307 if (unlock) 308 mutex_unlock(&dev->struct_mutex); 309 310 if (freed_pages || unbound || bound) 311 pr_info("Purging GPU memory, %lu bytes freed, %lu bytes still pinned.\n", 312 freed_pages << PAGE_SHIFT, pinned); 313 if (unbound || bound) 314 pr_err("%lu and %lu bytes still available in the " 315 "bound and unbound GPU page lists.\n", 316 bound, unbound); 317 318 *(unsigned long *)ptr += freed_pages; 319 return NOTIFY_DONE; 320 } 321 #endif 322 323 /** 324 * i915_gem_shrinker_init - Initialize i915 shrinker 325 * @dev_priv: i915 device 326 * 327 * This function registers and sets up the i915 shrinker and OOM handler. 328 */ 329 void i915_gem_shrinker_init(struct drm_i915_private *dev_priv) 330 { 331 #if 0 332 dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan; 333 dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count; 334 dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS; 335 register_shrinker(&dev_priv->mm.shrinker); 336 337 dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom; 338 register_oom_notifier(&dev_priv->mm.oom_notifier); 339 #endif 340 } 341