1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include "gem/i915_gem_context.h" 8 #include "gem/i915_gem_pm.h" 9 10 #include "i915_drv.h" 11 #include "i915_globals.h" 12 13 #include "intel_context.h" 14 #include "intel_engine.h" 15 #include "intel_engine_pm.h" 16 #include "intel_ring.h" 17 18 static struct i915_global_context { 19 struct i915_global base; 20 #ifdef __linux__ 21 struct kmem_cache *slab_ce; 22 #else 23 struct pool slab_ce; 24 #endif 25 } global; 26 27 static struct intel_context *intel_context_alloc(void) 28 { 29 #ifdef __linux__ 30 return kmem_cache_zalloc(global.slab_ce, GFP_KERNEL); 31 #else 32 return pool_get(&global.slab_ce, PR_WAITOK | PR_ZERO); 33 #endif 34 } 35 36 void intel_context_free(struct intel_context *ce) 37 { 38 #ifdef __linux__ 39 kmem_cache_free(global.slab_ce, ce); 40 #else 41 pool_put(&global.slab_ce, ce); 42 #endif 43 } 44 45 struct intel_context * 46 intel_context_create(struct intel_engine_cs *engine) 47 { 48 struct intel_context *ce; 49 50 ce = intel_context_alloc(); 51 if (!ce) 52 return ERR_PTR(-ENOMEM); 53 54 intel_context_init(ce, engine); 55 return ce; 56 } 57 58 int intel_context_alloc_state(struct intel_context *ce) 59 { 60 int err = 0; 61 62 if (mutex_lock_interruptible(&ce->pin_mutex)) 63 return -EINTR; 64 65 if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) { 66 if (intel_context_is_banned(ce)) { 67 err = -EIO; 68 goto unlock; 69 } 70 71 err = ce->ops->alloc(ce); 72 if (unlikely(err)) 73 goto unlock; 74 75 set_bit(CONTEXT_ALLOC_BIT, &ce->flags); 76 } 77 78 unlock: 79 mutex_unlock(&ce->pin_mutex); 80 return err; 81 } 82 83 static int intel_context_active_acquire(struct intel_context *ce) 84 { 85 int err; 86 87 __i915_active_acquire(&ce->active); 88 89 if (intel_context_is_barrier(ce)) 90 return 0; 91 92 /* Preallocate tracking nodes */ 93 err = i915_active_acquire_preallocate_barrier(&ce->active, 94 ce->engine); 95 if (err) 96 i915_active_release(&ce->active); 97 98 return err; 99 } 100 101 static void intel_context_active_release(struct intel_context *ce) 102 { 103 /* Nodes preallocated in intel_context_active() */ 104 i915_active_acquire_barrier(&ce->active); 105 i915_active_release(&ce->active); 106 } 107 108 int __intel_context_do_pin(struct intel_context *ce) 109 { 110 int err; 111 112 GEM_BUG_ON(intel_context_is_closed(ce)); 113 114 if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) { 115 err = intel_context_alloc_state(ce); 116 if (err) 117 return err; 118 } 119 120 err = i915_active_acquire(&ce->active); 121 if (err) 122 return err; 123 124 if (mutex_lock_interruptible(&ce->pin_mutex)) { 125 err = -EINTR; 126 goto out_release; 127 } 128 129 if (likely(!atomic_add_unless(&ce->pin_count, 1, 0))) { 130 err = intel_context_active_acquire(ce); 131 if (unlikely(err)) 132 goto out_unlock; 133 134 err = ce->ops->pin(ce); 135 if (unlikely(err)) 136 goto err_active; 137 138 CE_TRACE(ce, "pin ring:{start:%08x, head:%04x, tail:%04x}\n", 139 i915_ggtt_offset(ce->ring->vma), 140 ce->ring->head, ce->ring->tail); 141 142 smp_mb__before_atomic(); /* flush pin before it is visible */ 143 atomic_inc(&ce->pin_count); 144 } 145 146 GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */ 147 GEM_BUG_ON(i915_active_is_idle(&ce->active)); 148 goto out_unlock; 149 150 err_active: 151 intel_context_active_release(ce); 152 out_unlock: 153 mutex_unlock(&ce->pin_mutex); 154 out_release: 155 i915_active_release(&ce->active); 156 return err; 157 } 158 159 void intel_context_unpin(struct intel_context *ce) 160 { 161 if (!atomic_dec_and_test(&ce->pin_count)) 162 return; 163 164 CE_TRACE(ce, "unpin\n"); 165 ce->ops->unpin(ce); 166 167 /* 168 * Once released, we may asynchronously drop the active reference. 169 * As that may be the only reference keeping the context alive, 170 * take an extra now so that it is not freed before we finish 171 * dereferencing it. 172 */ 173 intel_context_get(ce); 174 intel_context_active_release(ce); 175 intel_context_put(ce); 176 } 177 178 static int __context_pin_state(struct i915_vma *vma) 179 { 180 unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS; 181 int err; 182 183 err = i915_ggtt_pin(vma, 0, bias | PIN_HIGH); 184 if (err) 185 return err; 186 187 err = i915_active_acquire(&vma->active); 188 if (err) 189 goto err_unpin; 190 191 /* 192 * And mark it as a globally pinned object to let the shrinker know 193 * it cannot reclaim the object until we release it. 194 */ 195 i915_vma_make_unshrinkable(vma); 196 vma->obj->mm.dirty = true; 197 198 return 0; 199 200 err_unpin: 201 i915_vma_unpin(vma); 202 return err; 203 } 204 205 static void __context_unpin_state(struct i915_vma *vma) 206 { 207 i915_vma_make_shrinkable(vma); 208 i915_active_release(&vma->active); 209 __i915_vma_unpin(vma); 210 } 211 212 static int __ring_active(struct intel_ring *ring) 213 { 214 int err; 215 216 err = intel_ring_pin(ring); 217 if (err) 218 return err; 219 220 err = i915_active_acquire(&ring->vma->active); 221 if (err) 222 goto err_pin; 223 224 return 0; 225 226 err_pin: 227 intel_ring_unpin(ring); 228 return err; 229 } 230 231 static void __ring_retire(struct intel_ring *ring) 232 { 233 i915_active_release(&ring->vma->active); 234 intel_ring_unpin(ring); 235 } 236 237 __i915_active_call 238 static void __intel_context_retire(struct i915_active *active) 239 { 240 struct intel_context *ce = container_of(active, typeof(*ce), active); 241 242 CE_TRACE(ce, "retire runtime: { total:%lluns, avg:%lluns }\n", 243 intel_context_get_total_runtime_ns(ce), 244 intel_context_get_avg_runtime_ns(ce)); 245 246 set_bit(CONTEXT_VALID_BIT, &ce->flags); 247 if (ce->state) 248 __context_unpin_state(ce->state); 249 250 intel_timeline_unpin(ce->timeline); 251 __ring_retire(ce->ring); 252 253 intel_context_put(ce); 254 } 255 256 static int __intel_context_active(struct i915_active *active) 257 { 258 struct intel_context *ce = container_of(active, typeof(*ce), active); 259 int err; 260 261 CE_TRACE(ce, "active\n"); 262 263 intel_context_get(ce); 264 265 err = __ring_active(ce->ring); 266 if (err) 267 goto err_put; 268 269 err = intel_timeline_pin(ce->timeline); 270 if (err) 271 goto err_ring; 272 273 if (!ce->state) 274 return 0; 275 276 err = __context_pin_state(ce->state); 277 if (err) 278 goto err_timeline; 279 280 return 0; 281 282 err_timeline: 283 intel_timeline_unpin(ce->timeline); 284 err_ring: 285 __ring_retire(ce->ring); 286 err_put: 287 intel_context_put(ce); 288 return err; 289 } 290 291 void 292 intel_context_init(struct intel_context *ce, 293 struct intel_engine_cs *engine) 294 { 295 GEM_BUG_ON(!engine->cops); 296 GEM_BUG_ON(!engine->gt->vm); 297 298 kref_init(&ce->ref); 299 300 ce->engine = engine; 301 ce->ops = engine->cops; 302 ce->sseu = engine->sseu; 303 ce->ring = __intel_context_ring_size(SZ_4K); 304 305 ewma_runtime_init(&ce->runtime.avg); 306 307 ce->vm = i915_vm_get(engine->gt->vm); 308 309 INIT_LIST_HEAD(&ce->signal_link); 310 INIT_LIST_HEAD(&ce->signals); 311 312 rw_init(&ce->pin_mutex, "cepin"); 313 314 i915_active_init(&ce->active, 315 __intel_context_active, __intel_context_retire); 316 } 317 318 void intel_context_fini(struct intel_context *ce) 319 { 320 if (ce->timeline) 321 intel_timeline_put(ce->timeline); 322 i915_vm_put(ce->vm); 323 324 mutex_destroy(&ce->pin_mutex); 325 i915_active_fini(&ce->active); 326 } 327 328 static void i915_global_context_shrink(void) 329 { 330 #ifdef notyet 331 kmem_cache_shrink(global.slab_ce); 332 #endif 333 } 334 335 static void i915_global_context_exit(void) 336 { 337 #ifdef __linux__ 338 kmem_cache_destroy(global.slab_ce); 339 #else 340 pool_destroy(&global.slab_ce); 341 #endif 342 } 343 344 static struct i915_global_context global = { { 345 .shrink = i915_global_context_shrink, 346 .exit = i915_global_context_exit, 347 } }; 348 349 int __init i915_global_context_init(void) 350 { 351 #ifdef __linux__ 352 global.slab_ce = KMEM_CACHE(intel_context, SLAB_HWCACHE_ALIGN); 353 if (!global.slab_ce) 354 return -ENOMEM; 355 #else 356 pool_init(&global.slab_ce, sizeof(struct intel_context), 357 CACHELINESIZE, IPL_TTY, 0, "ictx", NULL); 358 #endif 359 360 i915_global_register(&global.base); 361 return 0; 362 } 363 364 void intel_context_enter_engine(struct intel_context *ce) 365 { 366 intel_engine_pm_get(ce->engine); 367 intel_timeline_enter(ce->timeline); 368 } 369 370 void intel_context_exit_engine(struct intel_context *ce) 371 { 372 intel_timeline_exit(ce->timeline); 373 intel_engine_pm_put(ce->engine); 374 } 375 376 int intel_context_prepare_remote_request(struct intel_context *ce, 377 struct i915_request *rq) 378 { 379 struct intel_timeline *tl = ce->timeline; 380 int err; 381 382 /* Only suitable for use in remotely modifying this context */ 383 GEM_BUG_ON(rq->context == ce); 384 385 if (rcu_access_pointer(rq->timeline) != tl) { /* timeline sharing! */ 386 /* Queue this switch after current activity by this context. */ 387 err = i915_active_fence_set(&tl->last_request, rq); 388 if (err) 389 return err; 390 } 391 392 /* 393 * Guarantee context image and the timeline remains pinned until the 394 * modifying request is retired by setting the ce activity tracker. 395 * 396 * But we only need to take one pin on the account of it. Or in other 397 * words transfer the pinned ce object to tracked active request. 398 */ 399 GEM_BUG_ON(i915_active_is_idle(&ce->active)); 400 return i915_active_add_request(&ce->active, rq); 401 } 402 403 struct i915_request *intel_context_create_request(struct intel_context *ce) 404 { 405 struct i915_request *rq; 406 int err; 407 408 err = intel_context_pin(ce); 409 if (unlikely(err)) 410 return ERR_PTR(err); 411 412 rq = i915_request_create(ce); 413 intel_context_unpin(ce); 414 415 return rq; 416 } 417 418 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 419 #include "selftest_context.c" 420 #endif 421