1 // SPDX-License-Identifier: MIT 2 3 #include <linux/slab.h> 4 #include <drm/gpu_scheduler.h> 5 #include <drm/drm_syncobj.h> 6 7 #include "nouveau_drv.h" 8 #include "nouveau_gem.h" 9 #include "nouveau_mem.h" 10 #include "nouveau_dma.h" 11 #include "nouveau_exec.h" 12 #include "nouveau_abi16.h" 13 #include "nouveau_sched.h" 14 15 #define NOUVEAU_SCHED_JOB_TIMEOUT_MS 10000 16 17 /* Starts at 0, since the DRM scheduler interprets those parameters as (initial) 18 * index to the run-queue array. 19 */ 20 enum nouveau_sched_priority { 21 NOUVEAU_SCHED_PRIORITY_SINGLE = DRM_SCHED_PRIORITY_KERNEL, 22 NOUVEAU_SCHED_PRIORITY_COUNT, 23 }; 24 25 int 26 nouveau_job_init(struct nouveau_job *job, 27 struct nouveau_job_args *args) 28 { 29 struct nouveau_sched *sched = args->sched; 30 int ret; 31 32 INIT_LIST_HEAD(&job->entry); 33 34 job->file_priv = args->file_priv; 35 job->cli = nouveau_cli(args->file_priv); 36 job->sched = sched; 37 38 job->sync = args->sync; 39 job->resv_usage = args->resv_usage; 40 41 job->ops = args->ops; 42 43 job->in_sync.count = args->in_sync.count; 44 if (job->in_sync.count) { 45 if (job->sync) 46 return -EINVAL; 47 48 job->in_sync.data = kmemdup(args->in_sync.s, 49 sizeof(*args->in_sync.s) * 50 args->in_sync.count, 51 GFP_KERNEL); 52 if (!job->in_sync.data) 53 return -ENOMEM; 54 } 55 56 job->out_sync.count = args->out_sync.count; 57 if (job->out_sync.count) { 58 if (job->sync) { 59 ret = -EINVAL; 60 goto err_free_in_sync; 61 } 62 63 job->out_sync.data = kmemdup(args->out_sync.s, 64 sizeof(*args->out_sync.s) * 65 args->out_sync.count, 66 GFP_KERNEL); 67 if (!job->out_sync.data) { 68 ret = -ENOMEM; 69 goto err_free_in_sync; 70 } 71 72 job->out_sync.objs = kcalloc(job->out_sync.count, 73 sizeof(*job->out_sync.objs), 74 GFP_KERNEL); 75 if (!job->out_sync.objs) { 76 ret = -ENOMEM; 77 goto err_free_out_sync; 78 } 79 80 job->out_sync.chains = kcalloc(job->out_sync.count, 81 sizeof(*job->out_sync.chains), 82 GFP_KERNEL); 83 if (!job->out_sync.chains) { 84 ret = -ENOMEM; 85 goto err_free_objs; 86 } 87 } 88 89 ret = drm_sched_job_init(&job->base, &sched->entity, 90 args->credits, NULL); 91 if (ret) 92 goto err_free_chains; 93 94 job->state = NOUVEAU_JOB_INITIALIZED; 95 96 return 0; 97 98 err_free_chains: 99 kfree(job->out_sync.chains); 100 err_free_objs: 101 kfree(job->out_sync.objs); 102 err_free_out_sync: 103 kfree(job->out_sync.data); 104 err_free_in_sync: 105 kfree(job->in_sync.data); 106 return ret; 107 } 108 109 void 110 nouveau_job_fini(struct nouveau_job *job) 111 { 112 dma_fence_put(job->done_fence); 113 drm_sched_job_cleanup(&job->base); 114 115 job->ops->free(job); 116 } 117 118 void 119 nouveau_job_done(struct nouveau_job *job) 120 { 121 struct nouveau_sched *sched = job->sched; 122 123 spin_lock(&sched->job.list.lock); 124 list_del(&job->entry); 125 spin_unlock(&sched->job.list.lock); 126 127 wake_up(&sched->job.wq); 128 } 129 130 void 131 nouveau_job_free(struct nouveau_job *job) 132 { 133 kfree(job->in_sync.data); 134 kfree(job->out_sync.data); 135 kfree(job->out_sync.objs); 136 kfree(job->out_sync.chains); 137 } 138 139 static int 140 sync_find_fence(struct nouveau_job *job, 141 struct drm_nouveau_sync *sync, 142 struct dma_fence **fence) 143 { 144 u32 stype = sync->flags & DRM_NOUVEAU_SYNC_TYPE_MASK; 145 u64 point = 0; 146 int ret; 147 148 if (stype != DRM_NOUVEAU_SYNC_SYNCOBJ && 149 stype != DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ) 150 return -EOPNOTSUPP; 151 152 if (stype == DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ) 153 point = sync->timeline_value; 154 155 ret = drm_syncobj_find_fence(job->file_priv, 156 sync->handle, point, 157 0 /* flags */, fence); 158 if (ret) 159 return ret; 160 161 return 0; 162 } 163 164 static int 165 nouveau_job_add_deps(struct nouveau_job *job) 166 { 167 struct dma_fence *in_fence = NULL; 168 int ret, i; 169 170 for (i = 0; i < job->in_sync.count; i++) { 171 struct drm_nouveau_sync *sync = &job->in_sync.data[i]; 172 173 ret = sync_find_fence(job, sync, &in_fence); 174 if (ret) { 175 NV_PRINTK(warn, job->cli, 176 "Failed to find syncobj (-> in): handle=%d\n", 177 sync->handle); 178 return ret; 179 } 180 181 ret = drm_sched_job_add_dependency(&job->base, in_fence); 182 if (ret) 183 return ret; 184 } 185 186 return 0; 187 } 188 189 static void 190 nouveau_job_fence_attach_cleanup(struct nouveau_job *job) 191 { 192 int i; 193 194 for (i = 0; i < job->out_sync.count; i++) { 195 struct drm_syncobj *obj = job->out_sync.objs[i]; 196 struct dma_fence_chain *chain = job->out_sync.chains[i]; 197 198 if (obj) 199 drm_syncobj_put(obj); 200 201 if (chain) 202 dma_fence_chain_free(chain); 203 } 204 } 205 206 static int 207 nouveau_job_fence_attach_prepare(struct nouveau_job *job) 208 { 209 int i, ret; 210 211 for (i = 0; i < job->out_sync.count; i++) { 212 struct drm_nouveau_sync *sync = &job->out_sync.data[i]; 213 struct drm_syncobj **pobj = &job->out_sync.objs[i]; 214 struct dma_fence_chain **pchain = &job->out_sync.chains[i]; 215 u32 stype = sync->flags & DRM_NOUVEAU_SYNC_TYPE_MASK; 216 217 if (stype != DRM_NOUVEAU_SYNC_SYNCOBJ && 218 stype != DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ) { 219 ret = -EINVAL; 220 goto err_sync_cleanup; 221 } 222 223 *pobj = drm_syncobj_find(job->file_priv, sync->handle); 224 if (!*pobj) { 225 NV_PRINTK(warn, job->cli, 226 "Failed to find syncobj (-> out): handle=%d\n", 227 sync->handle); 228 ret = -ENOENT; 229 goto err_sync_cleanup; 230 } 231 232 if (stype == DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ) { 233 *pchain = dma_fence_chain_alloc(); 234 if (!*pchain) { 235 ret = -ENOMEM; 236 goto err_sync_cleanup; 237 } 238 } 239 } 240 241 return 0; 242 243 err_sync_cleanup: 244 nouveau_job_fence_attach_cleanup(job); 245 return ret; 246 } 247 248 static void 249 nouveau_job_fence_attach(struct nouveau_job *job) 250 { 251 struct dma_fence *fence = job->done_fence; 252 int i; 253 254 for (i = 0; i < job->out_sync.count; i++) { 255 struct drm_nouveau_sync *sync = &job->out_sync.data[i]; 256 struct drm_syncobj **pobj = &job->out_sync.objs[i]; 257 struct dma_fence_chain **pchain = &job->out_sync.chains[i]; 258 u32 stype = sync->flags & DRM_NOUVEAU_SYNC_TYPE_MASK; 259 260 if (stype == DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ) { 261 drm_syncobj_add_point(*pobj, *pchain, fence, 262 sync->timeline_value); 263 } else { 264 drm_syncobj_replace_fence(*pobj, fence); 265 } 266 267 drm_syncobj_put(*pobj); 268 *pobj = NULL; 269 *pchain = NULL; 270 } 271 } 272 273 int 274 nouveau_job_submit(struct nouveau_job *job) 275 { 276 struct nouveau_sched *sched = job->sched; 277 struct dma_fence *done_fence = NULL; 278 struct drm_gpuvm_exec vm_exec = { 279 .vm = &nouveau_cli_uvmm(job->cli)->base, 280 .flags = DRM_EXEC_IGNORE_DUPLICATES, 281 .num_fences = 1, 282 }; 283 int ret; 284 285 ret = nouveau_job_add_deps(job); 286 if (ret) 287 goto err; 288 289 ret = nouveau_job_fence_attach_prepare(job); 290 if (ret) 291 goto err; 292 293 /* Make sure the job appears on the sched_entity's queue in the same 294 * order as it was submitted. 295 */ 296 mutex_lock(&sched->mutex); 297 298 /* Guarantee we won't fail after the submit() callback returned 299 * successfully. 300 */ 301 if (job->ops->submit) { 302 ret = job->ops->submit(job, &vm_exec); 303 if (ret) 304 goto err_cleanup; 305 } 306 307 /* Submit was successful; add the job to the schedulers job list. */ 308 spin_lock(&sched->job.list.lock); 309 list_add(&job->entry, &sched->job.list.head); 310 spin_unlock(&sched->job.list.lock); 311 312 drm_sched_job_arm(&job->base); 313 job->done_fence = dma_fence_get(&job->base.s_fence->finished); 314 if (job->sync) 315 done_fence = dma_fence_get(job->done_fence); 316 317 if (job->ops->armed_submit) 318 job->ops->armed_submit(job, &vm_exec); 319 320 nouveau_job_fence_attach(job); 321 322 /* Set job state before pushing the job to the scheduler, 323 * such that we do not overwrite the job state set in run(). 324 */ 325 job->state = NOUVEAU_JOB_SUBMIT_SUCCESS; 326 327 drm_sched_entity_push_job(&job->base); 328 329 mutex_unlock(&sched->mutex); 330 331 if (done_fence) { 332 dma_fence_wait(done_fence, true); 333 dma_fence_put(done_fence); 334 } 335 336 return 0; 337 338 err_cleanup: 339 mutex_unlock(&sched->mutex); 340 nouveau_job_fence_attach_cleanup(job); 341 err: 342 job->state = NOUVEAU_JOB_SUBMIT_FAILED; 343 return ret; 344 } 345 346 static struct dma_fence * 347 nouveau_job_run(struct nouveau_job *job) 348 { 349 struct dma_fence *fence; 350 351 fence = job->ops->run(job); 352 if (IS_ERR(fence)) 353 job->state = NOUVEAU_JOB_RUN_FAILED; 354 else 355 job->state = NOUVEAU_JOB_RUN_SUCCESS; 356 357 return fence; 358 } 359 360 static struct dma_fence * 361 nouveau_sched_run_job(struct drm_sched_job *sched_job) 362 { 363 struct nouveau_job *job = to_nouveau_job(sched_job); 364 365 return nouveau_job_run(job); 366 } 367 368 static enum drm_gpu_sched_stat 369 nouveau_sched_timedout_job(struct drm_sched_job *sched_job) 370 { 371 struct drm_gpu_scheduler *sched = sched_job->sched; 372 struct nouveau_job *job = to_nouveau_job(sched_job); 373 enum drm_gpu_sched_stat stat = DRM_GPU_SCHED_STAT_NOMINAL; 374 375 drm_sched_stop(sched, sched_job); 376 377 if (job->ops->timeout) 378 stat = job->ops->timeout(job); 379 else 380 NV_PRINTK(warn, job->cli, "Generic job timeout.\n"); 381 382 drm_sched_start(sched, true); 383 384 return stat; 385 } 386 387 static void 388 nouveau_sched_free_job(struct drm_sched_job *sched_job) 389 { 390 struct nouveau_job *job = to_nouveau_job(sched_job); 391 392 nouveau_job_fini(job); 393 } 394 395 static const struct drm_sched_backend_ops nouveau_sched_ops = { 396 .run_job = nouveau_sched_run_job, 397 .timedout_job = nouveau_sched_timedout_job, 398 .free_job = nouveau_sched_free_job, 399 }; 400 401 int 402 nouveau_sched_init(struct nouveau_sched *sched, struct nouveau_drm *drm, 403 struct workqueue_struct *wq, u32 credit_limit) 404 { 405 struct drm_gpu_scheduler *drm_sched = &sched->base; 406 struct drm_sched_entity *entity = &sched->entity; 407 long job_hang_limit = msecs_to_jiffies(NOUVEAU_SCHED_JOB_TIMEOUT_MS); 408 int ret; 409 410 if (!wq) { 411 wq = alloc_workqueue("nouveau_sched_wq_%d", 0, WQ_MAX_ACTIVE, 412 current->pid); 413 if (!wq) 414 return -ENOMEM; 415 416 sched->wq = wq; 417 } 418 419 ret = drm_sched_init(drm_sched, &nouveau_sched_ops, wq, 420 NOUVEAU_SCHED_PRIORITY_COUNT, 421 credit_limit, 0, job_hang_limit, 422 NULL, NULL, "nouveau_sched", drm->dev->dev); 423 if (ret) 424 goto fail_wq; 425 426 /* Using DRM_SCHED_PRIORITY_KERNEL, since that's what we're required to use 427 * when we want to have a single run-queue only. 428 * 429 * It's not documented, but one will find out when trying to use any 430 * other priority running into faults, because the scheduler uses the 431 * priority as array index. 432 * 433 * Can't use NOUVEAU_SCHED_PRIORITY_SINGLE either, because it's not 434 * matching the enum type used in drm_sched_entity_init(). 435 */ 436 ret = drm_sched_entity_init(entity, DRM_SCHED_PRIORITY_KERNEL, 437 &drm_sched, 1, NULL); 438 if (ret) 439 goto fail_sched; 440 441 mutex_init(&sched->mutex); 442 spin_lock_init(&sched->job.list.lock); 443 INIT_LIST_HEAD(&sched->job.list.head); 444 init_waitqueue_head(&sched->job.wq); 445 446 return 0; 447 448 fail_sched: 449 drm_sched_fini(drm_sched); 450 fail_wq: 451 if (sched->wq) 452 destroy_workqueue(sched->wq); 453 return ret; 454 } 455 456 void 457 nouveau_sched_fini(struct nouveau_sched *sched) 458 { 459 struct drm_gpu_scheduler *drm_sched = &sched->base; 460 struct drm_sched_entity *entity = &sched->entity; 461 462 rmb(); /* for list_empty to work without lock */ 463 wait_event(sched->job.wq, list_empty(&sched->job.list.head)); 464 465 drm_sched_entity_fini(entity); 466 drm_sched_fini(drm_sched); 467 468 /* Destroy workqueue after scheduler tear down, otherwise it might still 469 * be in use. 470 */ 471 if (sched->wq) 472 destroy_workqueue(sched->wq); 473 } 474