1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_sync.h" 7 8 #include <linux/dma-fence-array.h> 9 #include <linux/kthread.h> 10 #include <linux/sched/mm.h> 11 #include <linux/uaccess.h> 12 13 #include <drm/drm_print.h> 14 #include <drm/drm_syncobj.h> 15 #include <drm/xe_drm.h> 16 17 #include "xe_device_types.h" 18 #include "xe_exec_queue.h" 19 #include "xe_macros.h" 20 #include "xe_sched_job_types.h" 21 22 struct user_fence { 23 struct xe_device *xe; 24 struct kref refcount; 25 struct dma_fence_cb cb; 26 struct work_struct worker; 27 struct mm_struct *mm; 28 u64 __user *addr; 29 u64 value; 30 }; 31 32 static void user_fence_destroy(struct kref *kref) 33 { 34 struct user_fence *ufence = container_of(kref, struct user_fence, 35 refcount); 36 37 mmdrop(ufence->mm); 38 kfree(ufence); 39 } 40 41 static void user_fence_get(struct user_fence *ufence) 42 { 43 kref_get(&ufence->refcount); 44 } 45 46 static void user_fence_put(struct user_fence *ufence) 47 { 48 kref_put(&ufence->refcount, user_fence_destroy); 49 } 50 51 static struct user_fence *user_fence_create(struct xe_device *xe, u64 addr, 52 u64 value) 53 { 54 struct user_fence *ufence; 55 56 ufence = kmalloc(sizeof(*ufence), GFP_KERNEL); 57 if (!ufence) 58 return NULL; 59 60 ufence->xe = xe; 61 kref_init(&ufence->refcount); 62 ufence->addr = u64_to_user_ptr(addr); 63 ufence->value = value; 64 ufence->mm = current->mm; 65 mmgrab(ufence->mm); 66 67 return ufence; 68 } 69 70 static void user_fence_worker(struct work_struct *w) 71 { 72 struct user_fence *ufence = container_of(w, struct user_fence, worker); 73 74 if (mmget_not_zero(ufence->mm)) { 75 kthread_use_mm(ufence->mm); 76 if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) 77 XE_WARN_ON("Copy to user failed"); 78 kthread_unuse_mm(ufence->mm); 79 mmput(ufence->mm); 80 } 81 82 wake_up_all(&ufence->xe->ufence_wq); 83 user_fence_put(ufence); 84 } 85 86 static void kick_ufence(struct user_fence *ufence, struct dma_fence *fence) 87 { 88 INIT_WORK(&ufence->worker, user_fence_worker); 89 queue_work(ufence->xe->ordered_wq, &ufence->worker); 90 dma_fence_put(fence); 91 } 92 93 static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 94 { 95 struct user_fence *ufence = container_of(cb, struct user_fence, cb); 96 97 kick_ufence(ufence, fence); 98 } 99 100 int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, 101 struct xe_sync_entry *sync, 102 struct drm_xe_sync __user *sync_user, 103 unsigned int flags) 104 { 105 struct drm_xe_sync sync_in; 106 int err; 107 bool exec = flags & SYNC_PARSE_FLAG_EXEC; 108 bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE; 109 bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE; 110 bool signal; 111 112 if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) 113 return -EFAULT; 114 115 if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) || 116 XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1])) 117 return -EINVAL; 118 119 signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL; 120 switch (sync_in.type) { 121 case DRM_XE_SYNC_TYPE_SYNCOBJ: 122 if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) 123 return -EOPNOTSUPP; 124 125 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) 126 return -EINVAL; 127 128 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 129 if (XE_IOCTL_DBG(xe, !sync->syncobj)) 130 return -ENOENT; 131 132 if (!signal) { 133 sync->fence = drm_syncobj_fence_get(sync->syncobj); 134 if (XE_IOCTL_DBG(xe, !sync->fence)) 135 return -EINVAL; 136 } 137 break; 138 139 case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ: 140 if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) 141 return -EOPNOTSUPP; 142 143 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) 144 return -EINVAL; 145 146 if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0)) 147 return -EINVAL; 148 149 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 150 if (XE_IOCTL_DBG(xe, !sync->syncobj)) 151 return -ENOENT; 152 153 if (signal) { 154 sync->chain_fence = dma_fence_chain_alloc(); 155 if (!sync->chain_fence) 156 return -ENOMEM; 157 } else { 158 sync->fence = drm_syncobj_fence_get(sync->syncobj); 159 if (XE_IOCTL_DBG(xe, !sync->fence)) 160 return -EINVAL; 161 162 err = dma_fence_chain_find_seqno(&sync->fence, 163 sync_in.timeline_value); 164 if (err) 165 return err; 166 } 167 break; 168 169 case DRM_XE_SYNC_TYPE_USER_FENCE: 170 if (XE_IOCTL_DBG(xe, disallow_user_fence)) 171 return -EOPNOTSUPP; 172 173 if (XE_IOCTL_DBG(xe, !signal)) 174 return -EOPNOTSUPP; 175 176 if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7)) 177 return -EINVAL; 178 179 if (exec) { 180 sync->addr = sync_in.addr; 181 } else { 182 sync->ufence = user_fence_create(xe, sync_in.addr, 183 sync_in.timeline_value); 184 if (XE_IOCTL_DBG(xe, !sync->ufence)) 185 return -ENOMEM; 186 } 187 188 break; 189 190 default: 191 return -EINVAL; 192 } 193 194 sync->type = sync_in.type; 195 sync->flags = sync_in.flags; 196 sync->timeline_value = sync_in.timeline_value; 197 198 return 0; 199 } 200 201 int xe_sync_entry_wait(struct xe_sync_entry *sync) 202 { 203 if (sync->fence) 204 dma_fence_wait(sync->fence, true); 205 206 return 0; 207 } 208 209 int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) 210 { 211 int err; 212 213 if (sync->fence) { 214 err = drm_sched_job_add_dependency(&job->drm, 215 dma_fence_get(sync->fence)); 216 if (err) { 217 dma_fence_put(sync->fence); 218 return err; 219 } 220 } 221 222 return 0; 223 } 224 225 void xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job, 226 struct dma_fence *fence) 227 { 228 if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) 229 return; 230 231 if (sync->chain_fence) { 232 drm_syncobj_add_point(sync->syncobj, sync->chain_fence, 233 fence, sync->timeline_value); 234 /* 235 * The chain's ownership is transferred to the 236 * timeline. 237 */ 238 sync->chain_fence = NULL; 239 } else if (sync->syncobj) { 240 drm_syncobj_replace_fence(sync->syncobj, fence); 241 } else if (sync->ufence) { 242 int err; 243 244 dma_fence_get(fence); 245 user_fence_get(sync->ufence); 246 err = dma_fence_add_callback(fence, &sync->ufence->cb, 247 user_fence_cb); 248 if (err == -ENOENT) { 249 kick_ufence(sync->ufence, fence); 250 } else if (err) { 251 XE_WARN_ON("failed to add user fence"); 252 user_fence_put(sync->ufence); 253 dma_fence_put(fence); 254 } 255 } else if (sync->type == DRM_XE_SYNC_TYPE_USER_FENCE) { 256 job->user_fence.used = true; 257 job->user_fence.addr = sync->addr; 258 job->user_fence.value = sync->timeline_value; 259 } 260 } 261 262 void xe_sync_entry_cleanup(struct xe_sync_entry *sync) 263 { 264 if (sync->syncobj) 265 drm_syncobj_put(sync->syncobj); 266 if (sync->fence) 267 dma_fence_put(sync->fence); 268 if (sync->chain_fence) 269 dma_fence_put(&sync->chain_fence->base); 270 if (sync->ufence) 271 user_fence_put(sync->ufence); 272 } 273 274 /** 275 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM 276 * @sync: input syncs 277 * @num_sync: number of syncs 278 * @q: exec queue 279 * @vm: VM 280 * 281 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create 282 * and return a composite fence of all in-fences + last fence. If no in-fences 283 * return last fence on input exec queue. Caller must drop reference to 284 * returned fence. 285 * 286 * Return: fence on success, ERR_PTR(-ENOMEM) on failure 287 */ 288 struct dma_fence * 289 xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, 290 struct xe_exec_queue *q, struct xe_vm *vm) 291 { 292 struct dma_fence **fences = NULL; 293 struct dma_fence_array *cf = NULL; 294 struct dma_fence *fence; 295 int i, num_in_fence = 0, current_fence = 0; 296 297 lockdep_assert_held(&vm->lock); 298 299 /* Count in-fences */ 300 for (i = 0; i < num_sync; ++i) { 301 if (sync[i].fence) { 302 ++num_in_fence; 303 fence = sync[i].fence; 304 } 305 } 306 307 /* Easy case... */ 308 if (!num_in_fence) { 309 fence = xe_exec_queue_last_fence_get(q, vm); 310 return fence; 311 } 312 313 /* Create composite fence */ 314 fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL); 315 if (!fences) 316 return ERR_PTR(-ENOMEM); 317 for (i = 0; i < num_sync; ++i) { 318 if (sync[i].fence) { 319 dma_fence_get(sync[i].fence); 320 fences[current_fence++] = sync[i].fence; 321 } 322 } 323 fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm); 324 cf = dma_fence_array_create(num_in_fence, fences, 325 vm->composite_fence_ctx, 326 vm->composite_fence_seqno++, 327 false); 328 if (!cf) { 329 --vm->composite_fence_seqno; 330 goto err_out; 331 } 332 333 return &cf->base; 334 335 err_out: 336 while (current_fence) 337 dma_fence_put(fences[--current_fence]); 338 kfree(fences); 339 kfree(cf); 340 341 return ERR_PTR(-ENOMEM); 342 } 343