1 // SPDX-License-Identifier: GPL-2.0
2 
3 /*
4  * Copyright 2016-2021 HabanaLabs, Ltd.
5  * All Rights Reserved.
6  */
7 
8 #include "habanalabs.h"
9 
10 #include <linux/slab.h>
11 
12 static void encaps_handle_do_release(struct hl_cs_encaps_sig_handle *handle, bool put_hw_sob,
13 					bool put_ctx)
14 {
15 	struct hl_encaps_signals_mgr *mgr = &handle->ctx->sig_mgr;
16 
17 	if (put_hw_sob)
18 		hw_sob_put(handle->hw_sob);
19 
20 	spin_lock(&mgr->lock);
21 	idr_remove(&mgr->handles, handle->id);
22 	spin_unlock(&mgr->lock);
23 
24 	if (put_ctx)
25 		hl_ctx_put(handle->ctx);
26 
27 	kfree(handle);
28 }
29 
30 void hl_encaps_release_handle_and_put_ctx(struct kref *ref)
31 {
32 	struct hl_cs_encaps_sig_handle *handle =
33 			container_of(ref, struct hl_cs_encaps_sig_handle, refcount);
34 
35 	encaps_handle_do_release(handle, false, true);
36 }
37 
38 static void hl_encaps_release_handle_and_put_sob(struct kref *ref)
39 {
40 	struct hl_cs_encaps_sig_handle *handle =
41 			container_of(ref, struct hl_cs_encaps_sig_handle, refcount);
42 
43 	encaps_handle_do_release(handle, true, false);
44 }
45 
46 void hl_encaps_release_handle_and_put_sob_ctx(struct kref *ref)
47 {
48 	struct hl_cs_encaps_sig_handle *handle =
49 			container_of(ref, struct hl_cs_encaps_sig_handle, refcount);
50 
51 	encaps_handle_do_release(handle, true, true);
52 }
53 
54 static void hl_encaps_sig_mgr_init(struct hl_encaps_signals_mgr *mgr)
55 {
56 	spin_lock_init(&mgr->lock);
57 	idr_init(&mgr->handles);
58 }
59 
60 static void hl_encaps_sig_mgr_fini(struct hl_device *hdev, struct hl_encaps_signals_mgr *mgr)
61 {
62 	struct hl_cs_encaps_sig_handle *handle;
63 	struct idr *idp;
64 	u32 id;
65 
66 	idp = &mgr->handles;
67 
68 	/* The IDR is expected to be empty at this stage, because any left signal should have been
69 	 * released as part of CS roll-back.
70 	 */
71 	if (!idr_is_empty(idp)) {
72 		dev_warn(hdev->dev,
73 			"device released while some encaps signals handles are still allocated\n");
74 		idr_for_each_entry(idp, handle, id)
75 			kref_put(&handle->refcount, hl_encaps_release_handle_and_put_sob);
76 	}
77 
78 	idr_destroy(&mgr->handles);
79 }
80 
81 static void hl_ctx_fini(struct hl_ctx *ctx)
82 {
83 	struct hl_device *hdev = ctx->hdev;
84 	int i;
85 
86 	/* Release all allocated HW block mapped list entries and destroy
87 	 * the mutex.
88 	 */
89 	hl_hw_block_mem_fini(ctx);
90 
91 	/*
92 	 * If we arrived here, there are no jobs waiting for this context
93 	 * on its queues so we can safely remove it.
94 	 * This is because for each CS, we increment the ref count and for
95 	 * every CS that was finished we decrement it and we won't arrive
96 	 * to this function unless the ref count is 0
97 	 */
98 
99 	for (i = 0 ; i < hdev->asic_prop.max_pending_cs ; i++)
100 		hl_fence_put(ctx->cs_pending[i]);
101 
102 	kfree(ctx->cs_pending);
103 
104 	if (ctx->asid != HL_KERNEL_ASID_ID) {
105 		dev_dbg(hdev->dev, "closing user context, asid=%u\n", ctx->asid);
106 
107 		/* The engines are stopped as there is no executing CS, but the
108 		 * Coresight might be still working by accessing addresses
109 		 * related to the stopped engines. Hence stop it explicitly.
110 		 */
111 		if (hdev->in_debug)
112 			hl_device_set_debug_mode(hdev, ctx, false);
113 
114 		hdev->asic_funcs->ctx_fini(ctx);
115 
116 		hl_dec_ctx_fini(ctx);
117 
118 		hl_cb_va_pool_fini(ctx);
119 		hl_vm_ctx_fini(ctx);
120 		hl_asid_free(hdev, ctx->asid);
121 		hl_encaps_sig_mgr_fini(hdev, &ctx->sig_mgr);
122 		mutex_destroy(&ctx->ts_reg_lock);
123 	} else {
124 		dev_dbg(hdev->dev, "closing kernel context\n");
125 		hdev->asic_funcs->ctx_fini(ctx);
126 		hl_vm_ctx_fini(ctx);
127 		hl_mmu_ctx_fini(ctx);
128 	}
129 }
130 
131 void hl_ctx_do_release(struct kref *ref)
132 {
133 	struct hl_ctx *ctx;
134 
135 	ctx = container_of(ref, struct hl_ctx, refcount);
136 
137 	hl_ctx_fini(ctx);
138 
139 	if (ctx->hpriv) {
140 		struct hl_fpriv *hpriv = ctx->hpriv;
141 
142 		mutex_lock(&hpriv->ctx_lock);
143 		hpriv->ctx = NULL;
144 		mutex_unlock(&hpriv->ctx_lock);
145 
146 		hl_hpriv_put(hpriv);
147 	}
148 
149 	kfree(ctx);
150 }
151 
152 int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv)
153 {
154 	struct hl_ctx_mgr *ctx_mgr = &hpriv->ctx_mgr;
155 	struct hl_ctx *ctx;
156 	int rc;
157 
158 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
159 	if (!ctx) {
160 		rc = -ENOMEM;
161 		goto out_err;
162 	}
163 
164 	mutex_lock(&ctx_mgr->lock);
165 	rc = idr_alloc(&ctx_mgr->handles, ctx, 1, 0, GFP_KERNEL);
166 	mutex_unlock(&ctx_mgr->lock);
167 
168 	if (rc < 0) {
169 		dev_err(hdev->dev, "Failed to allocate IDR for a new CTX\n");
170 		goto free_ctx;
171 	}
172 
173 	ctx->handle = rc;
174 
175 	rc = hl_ctx_init(hdev, ctx, false);
176 	if (rc)
177 		goto remove_from_idr;
178 
179 	hl_hpriv_get(hpriv);
180 	ctx->hpriv = hpriv;
181 
182 	/* TODO: remove for multiple contexts per process */
183 	hpriv->ctx = ctx;
184 
185 	/* TODO: remove the following line for multiple process support */
186 	hdev->is_compute_ctx_active = true;
187 
188 	return 0;
189 
190 remove_from_idr:
191 	mutex_lock(&ctx_mgr->lock);
192 	idr_remove(&ctx_mgr->handles, ctx->handle);
193 	mutex_unlock(&ctx_mgr->lock);
194 free_ctx:
195 	kfree(ctx);
196 out_err:
197 	return rc;
198 }
199 
200 int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)
201 {
202 	char task_comm[TASK_COMM_LEN];
203 	int rc = 0, i;
204 
205 	ctx->hdev = hdev;
206 
207 	kref_init(&ctx->refcount);
208 
209 	ctx->cs_sequence = 1;
210 	spin_lock_init(&ctx->cs_lock);
211 	atomic_set(&ctx->thread_ctx_switch_token, 1);
212 	ctx->thread_ctx_switch_wait_token = 0;
213 	ctx->cs_pending = kcalloc(hdev->asic_prop.max_pending_cs,
214 				sizeof(struct hl_fence *),
215 				GFP_KERNEL);
216 	if (!ctx->cs_pending)
217 		return -ENOMEM;
218 
219 	INIT_LIST_HEAD(&ctx->outcome_store.used_list);
220 	INIT_LIST_HEAD(&ctx->outcome_store.free_list);
221 	hash_init(ctx->outcome_store.outcome_map);
222 	for (i = 0; i < ARRAY_SIZE(ctx->outcome_store.nodes_pool); ++i)
223 		list_add(&ctx->outcome_store.nodes_pool[i].list_link,
224 			 &ctx->outcome_store.free_list);
225 
226 	hl_hw_block_mem_init(ctx);
227 
228 	if (is_kernel_ctx) {
229 		ctx->asid = HL_KERNEL_ASID_ID; /* Kernel driver gets ASID 0 */
230 		rc = hl_vm_ctx_init(ctx);
231 		if (rc) {
232 			dev_err(hdev->dev, "Failed to init mem ctx module\n");
233 			rc = -ENOMEM;
234 			goto err_hw_block_mem_fini;
235 		}
236 
237 		rc = hdev->asic_funcs->ctx_init(ctx);
238 		if (rc) {
239 			dev_err(hdev->dev, "ctx_init failed\n");
240 			goto err_vm_ctx_fini;
241 		}
242 	} else {
243 		ctx->asid = hl_asid_alloc(hdev);
244 		if (!ctx->asid) {
245 			dev_err(hdev->dev, "No free ASID, failed to create context\n");
246 			rc = -ENOMEM;
247 			goto err_hw_block_mem_fini;
248 		}
249 
250 		rc = hl_vm_ctx_init(ctx);
251 		if (rc) {
252 			dev_err(hdev->dev, "Failed to init mem ctx module\n");
253 			rc = -ENOMEM;
254 			goto err_asid_free;
255 		}
256 
257 		rc = hl_cb_va_pool_init(ctx);
258 		if (rc) {
259 			dev_err(hdev->dev,
260 				"Failed to init VA pool for mapped CB\n");
261 			goto err_vm_ctx_fini;
262 		}
263 
264 		rc = hdev->asic_funcs->ctx_init(ctx);
265 		if (rc) {
266 			dev_err(hdev->dev, "ctx_init failed\n");
267 			goto err_cb_va_pool_fini;
268 		}
269 
270 		hl_encaps_sig_mgr_init(&ctx->sig_mgr);
271 
272 		mutex_init(&ctx->ts_reg_lock);
273 
274 		dev_dbg(hdev->dev, "create user context, comm=\"%s\", asid=%u\n",
275 			get_task_comm(task_comm, current), ctx->asid);
276 	}
277 
278 	return 0;
279 
280 err_cb_va_pool_fini:
281 	hl_cb_va_pool_fini(ctx);
282 err_vm_ctx_fini:
283 	hl_vm_ctx_fini(ctx);
284 err_asid_free:
285 	if (ctx->asid != HL_KERNEL_ASID_ID)
286 		hl_asid_free(hdev, ctx->asid);
287 err_hw_block_mem_fini:
288 	hl_hw_block_mem_fini(ctx);
289 	kfree(ctx->cs_pending);
290 
291 	return rc;
292 }
293 
294 static int hl_ctx_get_unless_zero(struct hl_ctx *ctx)
295 {
296 	return kref_get_unless_zero(&ctx->refcount);
297 }
298 
299 void hl_ctx_get(struct hl_ctx *ctx)
300 {
301 	kref_get(&ctx->refcount);
302 }
303 
304 int hl_ctx_put(struct hl_ctx *ctx)
305 {
306 	return kref_put(&ctx->refcount, hl_ctx_do_release);
307 }
308 
309 struct hl_ctx *hl_get_compute_ctx(struct hl_device *hdev)
310 {
311 	struct hl_ctx *ctx = NULL;
312 	struct hl_fpriv *hpriv;
313 
314 	mutex_lock(&hdev->fpriv_list_lock);
315 
316 	list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) {
317 		mutex_lock(&hpriv->ctx_lock);
318 		ctx = hpriv->ctx;
319 		if (ctx && !hl_ctx_get_unless_zero(ctx))
320 			ctx = NULL;
321 		mutex_unlock(&hpriv->ctx_lock);
322 
323 		/* There can only be a single user which has opened the compute device, so exit
324 		 * immediately once we find its context or if we see that it has been released
325 		 */
326 		break;
327 	}
328 
329 	mutex_unlock(&hdev->fpriv_list_lock);
330 
331 	return ctx;
332 }
333 
334 /*
335  * hl_ctx_get_fence_locked - get CS fence under CS lock
336  *
337  * @ctx: pointer to the context structure.
338  * @seq: CS sequences number
339  *
340  * @return valid fence pointer on success, NULL if fence is gone, otherwise
341  *         error pointer.
342  *
343  * NOTE: this function shall be called with cs_lock locked
344  */
345 static struct hl_fence *hl_ctx_get_fence_locked(struct hl_ctx *ctx, u64 seq)
346 {
347 	struct asic_fixed_properties *asic_prop = &ctx->hdev->asic_prop;
348 	struct hl_fence *fence;
349 
350 	if (seq >= ctx->cs_sequence)
351 		return ERR_PTR(-EINVAL);
352 
353 	if (seq + asic_prop->max_pending_cs < ctx->cs_sequence)
354 		return NULL;
355 
356 	fence = ctx->cs_pending[seq & (asic_prop->max_pending_cs - 1)];
357 	hl_fence_get(fence);
358 	return fence;
359 }
360 
361 struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
362 {
363 	struct hl_fence *fence;
364 
365 	spin_lock(&ctx->cs_lock);
366 
367 	fence = hl_ctx_get_fence_locked(ctx, seq);
368 
369 	spin_unlock(&ctx->cs_lock);
370 
371 	return fence;
372 }
373 
374 /*
375  * hl_ctx_get_fences - get multiple CS fences under the same CS lock
376  *
377  * @ctx: pointer to the context structure.
378  * @seq_arr: array of CS sequences to wait for
379  * @fence: fence array to store the CS fences
380  * @arr_len: length of seq_arr and fence_arr
381  *
382  * @return 0 on success, otherwise non 0 error code
383  */
384 int hl_ctx_get_fences(struct hl_ctx *ctx, u64 *seq_arr,
385 				struct hl_fence **fence, u32 arr_len)
386 {
387 	struct hl_fence **fence_arr_base = fence;
388 	int i, rc = 0;
389 
390 	spin_lock(&ctx->cs_lock);
391 
392 	for (i = 0; i < arr_len; i++, fence++) {
393 		u64 seq = seq_arr[i];
394 
395 		*fence = hl_ctx_get_fence_locked(ctx, seq);
396 
397 		if (IS_ERR(*fence)) {
398 			dev_err(ctx->hdev->dev,
399 				"Failed to get fence for CS with seq 0x%llx\n",
400 					seq);
401 			rc = PTR_ERR(*fence);
402 			break;
403 		}
404 	}
405 
406 	spin_unlock(&ctx->cs_lock);
407 
408 	if (rc)
409 		hl_fences_put(fence_arr_base, i);
410 
411 	return rc;
412 }
413 
414 /*
415  * hl_ctx_mgr_init - initialize the context manager
416  *
417  * @ctx_mgr: pointer to context manager structure
418  *
419  * This manager is an object inside the hpriv object of the user process.
420  * The function is called when a user process opens the FD.
421  */
422 void hl_ctx_mgr_init(struct hl_ctx_mgr *ctx_mgr)
423 {
424 	mutex_init(&ctx_mgr->lock);
425 	idr_init(&ctx_mgr->handles);
426 }
427 
428 /*
429  * hl_ctx_mgr_fini - finalize the context manager
430  *
431  * @hdev: pointer to device structure
432  * @ctx_mgr: pointer to context manager structure
433  *
434  * This function goes over all the contexts in the manager and frees them.
435  * It is called when a process closes the FD.
436  */
437 void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *ctx_mgr)
438 {
439 	struct hl_ctx *ctx;
440 	struct idr *idp;
441 	u32 id;
442 
443 	idp = &ctx_mgr->handles;
444 
445 	idr_for_each_entry(idp, ctx, id)
446 		kref_put(&ctx->refcount, hl_ctx_do_release);
447 
448 	idr_destroy(&ctx_mgr->handles);
449 	mutex_destroy(&ctx_mgr->lock);
450 }
451