1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #include "habanalabs.h"
10 #include <linux/slab.h>
12 void hl_encaps_handle_do_release(struct kref *ref)
14 struct hl_cs_encaps_sig_handle *handle =
15 container_of(ref, struct hl_cs_encaps_sig_handle, refcount);
16 struct hl_ctx *ctx = handle->hdev->compute_ctx;
17 struct hl_encaps_signals_mgr *mgr = &ctx->sig_mgr;
19 spin_lock(&mgr->lock);
20 idr_remove(&mgr->handles, handle->id);
21 spin_unlock(&mgr->lock);
26 static void hl_encaps_handle_do_release_sob(struct kref *ref)
28 struct hl_cs_encaps_sig_handle *handle =
29 container_of(ref, struct hl_cs_encaps_sig_handle, refcount);
30 struct hl_ctx *ctx = handle->hdev->compute_ctx;
31 struct hl_encaps_signals_mgr *mgr = &ctx->sig_mgr;
33 /* if we're here, then there was a signals reservation but cs with
34 * encaps signals wasn't submitted, so need to put refcount
35 * to hw_sob taken at the reservation.
37 hw_sob_put(handle->hw_sob);
39 spin_lock(&mgr->lock);
40 idr_remove(&mgr->handles, handle->id);
41 spin_unlock(&mgr->lock);
46 static void hl_encaps_sig_mgr_init(struct hl_encaps_signals_mgr *mgr)
48 spin_lock_init(&mgr->lock);
49 idr_init(&mgr->handles);
52 static void hl_encaps_sig_mgr_fini(struct hl_device *hdev,
53 struct hl_encaps_signals_mgr *mgr)
55 struct hl_cs_encaps_sig_handle *handle;
61 if (!idr_is_empty(idp)) {
62 dev_warn(hdev->dev, "device released while some encaps signals handles are still allocated\n");
63 idr_for_each_entry(idp, handle, id)
64 kref_put(&handle->refcount,
65 hl_encaps_handle_do_release_sob);
68 idr_destroy(&mgr->handles);
71 static void hl_ctx_fini(struct hl_ctx *ctx)
73 struct hl_device *hdev = ctx->hdev;
76 /* Release all allocated HW block mapped list entries and destroy
79 hl_hw_block_mem_fini(ctx);
82 * If we arrived here, there are no jobs waiting for this context
83 * on its queues so we can safely remove it.
84 * This is because for each CS, we increment the ref count and for
85 * every CS that was finished we decrement it and we won't arrive
86 * to this function unless the ref count is 0
89 for (i = 0 ; i < hdev->asic_prop.max_pending_cs ; i++)
90 hl_fence_put(ctx->cs_pending[i]);
92 kfree(ctx->cs_pending);
94 if (ctx->asid != HL_KERNEL_ASID_ID) {
95 dev_dbg(hdev->dev, "closing user context %d\n", ctx->asid);
97 /* The engines are stopped as there is no executing CS, but the
98 * Coresight might be still working by accessing addresses
99 * related to the stopped engines. Hence stop it explicitly.
100 * Stop only if this is the compute context, as there can be
101 * only one compute context
103 if ((hdev->in_debug) && (hdev->compute_ctx == ctx))
104 hl_device_set_debug_mode(hdev, false);
106 hdev->asic_funcs->ctx_fini(ctx);
107 hl_cb_va_pool_fini(ctx);
109 hl_asid_free(hdev, ctx->asid);
110 hl_encaps_sig_mgr_fini(hdev, &ctx->sig_mgr);
112 /* Scrub both SRAM and DRAM */
113 hdev->asic_funcs->scrub_device_mem(hdev, 0, 0);
115 dev_dbg(hdev->dev, "closing kernel context\n");
116 hdev->asic_funcs->ctx_fini(ctx);
118 hl_mmu_ctx_fini(ctx);
122 void hl_ctx_do_release(struct kref *ref)
126 ctx = container_of(ref, struct hl_ctx, refcount);
131 hl_hpriv_put(ctx->hpriv);
136 int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv)
138 struct hl_ctx_mgr *mgr = &hpriv->ctx_mgr;
142 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
148 mutex_lock(&mgr->ctx_lock);
149 rc = idr_alloc(&mgr->ctx_handles, ctx, 1, 0, GFP_KERNEL);
150 mutex_unlock(&mgr->ctx_lock);
153 dev_err(hdev->dev, "Failed to allocate IDR for a new CTX\n");
159 rc = hl_ctx_init(hdev, ctx, false);
161 goto remove_from_idr;
166 /* TODO: remove for multiple contexts per process */
169 /* TODO: remove the following line for multiple process support */
170 hdev->compute_ctx = ctx;
175 mutex_lock(&mgr->ctx_lock);
176 idr_remove(&mgr->ctx_handles, ctx->handle);
177 mutex_unlock(&mgr->ctx_lock);
184 void hl_ctx_free(struct hl_device *hdev, struct hl_ctx *ctx)
186 if (kref_put(&ctx->refcount, hl_ctx_do_release) == 1)
190 int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)
196 kref_init(&ctx->refcount);
198 ctx->cs_sequence = 1;
199 spin_lock_init(&ctx->cs_lock);
200 atomic_set(&ctx->thread_ctx_switch_token, 1);
201 ctx->thread_ctx_switch_wait_token = 0;
202 ctx->cs_pending = kcalloc(hdev->asic_prop.max_pending_cs,
203 sizeof(struct hl_fence *),
205 if (!ctx->cs_pending)
208 hl_hw_block_mem_init(ctx);
211 ctx->asid = HL_KERNEL_ASID_ID; /* Kernel driver gets ASID 0 */
212 rc = hl_vm_ctx_init(ctx);
214 dev_err(hdev->dev, "Failed to init mem ctx module\n");
216 goto err_hw_block_mem_fini;
219 rc = hdev->asic_funcs->ctx_init(ctx);
221 dev_err(hdev->dev, "ctx_init failed\n");
222 goto err_vm_ctx_fini;
225 ctx->asid = hl_asid_alloc(hdev);
227 dev_err(hdev->dev, "No free ASID, failed to create context\n");
229 goto err_hw_block_mem_fini;
232 rc = hl_vm_ctx_init(ctx);
234 dev_err(hdev->dev, "Failed to init mem ctx module\n");
239 rc = hl_cb_va_pool_init(ctx);
242 "Failed to init VA pool for mapped CB\n");
243 goto err_vm_ctx_fini;
246 rc = hdev->asic_funcs->ctx_init(ctx);
248 dev_err(hdev->dev, "ctx_init failed\n");
249 goto err_cb_va_pool_fini;
252 hl_encaps_sig_mgr_init(&ctx->sig_mgr);
254 dev_dbg(hdev->dev, "create user context %d\n", ctx->asid);
260 hl_cb_va_pool_fini(ctx);
264 if (ctx->asid != HL_KERNEL_ASID_ID)
265 hl_asid_free(hdev, ctx->asid);
266 err_hw_block_mem_fini:
267 hl_hw_block_mem_fini(ctx);
268 kfree(ctx->cs_pending);
273 void hl_ctx_get(struct hl_device *hdev, struct hl_ctx *ctx)
275 kref_get(&ctx->refcount);
278 int hl_ctx_put(struct hl_ctx *ctx)
280 return kref_put(&ctx->refcount, hl_ctx_do_release);
284 * hl_ctx_get_fence_locked - get CS fence under CS lock
286 * @ctx: pointer to the context structure.
287 * @seq: CS sequences number
289 * @return valid fence pointer on success, NULL if fence is gone, otherwise
292 * NOTE: this function shall be called with cs_lock locked
294 static struct hl_fence *hl_ctx_get_fence_locked(struct hl_ctx *ctx, u64 seq)
296 struct asic_fixed_properties *asic_prop = &ctx->hdev->asic_prop;
297 struct hl_fence *fence;
299 if (seq >= ctx->cs_sequence)
300 return ERR_PTR(-EINVAL);
302 if (seq + asic_prop->max_pending_cs < ctx->cs_sequence)
305 fence = ctx->cs_pending[seq & (asic_prop->max_pending_cs - 1)];
310 struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
312 struct hl_fence *fence;
314 spin_lock(&ctx->cs_lock);
316 fence = hl_ctx_get_fence_locked(ctx, seq);
318 spin_unlock(&ctx->cs_lock);
324 * hl_ctx_get_fences - get multiple CS fences under the same CS lock
326 * @ctx: pointer to the context structure.
327 * @seq_arr: array of CS sequences to wait for
328 * @fence: fence array to store the CS fences
329 * @arr_len: length of seq_arr and fence_arr
331 * @return 0 on success, otherwise non 0 error code
333 int hl_ctx_get_fences(struct hl_ctx *ctx, u64 *seq_arr,
334 struct hl_fence **fence, u32 arr_len)
336 struct hl_fence **fence_arr_base = fence;
339 spin_lock(&ctx->cs_lock);
341 for (i = 0; i < arr_len; i++, fence++) {
342 u64 seq = seq_arr[i];
344 *fence = hl_ctx_get_fence_locked(ctx, seq);
346 if (IS_ERR(*fence)) {
347 dev_err(ctx->hdev->dev,
348 "Failed to get fence for CS with seq 0x%llx\n",
350 rc = PTR_ERR(*fence);
355 spin_unlock(&ctx->cs_lock);
358 hl_fences_put(fence_arr_base, i);
364 * hl_ctx_mgr_init - initialize the context manager
366 * @mgr: pointer to context manager structure
368 * This manager is an object inside the hpriv object of the user process.
369 * The function is called when a user process opens the FD.
371 void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr)
373 mutex_init(&mgr->ctx_lock);
374 idr_init(&mgr->ctx_handles);
378 * hl_ctx_mgr_fini - finalize the context manager
380 * @hdev: pointer to device structure
381 * @mgr: pointer to context manager structure
383 * This function goes over all the contexts in the manager and frees them.
384 * It is called when a process closes the FD.
386 void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr)
392 idp = &mgr->ctx_handles;
394 idr_for_each_entry(idp, ctx, id)
395 hl_ctx_free(hdev, ctx);
397 idr_destroy(&mgr->ctx_handles);
398 mutex_destroy(&mgr->ctx_lock);