1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #include <uapi/misc/habanalabs.h>
9 #include "habanalabs.h"
11 #include <linux/uaccess.h>
12 #include <linux/slab.h>
14 #define HL_CS_FLAGS_TYPE_MASK (HL_CS_FLAGS_SIGNAL | HL_CS_FLAGS_WAIT | \
15 HL_CS_FLAGS_COLLECTIVE_WAIT)
18 * enum hl_cs_wait_status - cs wait status
19 * @CS_WAIT_STATUS_BUSY: cs was not completed yet
20 * @CS_WAIT_STATUS_COMPLETED: cs completed
21 * @CS_WAIT_STATUS_GONE: cs completed but fence is already gone
23 enum hl_cs_wait_status {
25 CS_WAIT_STATUS_COMPLETED,
29 static void job_wq_completion(struct work_struct *work);
30 static int _hl_cs_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
31 u64 timeout_us, u64 seq,
32 enum hl_cs_wait_status *status, s64 *timestamp);
33 static void cs_do_release(struct kref *ref);
35 static void hl_sob_reset(struct kref *ref)
37 struct hl_hw_sob *hw_sob = container_of(ref, struct hl_hw_sob,
39 struct hl_device *hdev = hw_sob->hdev;
41 hdev->asic_funcs->reset_sob(hdev, hw_sob);
44 void hl_sob_reset_error(struct kref *ref)
46 struct hl_hw_sob *hw_sob = container_of(ref, struct hl_hw_sob,
48 struct hl_device *hdev = hw_sob->hdev;
51 "SOB release shouldn't be called here, q_idx: %d, sob_id: %d\n",
52 hw_sob->q_idx, hw_sob->sob_id);
56 * hl_gen_sob_mask() - Generates a sob mask to be used in a monitor arm packet
57 * @sob_base: sob base id
58 * @sob_mask: sob user mask, each bit represents a sob offset from sob base
59 * @mask: generated mask
61 * Return: 0 if given parameters are valid
63 int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask)
70 if (sob_mask == 0x1) {
71 *mask = ~(1 << (sob_base & 0x7));
73 /* find msb in order to verify sob range is valid */
74 for (i = BITS_PER_BYTE - 1 ; i >= 0 ; i--)
75 if (BIT(i) & sob_mask)
78 if (i > (HL_MAX_SOBS_PER_MONITOR - (sob_base & 0x7) - 1))
87 static void sob_reset_work(struct work_struct *work)
89 struct hl_cs_compl *hl_cs_cmpl =
90 container_of(work, struct hl_cs_compl, sob_reset_work);
91 struct hl_device *hdev = hl_cs_cmpl->hdev;
94 * A signal CS can get completion while the corresponding wait
95 * for signal CS is on its way to the PQ. The wait for signal CS
96 * will get stuck if the signal CS incremented the SOB to its
97 * max value and there are no pending (submitted) waits on this
99 * We do the following to void this situation:
100 * 1. The wait for signal CS must get a ref for the signal CS as
101 * soon as possible in cs_ioctl_signal_wait() and put it
102 * before being submitted to the PQ but after it incremented
103 * the SOB refcnt in init_signal_wait_cs().
104 * 2. Signal/Wait for signal CS will decrement the SOB refcnt
106 * These two measures guarantee that the wait for signal CS will
107 * reset the SOB upon completion rather than the signal CS and
108 * hence the above scenario is avoided.
110 kref_put(&hl_cs_cmpl->hw_sob->kref, hl_sob_reset);
112 if (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT)
113 hdev->asic_funcs->reset_sob_group(hdev,
114 hl_cs_cmpl->sob_group);
119 static void hl_fence_release(struct kref *kref)
121 struct hl_fence *fence =
122 container_of(kref, struct hl_fence, refcount);
123 struct hl_cs_compl *hl_cs_cmpl =
124 container_of(fence, struct hl_cs_compl, base_fence);
125 struct hl_device *hdev = hl_cs_cmpl->hdev;
127 /* EBUSY means the CS was never submitted and hence we don't have
128 * an attached hw_sob object that we should handle here
130 if (fence->error == -EBUSY)
133 if ((hl_cs_cmpl->type == CS_TYPE_SIGNAL) ||
134 (hl_cs_cmpl->type == CS_TYPE_WAIT) ||
135 (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT)) {
138 "CS 0x%llx type %d finished, sob_id: %d, sob_val: 0x%x\n",
141 hl_cs_cmpl->hw_sob->sob_id,
142 hl_cs_cmpl->sob_val);
144 queue_work(hdev->sob_reset_wq, &hl_cs_cmpl->sob_reset_work);
153 void hl_fence_put(struct hl_fence *fence)
156 kref_put(&fence->refcount, hl_fence_release);
159 void hl_fence_get(struct hl_fence *fence)
162 kref_get(&fence->refcount);
165 static void hl_fence_init(struct hl_fence *fence, u64 sequence)
167 kref_init(&fence->refcount);
168 fence->cs_sequence = sequence;
170 fence->timestamp = ktime_set(0, 0);
171 init_completion(&fence->completion);
174 void cs_get(struct hl_cs *cs)
176 kref_get(&cs->refcount);
179 static int cs_get_unless_zero(struct hl_cs *cs)
181 return kref_get_unless_zero(&cs->refcount);
184 static void cs_put(struct hl_cs *cs)
186 kref_put(&cs->refcount, cs_do_release);
189 static void cs_job_do_release(struct kref *ref)
191 struct hl_cs_job *job = container_of(ref, struct hl_cs_job, refcount);
196 static void cs_job_put(struct hl_cs_job *job)
198 kref_put(&job->refcount, cs_job_do_release);
201 bool cs_needs_completion(struct hl_cs *cs)
203 /* In case this is a staged CS, only the last CS in sequence should
204 * get a completion, any non staged CS will always get a completion
206 if (cs->staged_cs && !cs->staged_last)
212 bool cs_needs_timeout(struct hl_cs *cs)
214 /* In case this is a staged CS, only the first CS in sequence should
215 * get a timeout, any non staged CS will always get a timeout
217 if (cs->staged_cs && !cs->staged_first)
223 static bool is_cb_patched(struct hl_device *hdev, struct hl_cs_job *job)
226 * Patched CB is created for external queues jobs, and for H/W queues
227 * jobs if the user CB was allocated by driver and MMU is disabled.
229 return (job->queue_type == QUEUE_TYPE_EXT ||
230 (job->queue_type == QUEUE_TYPE_HW &&
231 job->is_kernel_allocated_cb &&
236 * cs_parser - parse the user command submission
238 * @hpriv : pointer to the private data of the fd
239 * @job : pointer to the job that holds the command submission info
241 * The function parses the command submission of the user. It calls the
242 * ASIC specific parser, which returns a list of memory blocks to send
243 * to the device as different command buffers
246 static int cs_parser(struct hl_fpriv *hpriv, struct hl_cs_job *job)
248 struct hl_device *hdev = hpriv->hdev;
249 struct hl_cs_parser parser;
252 parser.ctx_id = job->cs->ctx->asid;
253 parser.cs_sequence = job->cs->sequence;
254 parser.job_id = job->id;
256 parser.hw_queue_id = job->hw_queue_id;
257 parser.job_userptr_list = &job->userptr_list;
258 parser.patched_cb = NULL;
259 parser.user_cb = job->user_cb;
260 parser.user_cb_size = job->user_cb_size;
261 parser.queue_type = job->queue_type;
262 parser.is_kernel_allocated_cb = job->is_kernel_allocated_cb;
263 job->patched_cb = NULL;
264 parser.completion = cs_needs_completion(job->cs);
266 rc = hdev->asic_funcs->cs_parser(hdev, &parser);
268 if (is_cb_patched(hdev, job)) {
270 job->patched_cb = parser.patched_cb;
271 job->job_cb_size = parser.patched_cb_size;
272 job->contains_dma_pkt = parser.contains_dma_pkt;
273 atomic_inc(&job->patched_cb->cs_cnt);
277 * Whether the parsing worked or not, we don't need the
278 * original CB anymore because it was already parsed and
279 * won't be accessed again for this CS
281 atomic_dec(&job->user_cb->cs_cnt);
282 hl_cb_put(job->user_cb);
285 job->job_cb_size = job->user_cb_size;
291 static void complete_job(struct hl_device *hdev, struct hl_cs_job *job)
293 struct hl_cs *cs = job->cs;
295 if (is_cb_patched(hdev, job)) {
296 hl_userptr_delete_list(hdev, &job->userptr_list);
299 * We might arrive here from rollback and patched CB wasn't
300 * created, so we need to check it's not NULL
302 if (job->patched_cb) {
303 atomic_dec(&job->patched_cb->cs_cnt);
304 hl_cb_put(job->patched_cb);
308 /* For H/W queue jobs, if a user CB was allocated by driver and MMU is
309 * enabled, the user CB isn't released in cs_parser() and thus should be
311 * This is also true for INT queues jobs which were allocated by driver
313 if (job->is_kernel_allocated_cb &&
314 ((job->queue_type == QUEUE_TYPE_HW && hdev->mmu_enable) ||
315 job->queue_type == QUEUE_TYPE_INT)) {
316 atomic_dec(&job->user_cb->cs_cnt);
317 hl_cb_put(job->user_cb);
321 * This is the only place where there can be multiple threads
322 * modifying the list at the same time
324 spin_lock(&cs->job_lock);
325 list_del(&job->cs_node);
326 spin_unlock(&cs->job_lock);
328 hl_debugfs_remove_job(hdev, job);
330 /* We decrement reference only for a CS that gets completion
331 * because the reference was incremented only for this kind of CS
332 * right before it was scheduled.
334 * In staged submission, only the last CS marked as 'staged_last'
335 * gets completion, hence its release function will be called from here.
336 * As for all the rest CS's in the staged submission which do not get
337 * completion, their CS reference will be decremented by the
338 * 'staged_last' CS during the CS release flow.
339 * All relevant PQ CI counters will be incremented during the CS release
340 * flow by calling 'hl_hw_queue_update_ci'.
342 if (cs_needs_completion(cs) &&
343 (job->queue_type == QUEUE_TYPE_EXT ||
344 job->queue_type == QUEUE_TYPE_HW))
351 * hl_staged_cs_find_first - locate the first CS in this staged submission
353 * @hdev: pointer to device structure
354 * @cs_seq: staged submission sequence number
356 * @note: This function must be called under 'hdev->cs_mirror_lock'
358 * Find and return a CS pointer with the given sequence
360 struct hl_cs *hl_staged_cs_find_first(struct hl_device *hdev, u64 cs_seq)
364 list_for_each_entry_reverse(cs, &hdev->cs_mirror_list, mirror_node)
365 if (cs->staged_cs && cs->staged_first &&
366 cs->sequence == cs_seq)
373 * is_staged_cs_last_exists - returns true if the last CS in sequence exists
375 * @hdev: pointer to device structure
376 * @cs: staged submission member
379 bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs)
381 struct hl_cs *last_entry;
383 last_entry = list_last_entry(&cs->staged_cs_node, struct hl_cs,
386 if (last_entry->staged_last)
393 * staged_cs_get - get CS reference if this CS is a part of a staged CS
395 * @hdev: pointer to device structure
397 * @cs_seq: staged submission sequence number
399 * Increment CS reference for every CS in this staged submission except for
400 * the CS which get completion.
402 static void staged_cs_get(struct hl_device *hdev, struct hl_cs *cs)
404 /* Only the last CS in this staged submission will get a completion.
405 * We must increment the reference for all other CS's in this
407 * Once we get a completion we will release the whole staged submission.
409 if (!cs->staged_last)
414 * staged_cs_put - put a CS in case it is part of staged submission
416 * @hdev: pointer to device structure
419 * This function decrements a CS reference (for a non completion CS)
421 static void staged_cs_put(struct hl_device *hdev, struct hl_cs *cs)
423 /* We release all CS's in a staged submission except the last
424 * CS which we have never incremented its reference.
426 if (!cs_needs_completion(cs))
430 static void cs_handle_tdr(struct hl_device *hdev, struct hl_cs *cs)
432 bool next_entry_found = false;
435 if (!cs_needs_timeout(cs))
438 spin_lock(&hdev->cs_mirror_lock);
440 /* We need to handle tdr only once for the complete staged submission.
441 * Hence, we choose the CS that reaches this function first which is
442 * the CS marked as 'staged_last'.
444 if (cs->staged_cs && cs->staged_last)
445 cs = hl_staged_cs_find_first(hdev, cs->staged_sequence);
447 spin_unlock(&hdev->cs_mirror_lock);
449 /* Don't cancel TDR in case this CS was timedout because we might be
450 * running from the TDR context
452 if (cs && (cs->timedout ||
453 hdev->timeout_jiffies == MAX_SCHEDULE_TIMEOUT))
456 if (cs && cs->tdr_active)
457 cancel_delayed_work_sync(&cs->work_tdr);
459 spin_lock(&hdev->cs_mirror_lock);
461 /* queue TDR for next CS */
462 list_for_each_entry(next, &hdev->cs_mirror_list, mirror_node)
463 if (cs_needs_timeout(next)) {
464 next_entry_found = true;
468 if (next_entry_found && !next->tdr_active) {
469 next->tdr_active = true;
470 schedule_delayed_work(&next->work_tdr, next->timeout_jiffies);
473 spin_unlock(&hdev->cs_mirror_lock);
476 static void cs_do_release(struct kref *ref)
478 struct hl_cs *cs = container_of(ref, struct hl_cs, refcount);
479 struct hl_device *hdev = cs->ctx->hdev;
480 struct hl_cs_job *job, *tmp;
482 cs->completed = true;
485 * Although if we reached here it means that all external jobs have
486 * finished, because each one of them took refcnt to CS, we still
487 * need to go over the internal jobs and complete them. Otherwise, we
488 * will have leaked memory and what's worse, the CS object (and
489 * potentially the CTX object) could be released, while the JOB
490 * still holds a pointer to them (but no reference).
492 list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)
493 complete_job(hdev, job);
495 if (!cs->submitted) {
496 /* In case the wait for signal CS was submitted, the put occurs
497 * in init_signal_wait_cs() or collective_wait_init_cs()
498 * right before hanging on the PQ.
500 if (cs->type == CS_TYPE_WAIT ||
501 cs->type == CS_TYPE_COLLECTIVE_WAIT)
502 hl_fence_put(cs->signal_fence);
507 /* Need to update CI for all queue jobs that does not get completion */
508 hl_hw_queue_update_ci(cs);
510 /* remove CS from CS mirror list */
511 spin_lock(&hdev->cs_mirror_lock);
512 list_del_init(&cs->mirror_node);
513 spin_unlock(&hdev->cs_mirror_lock);
515 cs_handle_tdr(hdev, cs);
518 /* the completion CS decrements reference for the entire
521 if (cs->staged_last) {
522 struct hl_cs *staged_cs, *tmp;
524 list_for_each_entry_safe(staged_cs, tmp,
525 &cs->staged_cs_node, staged_cs_node)
526 staged_cs_put(hdev, staged_cs);
529 /* A staged CS will be a member in the list only after it
530 * was submitted. We used 'cs_mirror_lock' when inserting
531 * it to list so we will use it again when removing it
534 spin_lock(&hdev->cs_mirror_lock);
535 list_del(&cs->staged_cs_node);
536 spin_unlock(&hdev->cs_mirror_lock);
541 /* Must be called before hl_ctx_put because inside we use ctx to get
544 hl_debugfs_remove_cs(cs);
548 /* We need to mark an error for not submitted because in that case
549 * the hl fence release flow is different. Mainly, we don't need
550 * to handle hw_sob for signal/wait
553 cs->fence->error = -ETIMEDOUT;
554 else if (cs->aborted)
555 cs->fence->error = -EIO;
556 else if (!cs->submitted)
557 cs->fence->error = -EBUSY;
559 if (unlikely(cs->skip_reset_on_timeout)) {
561 "Command submission %llu completed after %llu (s)\n",
563 div_u64(jiffies - cs->submission_time_jiffies, HZ));
567 cs->fence->timestamp = ktime_get();
568 complete_all(&cs->fence->completion);
569 hl_fence_put(cs->fence);
571 kfree(cs->jobs_in_queue_cnt);
575 static void cs_timedout(struct work_struct *work)
577 struct hl_device *hdev;
579 struct hl_cs *cs = container_of(work, struct hl_cs,
581 bool skip_reset_on_timeout = cs->skip_reset_on_timeout;
583 rc = cs_get_unless_zero(cs);
587 if ((!cs->submitted) || (cs->completed)) {
592 /* Mark the CS is timed out so we won't try to cancel its TDR */
593 if (likely(!skip_reset_on_timeout))
596 hdev = cs->ctx->hdev;
601 "Signal command submission %llu has not finished in time!\n",
607 "Wait command submission %llu has not finished in time!\n",
611 case CS_TYPE_COLLECTIVE_WAIT:
613 "Collective Wait command submission %llu has not finished in time!\n",
619 "Command submission %llu has not finished in time!\n",
626 if (likely(!skip_reset_on_timeout)) {
627 if (hdev->reset_on_lockup)
628 hl_device_reset(hdev, HL_RESET_TDR);
630 hdev->needs_reset = true;
634 static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
635 enum hl_cs_type cs_type, u64 user_sequence,
636 struct hl_cs **cs_new, u32 flags, u32 timeout)
638 struct hl_cs_counters_atomic *cntr;
639 struct hl_fence *other = NULL;
640 struct hl_cs_compl *cs_cmpl;
644 cntr = &hdev->aggregated_cs_counters;
646 cs = kzalloc(sizeof(*cs), GFP_ATOMIC);
648 cs = kzalloc(sizeof(*cs), GFP_KERNEL);
651 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
652 atomic64_inc(&cntr->out_of_mem_drop_cnt);
656 /* increment refcnt for context */
657 hl_ctx_get(hdev, ctx);
660 cs->submitted = false;
661 cs->completed = false;
663 cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP);
664 cs->timeout_jiffies = timeout;
665 cs->skip_reset_on_timeout =
666 hdev->skip_reset_on_timeout ||
667 !!(flags & HL_CS_FLAGS_SKIP_RESET_ON_TIMEOUT);
668 cs->submission_time_jiffies = jiffies;
669 INIT_LIST_HEAD(&cs->job_list);
670 INIT_DELAYED_WORK(&cs->work_tdr, cs_timedout);
671 kref_init(&cs->refcount);
672 spin_lock_init(&cs->job_lock);
674 cs_cmpl = kmalloc(sizeof(*cs_cmpl), GFP_ATOMIC);
676 cs_cmpl = kmalloc(sizeof(*cs_cmpl), GFP_KERNEL);
679 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
680 atomic64_inc(&cntr->out_of_mem_drop_cnt);
685 cs->jobs_in_queue_cnt = kcalloc(hdev->asic_prop.max_queues,
686 sizeof(*cs->jobs_in_queue_cnt), GFP_ATOMIC);
687 if (!cs->jobs_in_queue_cnt)
688 cs->jobs_in_queue_cnt = kcalloc(hdev->asic_prop.max_queues,
689 sizeof(*cs->jobs_in_queue_cnt), GFP_KERNEL);
691 if (!cs->jobs_in_queue_cnt) {
692 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
693 atomic64_inc(&cntr->out_of_mem_drop_cnt);
698 cs_cmpl->hdev = hdev;
699 cs_cmpl->type = cs->type;
700 spin_lock_init(&cs_cmpl->lock);
701 INIT_WORK(&cs_cmpl->sob_reset_work, sob_reset_work);
702 cs->fence = &cs_cmpl->base_fence;
704 spin_lock(&ctx->cs_lock);
706 cs_cmpl->cs_seq = ctx->cs_sequence;
707 other = ctx->cs_pending[cs_cmpl->cs_seq &
708 (hdev->asic_prop.max_pending_cs - 1)];
710 if (other && !completion_done(&other->completion)) {
711 /* If the following statement is true, it means we have reached
712 * a point in which only part of the staged submission was
713 * submitted and we don't have enough room in the 'cs_pending'
714 * array for the rest of the submission.
715 * This causes a deadlock because this CS will never be
716 * completed as it depends on future CS's for completion.
718 if (other->cs_sequence == user_sequence)
719 dev_crit_ratelimited(hdev->dev,
720 "Staged CS %llu deadlock due to lack of resources",
723 dev_dbg_ratelimited(hdev->dev,
724 "Rejecting CS because of too many in-flights CS\n");
725 atomic64_inc(&ctx->cs_counters.max_cs_in_flight_drop_cnt);
726 atomic64_inc(&cntr->max_cs_in_flight_drop_cnt);
732 hl_fence_init(&cs_cmpl->base_fence, cs_cmpl->cs_seq);
734 cs->sequence = cs_cmpl->cs_seq;
736 ctx->cs_pending[cs_cmpl->cs_seq &
737 (hdev->asic_prop.max_pending_cs - 1)] =
738 &cs_cmpl->base_fence;
741 hl_fence_get(&cs_cmpl->base_fence);
745 spin_unlock(&ctx->cs_lock);
752 spin_unlock(&ctx->cs_lock);
753 kfree(cs->jobs_in_queue_cnt);
762 static void cs_rollback(struct hl_device *hdev, struct hl_cs *cs)
764 struct hl_cs_job *job, *tmp;
766 staged_cs_put(hdev, cs);
768 list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)
769 complete_job(hdev, job);
772 void hl_cs_rollback_all(struct hl_device *hdev)
775 struct hl_cs *cs, *tmp;
777 flush_workqueue(hdev->sob_reset_wq);
779 /* flush all completions before iterating over the CS mirror list in
780 * order to avoid a race with the release functions
782 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
783 flush_workqueue(hdev->cq_wq[i]);
785 /* Make sure we don't have leftovers in the CS mirror list */
786 list_for_each_entry_safe(cs, tmp, &hdev->cs_mirror_list, mirror_node) {
789 dev_warn_ratelimited(hdev->dev, "Killing CS %d.%llu\n",
790 cs->ctx->asid, cs->sequence);
791 cs_rollback(hdev, cs);
796 void hl_pending_cb_list_flush(struct hl_ctx *ctx)
798 struct hl_pending_cb *pending_cb, *tmp;
800 list_for_each_entry_safe(pending_cb, tmp,
801 &ctx->pending_cb_list, cb_node) {
802 list_del(&pending_cb->cb_node);
803 hl_cb_put(pending_cb->cb);
809 wake_pending_user_interrupt_threads(struct hl_user_interrupt *interrupt)
811 struct hl_user_pending_interrupt *pend;
813 spin_lock(&interrupt->wait_list_lock);
814 list_for_each_entry(pend, &interrupt->wait_list_head, wait_list_node) {
815 pend->fence.error = -EIO;
816 complete_all(&pend->fence.completion);
818 spin_unlock(&interrupt->wait_list_lock);
821 void hl_release_pending_user_interrupts(struct hl_device *hdev)
823 struct asic_fixed_properties *prop = &hdev->asic_prop;
824 struct hl_user_interrupt *interrupt;
827 if (!prop->user_interrupt_count)
830 /* We iterate through the user interrupt requests and waking up all
831 * user threads waiting for interrupt completion. We iterate the
832 * list under a lock, this is why all user threads, once awake,
833 * will wait on the same lock and will release the waiting object upon
837 for (i = 0 ; i < prop->user_interrupt_count ; i++) {
838 interrupt = &hdev->user_interrupt[i];
839 wake_pending_user_interrupt_threads(interrupt);
842 interrupt = &hdev->common_user_interrupt;
843 wake_pending_user_interrupt_threads(interrupt);
846 static void job_wq_completion(struct work_struct *work)
848 struct hl_cs_job *job = container_of(work, struct hl_cs_job,
850 struct hl_cs *cs = job->cs;
851 struct hl_device *hdev = cs->ctx->hdev;
853 /* job is no longer needed */
854 complete_job(hdev, job);
857 static int validate_queue_index(struct hl_device *hdev,
858 struct hl_cs_chunk *chunk,
859 enum hl_queue_type *queue_type,
860 bool *is_kernel_allocated_cb)
862 struct asic_fixed_properties *asic = &hdev->asic_prop;
863 struct hw_queue_properties *hw_queue_prop;
865 /* This must be checked here to prevent out-of-bounds access to
866 * hw_queues_props array
868 if (chunk->queue_index >= asic->max_queues) {
869 dev_err(hdev->dev, "Queue index %d is invalid\n",
874 hw_queue_prop = &asic->hw_queues_props[chunk->queue_index];
876 if (hw_queue_prop->type == QUEUE_TYPE_NA) {
877 dev_err(hdev->dev, "Queue index %d is invalid\n",
882 if (hw_queue_prop->driver_only) {
884 "Queue index %d is restricted for the kernel driver\n",
889 /* When hw queue type isn't QUEUE_TYPE_HW,
890 * USER_ALLOC_CB flag shall be referred as "don't care".
892 if (hw_queue_prop->type == QUEUE_TYPE_HW) {
893 if (chunk->cs_chunk_flags & HL_CS_CHUNK_FLAGS_USER_ALLOC_CB) {
894 if (!(hw_queue_prop->cb_alloc_flags & CB_ALLOC_USER)) {
896 "Queue index %d doesn't support user CB\n",
901 *is_kernel_allocated_cb = false;
903 if (!(hw_queue_prop->cb_alloc_flags &
906 "Queue index %d doesn't support kernel CB\n",
911 *is_kernel_allocated_cb = true;
914 *is_kernel_allocated_cb = !!(hw_queue_prop->cb_alloc_flags
918 *queue_type = hw_queue_prop->type;
922 static struct hl_cb *get_cb_from_cs_chunk(struct hl_device *hdev,
923 struct hl_cb_mgr *cb_mgr,
924 struct hl_cs_chunk *chunk)
929 cb_handle = (u32) (chunk->cb_handle >> PAGE_SHIFT);
931 cb = hl_cb_get(hdev, cb_mgr, cb_handle);
933 dev_err(hdev->dev, "CB handle 0x%x invalid\n", cb_handle);
937 if ((chunk->cb_size < 8) || (chunk->cb_size > cb->size)) {
938 dev_err(hdev->dev, "CB size %u invalid\n", chunk->cb_size);
942 atomic_inc(&cb->cs_cnt);
951 struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev,
952 enum hl_queue_type queue_type, bool is_kernel_allocated_cb)
954 struct hl_cs_job *job;
956 job = kzalloc(sizeof(*job), GFP_ATOMIC);
958 job = kzalloc(sizeof(*job), GFP_KERNEL);
963 kref_init(&job->refcount);
964 job->queue_type = queue_type;
965 job->is_kernel_allocated_cb = is_kernel_allocated_cb;
967 if (is_cb_patched(hdev, job))
968 INIT_LIST_HEAD(&job->userptr_list);
970 if (job->queue_type == QUEUE_TYPE_EXT)
971 INIT_WORK(&job->finish_work, job_wq_completion);
976 static enum hl_cs_type hl_cs_get_cs_type(u32 cs_type_flags)
978 if (cs_type_flags & HL_CS_FLAGS_SIGNAL)
979 return CS_TYPE_SIGNAL;
980 else if (cs_type_flags & HL_CS_FLAGS_WAIT)
982 else if (cs_type_flags & HL_CS_FLAGS_COLLECTIVE_WAIT)
983 return CS_TYPE_COLLECTIVE_WAIT;
985 return CS_TYPE_DEFAULT;
988 static int hl_cs_sanity_checks(struct hl_fpriv *hpriv, union hl_cs_args *args)
990 struct hl_device *hdev = hpriv->hdev;
991 struct hl_ctx *ctx = hpriv->ctx;
992 u32 cs_type_flags, num_chunks;
993 enum hl_device_status status;
994 enum hl_cs_type cs_type;
996 if (!hl_device_operational(hdev, &status)) {
997 dev_warn_ratelimited(hdev->dev,
998 "Device is %s. Can't submit new CS\n",
999 hdev->status[status]);
1003 if ((args->in.cs_flags & HL_CS_FLAGS_STAGED_SUBMISSION) &&
1004 !hdev->supports_staged_submission) {
1005 dev_err(hdev->dev, "staged submission not supported");
1009 cs_type_flags = args->in.cs_flags & HL_CS_FLAGS_TYPE_MASK;
1011 if (unlikely(cs_type_flags && !is_power_of_2(cs_type_flags))) {
1013 "CS type flags are mutually exclusive, context %d\n",
1018 cs_type = hl_cs_get_cs_type(cs_type_flags);
1019 num_chunks = args->in.num_chunks_execute;
1021 if (unlikely((cs_type != CS_TYPE_DEFAULT) &&
1022 !hdev->supports_sync_stream)) {
1023 dev_err(hdev->dev, "Sync stream CS is not supported\n");
1027 if (cs_type == CS_TYPE_DEFAULT) {
1030 "Got execute CS with 0 chunks, context %d\n",
1034 } else if (num_chunks != 1) {
1036 "Sync stream CS mandates one chunk only, context %d\n",
1044 static int hl_cs_copy_chunk_array(struct hl_device *hdev,
1045 struct hl_cs_chunk **cs_chunk_array,
1046 void __user *chunks, u32 num_chunks,
1051 if (num_chunks > HL_MAX_JOBS_PER_CS) {
1052 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1053 atomic64_inc(&hdev->aggregated_cs_counters.validation_drop_cnt);
1055 "Number of chunks can NOT be larger than %d\n",
1056 HL_MAX_JOBS_PER_CS);
1060 *cs_chunk_array = kmalloc_array(num_chunks, sizeof(**cs_chunk_array),
1062 if (!*cs_chunk_array)
1063 *cs_chunk_array = kmalloc_array(num_chunks,
1064 sizeof(**cs_chunk_array), GFP_KERNEL);
1065 if (!*cs_chunk_array) {
1066 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
1067 atomic64_inc(&hdev->aggregated_cs_counters.out_of_mem_drop_cnt);
1071 size_to_copy = num_chunks * sizeof(struct hl_cs_chunk);
1072 if (copy_from_user(*cs_chunk_array, chunks, size_to_copy)) {
1073 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1074 atomic64_inc(&hdev->aggregated_cs_counters.validation_drop_cnt);
1075 dev_err(hdev->dev, "Failed to copy cs chunk array from user\n");
1076 kfree(*cs_chunk_array);
1083 static int cs_staged_submission(struct hl_device *hdev, struct hl_cs *cs,
1084 u64 sequence, u32 flags)
1086 if (!(flags & HL_CS_FLAGS_STAGED_SUBMISSION))
1089 cs->staged_last = !!(flags & HL_CS_FLAGS_STAGED_SUBMISSION_LAST);
1090 cs->staged_first = !!(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST);
1092 if (cs->staged_first) {
1093 /* Staged CS sequence is the first CS sequence */
1094 INIT_LIST_HEAD(&cs->staged_cs_node);
1095 cs->staged_sequence = cs->sequence;
1097 /* User sequence will be validated in 'hl_hw_queue_schedule_cs'
1098 * under the cs_mirror_lock
1100 cs->staged_sequence = sequence;
1103 /* Increment CS reference if needed */
1104 staged_cs_get(hdev, cs);
1106 cs->staged_cs = true;
1111 static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks,
1112 u32 num_chunks, u64 *cs_seq, u32 flags,
1115 bool staged_mid, int_queues_only = true;
1116 struct hl_device *hdev = hpriv->hdev;
1117 struct hl_cs_chunk *cs_chunk_array;
1118 struct hl_cs_counters_atomic *cntr;
1119 struct hl_ctx *ctx = hpriv->ctx;
1120 struct hl_cs_job *job;
1126 cntr = &hdev->aggregated_cs_counters;
1127 user_sequence = *cs_seq;
1128 *cs_seq = ULLONG_MAX;
1130 rc = hl_cs_copy_chunk_array(hdev, &cs_chunk_array, chunks, num_chunks,
1135 if ((flags & HL_CS_FLAGS_STAGED_SUBMISSION) &&
1136 !(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST))
1141 rc = allocate_cs(hdev, hpriv->ctx, CS_TYPE_DEFAULT,
1142 staged_mid ? user_sequence : ULLONG_MAX, &cs, flags,
1145 goto free_cs_chunk_array;
1147 *cs_seq = cs->sequence;
1149 hl_debugfs_add_cs(cs);
1151 rc = cs_staged_submission(hdev, cs, user_sequence, flags);
1153 goto free_cs_object;
1155 /* Validate ALL the CS chunks before submitting the CS */
1156 for (i = 0 ; i < num_chunks ; i++) {
1157 struct hl_cs_chunk *chunk = &cs_chunk_array[i];
1158 enum hl_queue_type queue_type;
1159 bool is_kernel_allocated_cb;
1161 rc = validate_queue_index(hdev, chunk, &queue_type,
1162 &is_kernel_allocated_cb);
1164 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1165 atomic64_inc(&cntr->validation_drop_cnt);
1166 goto free_cs_object;
1169 if (is_kernel_allocated_cb) {
1170 cb = get_cb_from_cs_chunk(hdev, &hpriv->cb_mgr, chunk);
1173 &ctx->cs_counters.validation_drop_cnt);
1174 atomic64_inc(&cntr->validation_drop_cnt);
1176 goto free_cs_object;
1179 cb = (struct hl_cb *) (uintptr_t) chunk->cb_handle;
1182 if (queue_type == QUEUE_TYPE_EXT || queue_type == QUEUE_TYPE_HW)
1183 int_queues_only = false;
1185 job = hl_cs_allocate_job(hdev, queue_type,
1186 is_kernel_allocated_cb);
1188 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
1189 atomic64_inc(&cntr->out_of_mem_drop_cnt);
1190 dev_err(hdev->dev, "Failed to allocate a new job\n");
1192 if (is_kernel_allocated_cb)
1195 goto free_cs_object;
1201 job->user_cb_size = chunk->cb_size;
1202 job->hw_queue_id = chunk->queue_index;
1204 cs->jobs_in_queue_cnt[job->hw_queue_id]++;
1206 list_add_tail(&job->cs_node, &cs->job_list);
1209 * Increment CS reference. When CS reference is 0, CS is
1210 * done and can be signaled to user and free all its resources
1211 * Only increment for JOB on external or H/W queues, because
1212 * only for those JOBs we get completion
1214 if (cs_needs_completion(cs) &&
1215 (job->queue_type == QUEUE_TYPE_EXT ||
1216 job->queue_type == QUEUE_TYPE_HW))
1219 hl_debugfs_add_job(hdev, job);
1221 rc = cs_parser(hpriv, job);
1223 atomic64_inc(&ctx->cs_counters.parsing_drop_cnt);
1224 atomic64_inc(&cntr->parsing_drop_cnt);
1226 "Failed to parse JOB %d.%llu.%d, err %d, rejecting the CS\n",
1227 cs->ctx->asid, cs->sequence, job->id, rc);
1228 goto free_cs_object;
1232 /* We allow a CS with any queue type combination as long as it does
1233 * not get a completion
1235 if (int_queues_only && cs_needs_completion(cs)) {
1236 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1237 atomic64_inc(&cntr->validation_drop_cnt);
1239 "Reject CS %d.%llu since it contains only internal queues jobs and needs completion\n",
1240 cs->ctx->asid, cs->sequence);
1242 goto free_cs_object;
1245 rc = hl_hw_queue_schedule_cs(cs);
1249 "Failed to submit CS %d.%llu to H/W queues, error %d\n",
1250 cs->ctx->asid, cs->sequence, rc);
1251 goto free_cs_object;
1254 rc = HL_CS_STATUS_SUCCESS;
1258 atomic_dec(&cb->cs_cnt);
1261 cs_rollback(hdev, cs);
1262 *cs_seq = ULLONG_MAX;
1263 /* The path below is both for good and erroneous exits */
1265 /* We finished with the CS in this function, so put the ref */
1267 free_cs_chunk_array:
1268 kfree(cs_chunk_array);
1273 static int pending_cb_create_job(struct hl_device *hdev, struct hl_ctx *ctx,
1274 struct hl_cs *cs, struct hl_cb *cb, u32 size, u32 hw_queue_id)
1276 struct hw_queue_properties *hw_queue_prop;
1277 struct hl_cs_counters_atomic *cntr;
1278 struct hl_cs_job *job;
1280 hw_queue_prop = &hdev->asic_prop.hw_queues_props[hw_queue_id];
1281 cntr = &hdev->aggregated_cs_counters;
1283 job = hl_cs_allocate_job(hdev, hw_queue_prop->type, true);
1285 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
1286 atomic64_inc(&cntr->out_of_mem_drop_cnt);
1287 dev_err(hdev->dev, "Failed to allocate a new job\n");
1294 atomic_inc(&job->user_cb->cs_cnt);
1295 job->user_cb_size = size;
1296 job->hw_queue_id = hw_queue_id;
1297 job->patched_cb = job->user_cb;
1298 job->job_cb_size = job->user_cb_size;
1300 /* increment refcount as for external queues we get completion */
1303 cs->jobs_in_queue_cnt[job->hw_queue_id]++;
1305 list_add_tail(&job->cs_node, &cs->job_list);
1307 hl_debugfs_add_job(hdev, job);
1312 static int hl_submit_pending_cb(struct hl_fpriv *hpriv)
1314 struct hl_device *hdev = hpriv->hdev;
1315 struct hl_ctx *ctx = hpriv->ctx;
1316 struct hl_pending_cb *pending_cb, *tmp;
1317 struct list_head local_cb_list;
1322 int process_list, rc = 0;
1324 if (list_empty(&ctx->pending_cb_list))
1327 process_list = atomic_cmpxchg(&ctx->thread_pending_cb_token, 1, 0);
1329 /* Only a single thread is allowed to process the list */
1333 if (list_empty(&ctx->pending_cb_list))
1334 goto free_pending_cb_token;
1336 /* move all list elements to a local list */
1337 INIT_LIST_HEAD(&local_cb_list);
1338 spin_lock(&ctx->pending_cb_lock);
1339 list_for_each_entry_safe(pending_cb, tmp, &ctx->pending_cb_list,
1341 list_move_tail(&pending_cb->cb_node, &local_cb_list);
1342 spin_unlock(&ctx->pending_cb_lock);
1344 rc = allocate_cs(hdev, ctx, CS_TYPE_DEFAULT, ULLONG_MAX, &cs, 0,
1345 hdev->timeout_jiffies);
1347 goto add_list_elements;
1349 hl_debugfs_add_cs(cs);
1351 /* Iterate through pending cb list, create jobs and add to CS */
1352 list_for_each_entry(pending_cb, &local_cb_list, cb_node) {
1353 cb = pending_cb->cb;
1354 cb_size = pending_cb->cb_size;
1355 hw_queue_id = pending_cb->hw_queue_id;
1357 rc = pending_cb_create_job(hdev, ctx, cs, cb, cb_size,
1360 goto free_cs_object;
1363 rc = hl_hw_queue_schedule_cs(cs);
1367 "Failed to submit CS %d.%llu (%d)\n",
1368 ctx->asid, cs->sequence, rc);
1369 goto free_cs_object;
1372 /* pending cb was scheduled successfully */
1373 list_for_each_entry_safe(pending_cb, tmp, &local_cb_list, cb_node) {
1374 list_del(&pending_cb->cb_node);
1380 goto free_pending_cb_token;
1383 cs_rollback(hdev, cs);
1386 spin_lock(&ctx->pending_cb_lock);
1387 list_for_each_entry_safe_reverse(pending_cb, tmp, &local_cb_list,
1389 list_move(&pending_cb->cb_node, &ctx->pending_cb_list);
1390 spin_unlock(&ctx->pending_cb_lock);
1391 free_pending_cb_token:
1392 atomic_set(&ctx->thread_pending_cb_token, 1);
1397 static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args,
1400 struct hl_device *hdev = hpriv->hdev;
1401 struct hl_ctx *ctx = hpriv->ctx;
1402 bool need_soft_reset = false;
1403 int rc = 0, do_ctx_switch;
1404 void __user *chunks;
1405 u32 num_chunks, tmp;
1408 do_ctx_switch = atomic_cmpxchg(&ctx->thread_ctx_switch_token, 1, 0);
1410 if (do_ctx_switch || (args->in.cs_flags & HL_CS_FLAGS_FORCE_RESTORE)) {
1411 mutex_lock(&hpriv->restore_phase_mutex);
1413 if (do_ctx_switch) {
1414 rc = hdev->asic_funcs->context_switch(hdev, ctx->asid);
1416 dev_err_ratelimited(hdev->dev,
1417 "Failed to switch to context %d, rejecting CS! %d\n",
1420 * If we timedout, or if the device is not IDLE
1421 * while we want to do context-switch (-EBUSY),
1422 * we need to soft-reset because QMAN is
1423 * probably stuck. However, we can't call to
1424 * reset here directly because of deadlock, so
1425 * need to do it at the very end of this
1428 if ((rc == -ETIMEDOUT) || (rc == -EBUSY))
1429 need_soft_reset = true;
1430 mutex_unlock(&hpriv->restore_phase_mutex);
1435 hdev->asic_funcs->restore_phase_topology(hdev);
1437 chunks = (void __user *) (uintptr_t) args->in.chunks_restore;
1438 num_chunks = args->in.num_chunks_restore;
1442 "Need to run restore phase but restore CS is empty\n");
1445 rc = cs_ioctl_default(hpriv, chunks, num_chunks,
1446 cs_seq, 0, hdev->timeout_jiffies);
1449 mutex_unlock(&hpriv->restore_phase_mutex);
1453 "Failed to submit restore CS for context %d (%d)\n",
1458 /* Need to wait for restore completion before execution phase */
1460 enum hl_cs_wait_status status;
1462 ret = _hl_cs_wait_ioctl(hdev, ctx,
1463 jiffies_to_usecs(hdev->timeout_jiffies),
1464 *cs_seq, &status, NULL);
1466 if (ret == -ERESTARTSYS) {
1467 usleep_range(100, 200);
1472 "Restore CS for context %d failed to complete %d\n",
1479 ctx->thread_ctx_switch_wait_token = 1;
1481 } else if (!ctx->thread_ctx_switch_wait_token) {
1482 rc = hl_poll_timeout_memory(hdev,
1483 &ctx->thread_ctx_switch_wait_token, tmp, (tmp == 1),
1484 100, jiffies_to_usecs(hdev->timeout_jiffies), false);
1486 if (rc == -ETIMEDOUT) {
1488 "context switch phase timeout (%d)\n", tmp);
1494 if ((rc == -ETIMEDOUT || rc == -EBUSY) && (need_soft_reset))
1495 hl_device_reset(hdev, 0);
1501 * hl_cs_signal_sob_wraparound_handler: handle SOB value wrapaound case.
1502 * if the SOB value reaches the max value move to the other SOB reserved
1504 * Note that this function must be called while hw_queues_lock is taken.
1506 int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx,
1507 struct hl_hw_sob **hw_sob, u32 count)
1509 struct hl_sync_stream_properties *prop;
1510 struct hl_hw_sob *sob = *hw_sob, *other_sob;
1511 u8 other_sob_offset;
1513 prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
1515 kref_get(&sob->kref);
1517 /* check for wraparound */
1518 if (prop->next_sob_val + count >= HL_MAX_SOB_VAL) {
1520 * Decrement as we reached the max value.
1521 * The release function won't be called here as we've
1522 * just incremented the refcount right before calling this
1525 kref_put(&sob->kref, hl_sob_reset_error);
1528 * check the other sob value, if it still in use then fail
1529 * otherwise make the switch
1531 other_sob_offset = (prop->curr_sob_offset + 1) % HL_RSVD_SOBS;
1532 other_sob = &prop->hw_sob[other_sob_offset];
1534 if (kref_read(&other_sob->kref) != 1) {
1535 dev_err(hdev->dev, "error: Cannot switch SOBs q_idx: %d\n",
1540 prop->next_sob_val = 1;
1542 /* only two SOBs are currently in use */
1543 prop->curr_sob_offset = other_sob_offset;
1544 *hw_sob = other_sob;
1546 dev_dbg(hdev->dev, "switched to SOB %d, q_idx: %d\n",
1547 prop->curr_sob_offset, q_idx);
1549 prop->next_sob_val += count;
1555 static int cs_ioctl_extract_signal_seq(struct hl_device *hdev,
1556 struct hl_cs_chunk *chunk, u64 *signal_seq, struct hl_ctx *ctx)
1558 u64 *signal_seq_arr = NULL;
1559 u32 size_to_copy, signal_seq_arr_len;
1562 signal_seq_arr_len = chunk->num_signal_seq_arr;
1564 /* currently only one signal seq is supported */
1565 if (signal_seq_arr_len != 1) {
1566 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1567 atomic64_inc(&hdev->aggregated_cs_counters.validation_drop_cnt);
1569 "Wait for signal CS supports only one signal CS seq\n");
1573 signal_seq_arr = kmalloc_array(signal_seq_arr_len,
1574 sizeof(*signal_seq_arr),
1576 if (!signal_seq_arr)
1577 signal_seq_arr = kmalloc_array(signal_seq_arr_len,
1578 sizeof(*signal_seq_arr),
1580 if (!signal_seq_arr) {
1581 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
1582 atomic64_inc(&hdev->aggregated_cs_counters.out_of_mem_drop_cnt);
1586 size_to_copy = chunk->num_signal_seq_arr * sizeof(*signal_seq_arr);
1587 if (copy_from_user(signal_seq_arr,
1588 u64_to_user_ptr(chunk->signal_seq_arr),
1590 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1591 atomic64_inc(&hdev->aggregated_cs_counters.validation_drop_cnt);
1593 "Failed to copy signal seq array from user\n");
1598 /* currently it is guaranteed to have only one signal seq */
1599 *signal_seq = signal_seq_arr[0];
1602 kfree(signal_seq_arr);
1607 static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev,
1608 struct hl_ctx *ctx, struct hl_cs *cs, enum hl_queue_type q_type,
1611 struct hl_cs_counters_atomic *cntr;
1612 struct hl_cs_job *job;
1616 cntr = &hdev->aggregated_cs_counters;
1618 job = hl_cs_allocate_job(hdev, q_type, true);
1620 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
1621 atomic64_inc(&cntr->out_of_mem_drop_cnt);
1622 dev_err(hdev->dev, "Failed to allocate a new job\n");
1626 if (cs->type == CS_TYPE_WAIT)
1627 cb_size = hdev->asic_funcs->get_wait_cb_size(hdev);
1629 cb_size = hdev->asic_funcs->get_signal_cb_size(hdev);
1631 cb = hl_cb_kernel_create(hdev, cb_size,
1632 q_type == QUEUE_TYPE_HW && hdev->mmu_enable);
1634 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
1635 atomic64_inc(&cntr->out_of_mem_drop_cnt);
1643 atomic_inc(&job->user_cb->cs_cnt);
1644 job->user_cb_size = cb_size;
1645 job->hw_queue_id = q_idx;
1648 * No need in parsing, user CB is the patched CB.
1649 * We call hl_cb_destroy() out of two reasons - we don't need the CB in
1650 * the CB idr anymore and to decrement its refcount as it was
1651 * incremented inside hl_cb_kernel_create().
1653 job->patched_cb = job->user_cb;
1654 job->job_cb_size = job->user_cb_size;
1655 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
1657 /* increment refcount as for external queues we get completion */
1660 cs->jobs_in_queue_cnt[job->hw_queue_id]++;
1662 list_add_tail(&job->cs_node, &cs->job_list);
1664 hl_debugfs_add_job(hdev, job);
1669 static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
1670 void __user *chunks, u32 num_chunks,
1671 u64 *cs_seq, u32 flags, u32 timeout)
1673 struct hl_cs_chunk *cs_chunk_array, *chunk;
1674 struct hw_queue_properties *hw_queue_prop;
1675 struct hl_device *hdev = hpriv->hdev;
1676 struct hl_cs_compl *sig_waitcs_cmpl;
1677 u32 q_idx, collective_engine_id = 0;
1678 struct hl_cs_counters_atomic *cntr;
1679 struct hl_fence *sig_fence = NULL;
1680 struct hl_ctx *ctx = hpriv->ctx;
1681 enum hl_queue_type q_type;
1686 cntr = &hdev->aggregated_cs_counters;
1687 *cs_seq = ULLONG_MAX;
1689 rc = hl_cs_copy_chunk_array(hdev, &cs_chunk_array, chunks, num_chunks,
1694 /* currently it is guaranteed to have only one chunk */
1695 chunk = &cs_chunk_array[0];
1697 if (chunk->queue_index >= hdev->asic_prop.max_queues) {
1698 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1699 atomic64_inc(&cntr->validation_drop_cnt);
1700 dev_err(hdev->dev, "Queue index %d is invalid\n",
1701 chunk->queue_index);
1703 goto free_cs_chunk_array;
1706 q_idx = chunk->queue_index;
1707 hw_queue_prop = &hdev->asic_prop.hw_queues_props[q_idx];
1708 q_type = hw_queue_prop->type;
1710 if (!hw_queue_prop->supports_sync_stream) {
1711 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1712 atomic64_inc(&cntr->validation_drop_cnt);
1714 "Queue index %d does not support sync stream operations\n",
1717 goto free_cs_chunk_array;
1720 if (cs_type == CS_TYPE_COLLECTIVE_WAIT) {
1721 if (!(hw_queue_prop->collective_mode == HL_COLLECTIVE_MASTER)) {
1722 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1723 atomic64_inc(&cntr->validation_drop_cnt);
1725 "Queue index %d is invalid\n", q_idx);
1727 goto free_cs_chunk_array;
1730 collective_engine_id = chunk->collective_engine_id;
1733 if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_COLLECTIVE_WAIT) {
1734 rc = cs_ioctl_extract_signal_seq(hdev, chunk, &signal_seq, ctx);
1736 goto free_cs_chunk_array;
1738 sig_fence = hl_ctx_get_fence(ctx, signal_seq);
1739 if (IS_ERR(sig_fence)) {
1740 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1741 atomic64_inc(&cntr->validation_drop_cnt);
1743 "Failed to get signal CS with seq 0x%llx\n",
1745 rc = PTR_ERR(sig_fence);
1746 goto free_cs_chunk_array;
1750 /* signal CS already finished */
1752 goto free_cs_chunk_array;
1756 container_of(sig_fence, struct hl_cs_compl, base_fence);
1758 if (sig_waitcs_cmpl->type != CS_TYPE_SIGNAL) {
1759 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1760 atomic64_inc(&cntr->validation_drop_cnt);
1762 "CS seq 0x%llx is not of a signal CS\n",
1764 hl_fence_put(sig_fence);
1766 goto free_cs_chunk_array;
1769 if (completion_done(&sig_fence->completion)) {
1770 /* signal CS already finished */
1771 hl_fence_put(sig_fence);
1773 goto free_cs_chunk_array;
1777 rc = allocate_cs(hdev, ctx, cs_type, ULLONG_MAX, &cs, flags, timeout);
1779 if (cs_type == CS_TYPE_WAIT ||
1780 cs_type == CS_TYPE_COLLECTIVE_WAIT)
1781 hl_fence_put(sig_fence);
1782 goto free_cs_chunk_array;
1786 * Save the signal CS fence for later initialization right before
1787 * hanging the wait CS on the queue.
1789 if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_COLLECTIVE_WAIT)
1790 cs->signal_fence = sig_fence;
1792 hl_debugfs_add_cs(cs);
1794 *cs_seq = cs->sequence;
1796 if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_SIGNAL)
1797 rc = cs_ioctl_signal_wait_create_jobs(hdev, ctx, cs, q_type,
1799 else if (cs_type == CS_TYPE_COLLECTIVE_WAIT)
1800 rc = hdev->asic_funcs->collective_wait_create_jobs(hdev, ctx,
1801 cs, q_idx, collective_engine_id);
1803 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1804 atomic64_inc(&cntr->validation_drop_cnt);
1809 goto free_cs_object;
1811 rc = hl_hw_queue_schedule_cs(cs);
1815 "Failed to submit CS %d.%llu to H/W queues, error %d\n",
1816 ctx->asid, cs->sequence, rc);
1817 goto free_cs_object;
1820 rc = HL_CS_STATUS_SUCCESS;
1824 cs_rollback(hdev, cs);
1825 *cs_seq = ULLONG_MAX;
1826 /* The path below is both for good and erroneous exits */
1828 /* We finished with the CS in this function, so put the ref */
1830 free_cs_chunk_array:
1831 kfree(cs_chunk_array);
1836 int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
1838 union hl_cs_args *args = data;
1839 enum hl_cs_type cs_type;
1840 u64 cs_seq = ULONG_MAX;
1841 void __user *chunks;
1842 u32 num_chunks, flags, timeout;
1845 rc = hl_cs_sanity_checks(hpriv, args);
1849 rc = hl_cs_ctx_switch(hpriv, args, &cs_seq);
1853 rc = hl_submit_pending_cb(hpriv);
1857 cs_type = hl_cs_get_cs_type(args->in.cs_flags &
1858 ~HL_CS_FLAGS_FORCE_RESTORE);
1859 chunks = (void __user *) (uintptr_t) args->in.chunks_execute;
1860 num_chunks = args->in.num_chunks_execute;
1861 flags = args->in.cs_flags;
1863 /* In case this is a staged CS, user should supply the CS sequence */
1864 if ((flags & HL_CS_FLAGS_STAGED_SUBMISSION) &&
1865 !(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST))
1866 cs_seq = args->in.seq;
1868 timeout = flags & HL_CS_FLAGS_CUSTOM_TIMEOUT
1869 ? msecs_to_jiffies(args->in.timeout * 1000)
1870 : hpriv->hdev->timeout_jiffies;
1873 case CS_TYPE_SIGNAL:
1875 case CS_TYPE_COLLECTIVE_WAIT:
1876 rc = cs_ioctl_signal_wait(hpriv, cs_type, chunks, num_chunks,
1877 &cs_seq, args->in.cs_flags, timeout);
1880 rc = cs_ioctl_default(hpriv, chunks, num_chunks, &cs_seq,
1881 args->in.cs_flags, timeout);
1886 if (rc != -EAGAIN) {
1887 memset(args, 0, sizeof(*args));
1888 args->out.status = rc;
1889 args->out.seq = cs_seq;
1895 static int _hl_cs_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
1896 u64 timeout_us, u64 seq,
1897 enum hl_cs_wait_status *status, s64 *timestamp)
1899 struct hl_fence *fence;
1900 unsigned long timeout;
1907 if (timeout_us == MAX_SCHEDULE_TIMEOUT)
1908 timeout = timeout_us;
1910 timeout = usecs_to_jiffies(timeout_us);
1912 hl_ctx_get(hdev, ctx);
1914 fence = hl_ctx_get_fence(ctx, seq);
1915 if (IS_ERR(fence)) {
1916 rc = PTR_ERR(fence);
1918 dev_notice_ratelimited(hdev->dev,
1919 "Can't wait on CS %llu because current CS is at seq %llu\n",
1920 seq, ctx->cs_sequence);
1923 completion_rc = completion_done(&fence->completion);
1926 wait_for_completion_interruptible_timeout(
1927 &fence->completion, timeout);
1929 if (completion_rc > 0) {
1930 *status = CS_WAIT_STATUS_COMPLETED;
1932 *timestamp = ktime_to_ns(fence->timestamp);
1934 *status = CS_WAIT_STATUS_BUSY;
1937 if (fence->error == -ETIMEDOUT)
1939 else if (fence->error == -EIO)
1942 hl_fence_put(fence);
1945 "Can't wait on seq %llu because current CS is at seq %llu (Fence is gone)\n",
1946 seq, ctx->cs_sequence);
1947 *status = CS_WAIT_STATUS_GONE;
1955 static int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data)
1957 struct hl_device *hdev = hpriv->hdev;
1958 union hl_wait_cs_args *args = data;
1959 enum hl_cs_wait_status status;
1960 u64 seq = args->in.seq;
1964 rc = _hl_cs_wait_ioctl(hdev, hpriv->ctx, args->in.timeout_us, seq,
1965 &status, ×tamp);
1967 memset(args, 0, sizeof(*args));
1970 if (rc == -ERESTARTSYS) {
1971 dev_err_ratelimited(hdev->dev,
1972 "user process got signal while waiting for CS handle %llu\n",
1974 args->out.status = HL_WAIT_CS_STATUS_INTERRUPTED;
1976 } else if (rc == -ETIMEDOUT) {
1977 dev_err_ratelimited(hdev->dev,
1978 "CS %llu has timed-out while user process is waiting for it\n",
1980 args->out.status = HL_WAIT_CS_STATUS_TIMEDOUT;
1981 } else if (rc == -EIO) {
1982 dev_err_ratelimited(hdev->dev,
1983 "CS %llu has been aborted while user process is waiting for it\n",
1985 args->out.status = HL_WAIT_CS_STATUS_ABORTED;
1991 args->out.flags |= HL_WAIT_CS_STATUS_FLAG_TIMESTAMP_VLD;
1992 args->out.timestamp_nsec = timestamp;
1996 case CS_WAIT_STATUS_GONE:
1997 args->out.flags |= HL_WAIT_CS_STATUS_FLAG_GONE;
1999 case CS_WAIT_STATUS_COMPLETED:
2000 args->out.status = HL_WAIT_CS_STATUS_COMPLETED;
2002 case CS_WAIT_STATUS_BUSY:
2004 args->out.status = HL_WAIT_CS_STATUS_BUSY;
2011 static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
2012 u32 timeout_us, u64 user_address,
2013 u32 target_value, u16 interrupt_offset,
2014 enum hl_cs_wait_status *status)
2016 struct hl_user_pending_interrupt *pend;
2017 struct hl_user_interrupt *interrupt;
2018 unsigned long timeout;
2020 u32 completion_value;
2023 if (timeout_us == U32_MAX)
2024 timeout = timeout_us;
2026 timeout = usecs_to_jiffies(timeout_us);
2028 hl_ctx_get(hdev, ctx);
2030 pend = kmalloc(sizeof(*pend), GFP_KERNEL);
2036 hl_fence_init(&pend->fence, ULONG_MAX);
2038 if (interrupt_offset == HL_COMMON_USER_INTERRUPT_ID)
2039 interrupt = &hdev->common_user_interrupt;
2041 interrupt = &hdev->user_interrupt[interrupt_offset];
2043 spin_lock(&interrupt->wait_list_lock);
2044 if (!hl_device_operational(hdev, NULL)) {
2046 goto unlock_and_free_fence;
2049 if (copy_from_user(&completion_value, u64_to_user_ptr(user_address), 4)) {
2051 "Failed to copy completion value from user\n");
2053 goto unlock_and_free_fence;
2056 if (completion_value >= target_value)
2057 *status = CS_WAIT_STATUS_COMPLETED;
2059 *status = CS_WAIT_STATUS_BUSY;
2061 if (!timeout_us || (*status == CS_WAIT_STATUS_COMPLETED))
2062 goto unlock_and_free_fence;
2064 /* Add pending user interrupt to relevant list for the interrupt
2065 * handler to monitor
2067 list_add_tail(&pend->wait_list_node, &interrupt->wait_list_head);
2068 spin_unlock(&interrupt->wait_list_lock);
2071 /* Wait for interrupt handler to signal completion */
2073 wait_for_completion_interruptible_timeout(
2074 &pend->fence.completion, timeout);
2076 /* If timeout did not expire we need to perform the comparison.
2077 * If comparison fails, keep waiting until timeout expires
2079 if (completion_rc > 0) {
2080 if (copy_from_user(&completion_value,
2081 u64_to_user_ptr(user_address), 4)) {
2083 "Failed to copy completion value from user\n");
2085 goto remove_pending_user_interrupt;
2088 if (completion_value >= target_value) {
2089 *status = CS_WAIT_STATUS_COMPLETED;
2091 timeout = completion_rc;
2095 *status = CS_WAIT_STATUS_BUSY;
2098 remove_pending_user_interrupt:
2099 spin_lock(&interrupt->wait_list_lock);
2100 list_del(&pend->wait_list_node);
2102 unlock_and_free_fence:
2103 spin_unlock(&interrupt->wait_list_lock);
2110 static int hl_interrupt_wait_ioctl(struct hl_fpriv *hpriv, void *data)
2112 u16 interrupt_id, interrupt_offset, first_interrupt, last_interrupt;
2113 struct hl_device *hdev = hpriv->hdev;
2114 struct asic_fixed_properties *prop;
2115 union hl_wait_cs_args *args = data;
2116 enum hl_cs_wait_status status;
2119 prop = &hdev->asic_prop;
2121 if (!prop->user_interrupt_count) {
2122 dev_err(hdev->dev, "no user interrupts allowed");
2127 FIELD_GET(HL_WAIT_CS_FLAGS_INTERRUPT_MASK, args->in.flags);
2129 first_interrupt = prop->first_available_user_msix_interrupt;
2130 last_interrupt = prop->first_available_user_msix_interrupt +
2131 prop->user_interrupt_count - 1;
2133 if ((interrupt_id < first_interrupt || interrupt_id > last_interrupt) &&
2134 interrupt_id != HL_COMMON_USER_INTERRUPT_ID) {
2135 dev_err(hdev->dev, "invalid user interrupt %u", interrupt_id);
2139 if (interrupt_id == HL_COMMON_USER_INTERRUPT_ID)
2140 interrupt_offset = HL_COMMON_USER_INTERRUPT_ID;
2142 interrupt_offset = interrupt_id - first_interrupt;
2144 rc = _hl_interrupt_wait_ioctl(hdev, hpriv->ctx,
2145 args->in.interrupt_timeout_us, args->in.addr,
2146 args->in.target, interrupt_offset, &status);
2148 memset(args, 0, sizeof(*args));
2151 dev_err_ratelimited(hdev->dev,
2152 "interrupt_wait_ioctl failed (%d)\n", rc);
2158 case CS_WAIT_STATUS_COMPLETED:
2159 args->out.status = HL_WAIT_CS_STATUS_COMPLETED;
2161 case CS_WAIT_STATUS_BUSY:
2163 args->out.status = HL_WAIT_CS_STATUS_BUSY;
2170 int hl_wait_ioctl(struct hl_fpriv *hpriv, void *data)
2172 union hl_wait_cs_args *args = data;
2173 u32 flags = args->in.flags;
2176 if (flags & HL_WAIT_CS_FLAGS_INTERRUPT)
2177 rc = hl_interrupt_wait_ioctl(hpriv, data);
2179 rc = hl_cs_wait_ioctl(hpriv, data);