1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #include <uapi/misc/habanalabs.h>
9 #include "habanalabs.h"
11 #include <linux/uaccess.h>
12 #include <linux/slab.h>
14 #define HL_CS_FLAGS_TYPE_MASK (HL_CS_FLAGS_SIGNAL | HL_CS_FLAGS_WAIT | \
15 HL_CS_FLAGS_COLLECTIVE_WAIT)
18 * enum hl_cs_wait_status - cs wait status
19 * @CS_WAIT_STATUS_BUSY: cs was not completed yet
20 * @CS_WAIT_STATUS_COMPLETED: cs completed
21 * @CS_WAIT_STATUS_GONE: cs completed but fence is already gone
23 enum hl_cs_wait_status {
25 CS_WAIT_STATUS_COMPLETED,
29 static void job_wq_completion(struct work_struct *work);
30 static int _hl_cs_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
31 u64 timeout_us, u64 seq,
32 enum hl_cs_wait_status *status, s64 *timestamp);
33 static void cs_do_release(struct kref *ref);
35 static void hl_sob_reset(struct kref *ref)
37 struct hl_hw_sob *hw_sob = container_of(ref, struct hl_hw_sob,
39 struct hl_device *hdev = hw_sob->hdev;
41 dev_dbg(hdev->dev, "reset sob id %u\n", hw_sob->sob_id);
43 hdev->asic_funcs->reset_sob(hdev, hw_sob);
45 hw_sob->need_reset = false;
48 void hl_sob_reset_error(struct kref *ref)
50 struct hl_hw_sob *hw_sob = container_of(ref, struct hl_hw_sob,
52 struct hl_device *hdev = hw_sob->hdev;
55 "SOB release shouldn't be called here, q_idx: %d, sob_id: %d\n",
56 hw_sob->q_idx, hw_sob->sob_id);
59 void hw_sob_put(struct hl_hw_sob *hw_sob)
62 kref_put(&hw_sob->kref, hl_sob_reset);
65 static void hw_sob_put_err(struct hl_hw_sob *hw_sob)
68 kref_put(&hw_sob->kref, hl_sob_reset_error);
71 void hw_sob_get(struct hl_hw_sob *hw_sob)
74 kref_get(&hw_sob->kref);
78 * hl_gen_sob_mask() - Generates a sob mask to be used in a monitor arm packet
79 * @sob_base: sob base id
80 * @sob_mask: sob user mask, each bit represents a sob offset from sob base
81 * @mask: generated mask
83 * Return: 0 if given parameters are valid
85 int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask)
92 if (sob_mask == 0x1) {
93 *mask = ~(1 << (sob_base & 0x7));
95 /* find msb in order to verify sob range is valid */
96 for (i = BITS_PER_BYTE - 1 ; i >= 0 ; i--)
97 if (BIT(i) & sob_mask)
100 if (i > (HL_MAX_SOBS_PER_MONITOR - (sob_base & 0x7) - 1))
109 static void hl_fence_release(struct kref *kref)
111 struct hl_fence *fence =
112 container_of(kref, struct hl_fence, refcount);
113 struct hl_cs_compl *hl_cs_cmpl =
114 container_of(fence, struct hl_cs_compl, base_fence);
119 void hl_fence_put(struct hl_fence *fence)
121 if (IS_ERR_OR_NULL(fence))
123 kref_put(&fence->refcount, hl_fence_release);
126 void hl_fences_put(struct hl_fence **fence, int len)
130 for (i = 0; i < len; i++, fence++)
131 hl_fence_put(*fence);
134 void hl_fence_get(struct hl_fence *fence)
137 kref_get(&fence->refcount);
140 static void hl_fence_init(struct hl_fence *fence, u64 sequence)
142 kref_init(&fence->refcount);
143 fence->cs_sequence = sequence;
145 fence->timestamp = ktime_set(0, 0);
146 init_completion(&fence->completion);
149 void cs_get(struct hl_cs *cs)
151 kref_get(&cs->refcount);
154 static int cs_get_unless_zero(struct hl_cs *cs)
156 return kref_get_unless_zero(&cs->refcount);
159 static void cs_put(struct hl_cs *cs)
161 kref_put(&cs->refcount, cs_do_release);
164 static void cs_job_do_release(struct kref *ref)
166 struct hl_cs_job *job = container_of(ref, struct hl_cs_job, refcount);
171 static void cs_job_put(struct hl_cs_job *job)
173 kref_put(&job->refcount, cs_job_do_release);
176 bool cs_needs_completion(struct hl_cs *cs)
178 /* In case this is a staged CS, only the last CS in sequence should
179 * get a completion, any non staged CS will always get a completion
181 if (cs->staged_cs && !cs->staged_last)
187 bool cs_needs_timeout(struct hl_cs *cs)
189 /* In case this is a staged CS, only the first CS in sequence should
190 * get a timeout, any non staged CS will always get a timeout
192 if (cs->staged_cs && !cs->staged_first)
198 static bool is_cb_patched(struct hl_device *hdev, struct hl_cs_job *job)
201 * Patched CB is created for external queues jobs, and for H/W queues
202 * jobs if the user CB was allocated by driver and MMU is disabled.
204 return (job->queue_type == QUEUE_TYPE_EXT ||
205 (job->queue_type == QUEUE_TYPE_HW &&
206 job->is_kernel_allocated_cb &&
211 * cs_parser - parse the user command submission
213 * @hpriv : pointer to the private data of the fd
214 * @job : pointer to the job that holds the command submission info
216 * The function parses the command submission of the user. It calls the
217 * ASIC specific parser, which returns a list of memory blocks to send
218 * to the device as different command buffers
221 static int cs_parser(struct hl_fpriv *hpriv, struct hl_cs_job *job)
223 struct hl_device *hdev = hpriv->hdev;
224 struct hl_cs_parser parser;
227 parser.ctx_id = job->cs->ctx->asid;
228 parser.cs_sequence = job->cs->sequence;
229 parser.job_id = job->id;
231 parser.hw_queue_id = job->hw_queue_id;
232 parser.job_userptr_list = &job->userptr_list;
233 parser.patched_cb = NULL;
234 parser.user_cb = job->user_cb;
235 parser.user_cb_size = job->user_cb_size;
236 parser.queue_type = job->queue_type;
237 parser.is_kernel_allocated_cb = job->is_kernel_allocated_cb;
238 job->patched_cb = NULL;
239 parser.completion = cs_needs_completion(job->cs);
241 rc = hdev->asic_funcs->cs_parser(hdev, &parser);
243 if (is_cb_patched(hdev, job)) {
245 job->patched_cb = parser.patched_cb;
246 job->job_cb_size = parser.patched_cb_size;
247 job->contains_dma_pkt = parser.contains_dma_pkt;
248 atomic_inc(&job->patched_cb->cs_cnt);
252 * Whether the parsing worked or not, we don't need the
253 * original CB anymore because it was already parsed and
254 * won't be accessed again for this CS
256 atomic_dec(&job->user_cb->cs_cnt);
257 hl_cb_put(job->user_cb);
260 job->job_cb_size = job->user_cb_size;
266 static void complete_job(struct hl_device *hdev, struct hl_cs_job *job)
268 struct hl_cs *cs = job->cs;
270 if (is_cb_patched(hdev, job)) {
271 hl_userptr_delete_list(hdev, &job->userptr_list);
274 * We might arrive here from rollback and patched CB wasn't
275 * created, so we need to check it's not NULL
277 if (job->patched_cb) {
278 atomic_dec(&job->patched_cb->cs_cnt);
279 hl_cb_put(job->patched_cb);
283 /* For H/W queue jobs, if a user CB was allocated by driver and MMU is
284 * enabled, the user CB isn't released in cs_parser() and thus should be
286 * This is also true for INT queues jobs which were allocated by driver
288 if (job->is_kernel_allocated_cb &&
289 ((job->queue_type == QUEUE_TYPE_HW && hdev->mmu_enable) ||
290 job->queue_type == QUEUE_TYPE_INT)) {
291 atomic_dec(&job->user_cb->cs_cnt);
292 hl_cb_put(job->user_cb);
296 * This is the only place where there can be multiple threads
297 * modifying the list at the same time
299 spin_lock(&cs->job_lock);
300 list_del(&job->cs_node);
301 spin_unlock(&cs->job_lock);
303 hl_debugfs_remove_job(hdev, job);
305 /* We decrement reference only for a CS that gets completion
306 * because the reference was incremented only for this kind of CS
307 * right before it was scheduled.
309 * In staged submission, only the last CS marked as 'staged_last'
310 * gets completion, hence its release function will be called from here.
311 * As for all the rest CS's in the staged submission which do not get
312 * completion, their CS reference will be decremented by the
313 * 'staged_last' CS during the CS release flow.
314 * All relevant PQ CI counters will be incremented during the CS release
315 * flow by calling 'hl_hw_queue_update_ci'.
317 if (cs_needs_completion(cs) &&
318 (job->queue_type == QUEUE_TYPE_EXT ||
319 job->queue_type == QUEUE_TYPE_HW))
326 * hl_staged_cs_find_first - locate the first CS in this staged submission
328 * @hdev: pointer to device structure
329 * @cs_seq: staged submission sequence number
331 * @note: This function must be called under 'hdev->cs_mirror_lock'
333 * Find and return a CS pointer with the given sequence
335 struct hl_cs *hl_staged_cs_find_first(struct hl_device *hdev, u64 cs_seq)
339 list_for_each_entry_reverse(cs, &hdev->cs_mirror_list, mirror_node)
340 if (cs->staged_cs && cs->staged_first &&
341 cs->sequence == cs_seq)
348 * is_staged_cs_last_exists - returns true if the last CS in sequence exists
350 * @hdev: pointer to device structure
351 * @cs: staged submission member
354 bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs)
356 struct hl_cs *last_entry;
358 last_entry = list_last_entry(&cs->staged_cs_node, struct hl_cs,
361 if (last_entry->staged_last)
368 * staged_cs_get - get CS reference if this CS is a part of a staged CS
370 * @hdev: pointer to device structure
372 * @cs_seq: staged submission sequence number
374 * Increment CS reference for every CS in this staged submission except for
375 * the CS which get completion.
377 static void staged_cs_get(struct hl_device *hdev, struct hl_cs *cs)
379 /* Only the last CS in this staged submission will get a completion.
380 * We must increment the reference for all other CS's in this
382 * Once we get a completion we will release the whole staged submission.
384 if (!cs->staged_last)
389 * staged_cs_put - put a CS in case it is part of staged submission
391 * @hdev: pointer to device structure
394 * This function decrements a CS reference (for a non completion CS)
396 static void staged_cs_put(struct hl_device *hdev, struct hl_cs *cs)
398 /* We release all CS's in a staged submission except the last
399 * CS which we have never incremented its reference.
401 if (!cs_needs_completion(cs))
405 static void cs_handle_tdr(struct hl_device *hdev, struct hl_cs *cs)
407 bool next_entry_found = false;
408 struct hl_cs *next, *first_cs;
410 if (!cs_needs_timeout(cs))
413 spin_lock(&hdev->cs_mirror_lock);
415 /* We need to handle tdr only once for the complete staged submission.
416 * Hence, we choose the CS that reaches this function first which is
417 * the CS marked as 'staged_last'.
418 * In case single staged cs was submitted which has both first and last
419 * indications, then "cs_find_first" below will return NULL, since we
420 * removed the cs node from the list before getting here,
421 * in such cases just continue with the cs to cancel it's TDR work.
423 if (cs->staged_cs && cs->staged_last) {
424 first_cs = hl_staged_cs_find_first(hdev, cs->staged_sequence);
429 spin_unlock(&hdev->cs_mirror_lock);
431 /* Don't cancel TDR in case this CS was timedout because we might be
432 * running from the TDR context
434 if (cs && (cs->timedout ||
435 hdev->timeout_jiffies == MAX_SCHEDULE_TIMEOUT))
438 if (cs && cs->tdr_active)
439 cancel_delayed_work_sync(&cs->work_tdr);
441 spin_lock(&hdev->cs_mirror_lock);
443 /* queue TDR for next CS */
444 list_for_each_entry(next, &hdev->cs_mirror_list, mirror_node)
445 if (cs_needs_timeout(next)) {
446 next_entry_found = true;
450 if (next_entry_found && !next->tdr_active) {
451 next->tdr_active = true;
452 schedule_delayed_work(&next->work_tdr, next->timeout_jiffies);
455 spin_unlock(&hdev->cs_mirror_lock);
459 * force_complete_multi_cs - complete all contexts that wait on multi-CS
461 * @hdev: pointer to habanalabs device structure
463 static void force_complete_multi_cs(struct hl_device *hdev)
467 for (i = 0; i < MULTI_CS_MAX_USER_CTX; i++) {
468 struct multi_cs_completion *mcs_compl;
470 mcs_compl = &hdev->multi_cs_completion[i];
472 spin_lock(&mcs_compl->lock);
474 if (!mcs_compl->used) {
475 spin_unlock(&mcs_compl->lock);
479 /* when calling force complete no context should be waiting on
481 * We are calling the function as a protection for such case
482 * to free any pending context and print error message
485 "multi-CS completion context %d still waiting when calling force completion\n",
487 complete_all(&mcs_compl->completion);
488 spin_unlock(&mcs_compl->lock);
493 * complete_multi_cs - complete all waiting entities on multi-CS
495 * @hdev: pointer to habanalabs device structure
497 * The function signals a waiting entity that has an overlapping stream masters
498 * with the completed CS.
500 * - a completed CS worked on stream master QID 4, multi CS completion
501 * is actively waiting on stream master QIDs 3, 5. don't send signal as no
502 * common stream master QID
503 * - a completed CS worked on stream master QID 4, multi CS completion
504 * is actively waiting on stream master QIDs 3, 4. send signal as stream
505 * master QID 4 is common
507 static void complete_multi_cs(struct hl_device *hdev, struct hl_cs *cs)
509 struct hl_fence *fence = cs->fence;
512 /* in case of multi CS check for completion only for the first CS */
513 if (cs->staged_cs && !cs->staged_first)
516 for (i = 0; i < MULTI_CS_MAX_USER_CTX; i++) {
517 struct multi_cs_completion *mcs_compl;
519 mcs_compl = &hdev->multi_cs_completion[i];
520 if (!mcs_compl->used)
523 spin_lock(&mcs_compl->lock);
527 * 1. still waiting for completion
528 * 2. the completed CS has at least one overlapping stream
529 * master with the stream masters in the completion
531 if (mcs_compl->used &&
532 (fence->stream_master_qid_map &
533 mcs_compl->stream_master_qid_map)) {
534 /* extract the timestamp only of first completed CS */
535 if (!mcs_compl->timestamp)
536 mcs_compl->timestamp =
537 ktime_to_ns(fence->timestamp);
538 complete_all(&mcs_compl->completion);
541 spin_unlock(&mcs_compl->lock);
545 static inline void cs_release_sob_reset_handler(struct hl_device *hdev,
547 struct hl_cs_compl *hl_cs_cmpl)
549 /* Skip this handler if the cs wasn't submitted, to avoid putting
550 * the hw_sob twice, since this case already handled at this point,
551 * also skip if the hw_sob pointer wasn't set.
553 if (!hl_cs_cmpl->hw_sob || !cs->submitted)
556 spin_lock(&hl_cs_cmpl->lock);
559 * we get refcount upon reservation of signals or signal/wait cs for the
560 * hw_sob object, and need to put it when the first staged cs
561 * (which cotains the encaps signals) or cs signal/wait is completed.
563 if ((hl_cs_cmpl->type == CS_TYPE_SIGNAL) ||
564 (hl_cs_cmpl->type == CS_TYPE_WAIT) ||
565 (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT) ||
566 (!!hl_cs_cmpl->encaps_signals)) {
568 "CS 0x%llx type %d finished, sob_id: %d, sob_val: %u\n",
571 hl_cs_cmpl->hw_sob->sob_id,
572 hl_cs_cmpl->sob_val);
574 hw_sob_put(hl_cs_cmpl->hw_sob);
576 if (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT)
577 hdev->asic_funcs->reset_sob_group(hdev,
578 hl_cs_cmpl->sob_group);
581 spin_unlock(&hl_cs_cmpl->lock);
584 static void cs_do_release(struct kref *ref)
586 struct hl_cs *cs = container_of(ref, struct hl_cs, refcount);
587 struct hl_device *hdev = cs->ctx->hdev;
588 struct hl_cs_job *job, *tmp;
589 struct hl_cs_compl *hl_cs_cmpl =
590 container_of(cs->fence, struct hl_cs_compl, base_fence);
592 cs->completed = true;
595 * Although if we reached here it means that all external jobs have
596 * finished, because each one of them took refcnt to CS, we still
597 * need to go over the internal jobs and complete them. Otherwise, we
598 * will have leaked memory and what's worse, the CS object (and
599 * potentially the CTX object) could be released, while the JOB
600 * still holds a pointer to them (but no reference).
602 list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)
603 complete_job(hdev, job);
605 if (!cs->submitted) {
607 * In case the wait for signal CS was submitted, the fence put
608 * occurs in init_signal_wait_cs() or collective_wait_init_cs()
609 * right before hanging on the PQ.
611 if (cs->type == CS_TYPE_WAIT ||
612 cs->type == CS_TYPE_COLLECTIVE_WAIT)
613 hl_fence_put(cs->signal_fence);
618 /* Need to update CI for all queue jobs that does not get completion */
619 hl_hw_queue_update_ci(cs);
621 /* remove CS from CS mirror list */
622 spin_lock(&hdev->cs_mirror_lock);
623 list_del_init(&cs->mirror_node);
624 spin_unlock(&hdev->cs_mirror_lock);
626 cs_handle_tdr(hdev, cs);
629 /* the completion CS decrements reference for the entire
632 if (cs->staged_last) {
633 struct hl_cs *staged_cs, *tmp;
635 list_for_each_entry_safe(staged_cs, tmp,
636 &cs->staged_cs_node, staged_cs_node)
637 staged_cs_put(hdev, staged_cs);
640 /* A staged CS will be a member in the list only after it
641 * was submitted. We used 'cs_mirror_lock' when inserting
642 * it to list so we will use it again when removing it
645 spin_lock(&hdev->cs_mirror_lock);
646 list_del(&cs->staged_cs_node);
647 spin_unlock(&hdev->cs_mirror_lock);
650 /* decrement refcount to handle when first staged cs
651 * with encaps signals is completed.
653 if (hl_cs_cmpl->encaps_signals)
654 kref_put(&hl_cs_cmpl->encaps_sig_hdl->refcount,
655 hl_encaps_handle_do_release);
658 if ((cs->type == CS_TYPE_WAIT || cs->type == CS_TYPE_COLLECTIVE_WAIT)
659 && cs->encaps_signals)
660 kref_put(&cs->encaps_sig_hdl->refcount,
661 hl_encaps_handle_do_release);
664 /* Must be called before hl_ctx_put because inside we use ctx to get
667 hl_debugfs_remove_cs(cs);
671 /* We need to mark an error for not submitted because in that case
672 * the hl fence release flow is different. Mainly, we don't need
673 * to handle hw_sob for signal/wait
676 cs->fence->error = -ETIMEDOUT;
677 else if (cs->aborted)
678 cs->fence->error = -EIO;
679 else if (!cs->submitted)
680 cs->fence->error = -EBUSY;
682 if (unlikely(cs->skip_reset_on_timeout)) {
684 "Command submission %llu completed after %llu (s)\n",
686 div_u64(jiffies - cs->submission_time_jiffies, HZ));
690 cs->fence->timestamp = ktime_get();
691 complete_all(&cs->fence->completion);
692 complete_multi_cs(hdev, cs);
694 cs_release_sob_reset_handler(hdev, cs, hl_cs_cmpl);
696 hl_fence_put(cs->fence);
698 kfree(cs->jobs_in_queue_cnt);
702 static void cs_timedout(struct work_struct *work)
704 struct hl_device *hdev;
706 struct hl_cs *cs = container_of(work, struct hl_cs,
708 bool skip_reset_on_timeout = cs->skip_reset_on_timeout;
710 rc = cs_get_unless_zero(cs);
714 if ((!cs->submitted) || (cs->completed)) {
719 /* Mark the CS is timed out so we won't try to cancel its TDR */
720 if (likely(!skip_reset_on_timeout))
723 hdev = cs->ctx->hdev;
728 "Signal command submission %llu has not finished in time!\n",
734 "Wait command submission %llu has not finished in time!\n",
738 case CS_TYPE_COLLECTIVE_WAIT:
740 "Collective Wait command submission %llu has not finished in time!\n",
746 "Command submission %llu has not finished in time!\n",
751 rc = hl_state_dump(hdev);
753 dev_err(hdev->dev, "Error during system state dump %d\n", rc);
757 if (likely(!skip_reset_on_timeout)) {
758 if (hdev->reset_on_lockup)
759 hl_device_reset(hdev, HL_RESET_TDR);
761 hdev->needs_reset = true;
765 static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
766 enum hl_cs_type cs_type, u64 user_sequence,
767 struct hl_cs **cs_new, u32 flags, u32 timeout)
769 struct hl_cs_counters_atomic *cntr;
770 struct hl_fence *other = NULL;
771 struct hl_cs_compl *cs_cmpl;
775 cntr = &hdev->aggregated_cs_counters;
777 cs = kzalloc(sizeof(*cs), GFP_ATOMIC);
779 cs = kzalloc(sizeof(*cs), GFP_KERNEL);
782 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
783 atomic64_inc(&cntr->out_of_mem_drop_cnt);
787 /* increment refcnt for context */
788 hl_ctx_get(hdev, ctx);
791 cs->submitted = false;
792 cs->completed = false;
794 cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP);
795 cs->encaps_signals = !!(flags & HL_CS_FLAGS_ENCAP_SIGNALS);
796 cs->timeout_jiffies = timeout;
797 cs->skip_reset_on_timeout =
798 hdev->skip_reset_on_timeout ||
799 !!(flags & HL_CS_FLAGS_SKIP_RESET_ON_TIMEOUT);
800 cs->submission_time_jiffies = jiffies;
801 INIT_LIST_HEAD(&cs->job_list);
802 INIT_DELAYED_WORK(&cs->work_tdr, cs_timedout);
803 kref_init(&cs->refcount);
804 spin_lock_init(&cs->job_lock);
806 cs_cmpl = kzalloc(sizeof(*cs_cmpl), GFP_ATOMIC);
808 cs_cmpl = kzalloc(sizeof(*cs_cmpl), GFP_KERNEL);
811 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
812 atomic64_inc(&cntr->out_of_mem_drop_cnt);
817 cs->jobs_in_queue_cnt = kcalloc(hdev->asic_prop.max_queues,
818 sizeof(*cs->jobs_in_queue_cnt), GFP_ATOMIC);
819 if (!cs->jobs_in_queue_cnt)
820 cs->jobs_in_queue_cnt = kcalloc(hdev->asic_prop.max_queues,
821 sizeof(*cs->jobs_in_queue_cnt), GFP_KERNEL);
823 if (!cs->jobs_in_queue_cnt) {
824 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
825 atomic64_inc(&cntr->out_of_mem_drop_cnt);
830 cs_cmpl->hdev = hdev;
831 cs_cmpl->type = cs->type;
832 spin_lock_init(&cs_cmpl->lock);
833 cs->fence = &cs_cmpl->base_fence;
835 spin_lock(&ctx->cs_lock);
837 cs_cmpl->cs_seq = ctx->cs_sequence;
838 other = ctx->cs_pending[cs_cmpl->cs_seq &
839 (hdev->asic_prop.max_pending_cs - 1)];
841 if (other && !completion_done(&other->completion)) {
842 /* If the following statement is true, it means we have reached
843 * a point in which only part of the staged submission was
844 * submitted and we don't have enough room in the 'cs_pending'
845 * array for the rest of the submission.
846 * This causes a deadlock because this CS will never be
847 * completed as it depends on future CS's for completion.
849 if (other->cs_sequence == user_sequence)
850 dev_crit_ratelimited(hdev->dev,
851 "Staged CS %llu deadlock due to lack of resources",
854 dev_dbg_ratelimited(hdev->dev,
855 "Rejecting CS because of too many in-flights CS\n");
856 atomic64_inc(&ctx->cs_counters.max_cs_in_flight_drop_cnt);
857 atomic64_inc(&cntr->max_cs_in_flight_drop_cnt);
863 hl_fence_init(&cs_cmpl->base_fence, cs_cmpl->cs_seq);
865 cs->sequence = cs_cmpl->cs_seq;
867 ctx->cs_pending[cs_cmpl->cs_seq &
868 (hdev->asic_prop.max_pending_cs - 1)] =
869 &cs_cmpl->base_fence;
872 hl_fence_get(&cs_cmpl->base_fence);
876 spin_unlock(&ctx->cs_lock);
883 spin_unlock(&ctx->cs_lock);
884 kfree(cs->jobs_in_queue_cnt);
893 static void cs_rollback(struct hl_device *hdev, struct hl_cs *cs)
895 struct hl_cs_job *job, *tmp;
897 staged_cs_put(hdev, cs);
899 list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)
900 complete_job(hdev, job);
903 void hl_cs_rollback_all(struct hl_device *hdev)
906 struct hl_cs *cs, *tmp;
908 flush_workqueue(hdev->sob_reset_wq);
910 /* flush all completions before iterating over the CS mirror list in
911 * order to avoid a race with the release functions
913 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
914 flush_workqueue(hdev->cq_wq[i]);
916 /* Make sure we don't have leftovers in the CS mirror list */
917 list_for_each_entry_safe(cs, tmp, &hdev->cs_mirror_list, mirror_node) {
920 dev_warn_ratelimited(hdev->dev, "Killing CS %d.%llu\n",
921 cs->ctx->asid, cs->sequence);
922 cs_rollback(hdev, cs);
926 force_complete_multi_cs(hdev);
930 wake_pending_user_interrupt_threads(struct hl_user_interrupt *interrupt)
932 struct hl_user_pending_interrupt *pend;
935 spin_lock_irqsave(&interrupt->wait_list_lock, flags);
936 list_for_each_entry(pend, &interrupt->wait_list_head, wait_list_node) {
937 pend->fence.error = -EIO;
938 complete_all(&pend->fence.completion);
940 spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
943 void hl_release_pending_user_interrupts(struct hl_device *hdev)
945 struct asic_fixed_properties *prop = &hdev->asic_prop;
946 struct hl_user_interrupt *interrupt;
949 if (!prop->user_interrupt_count)
952 /* We iterate through the user interrupt requests and waking up all
953 * user threads waiting for interrupt completion. We iterate the
954 * list under a lock, this is why all user threads, once awake,
955 * will wait on the same lock and will release the waiting object upon
959 for (i = 0 ; i < prop->user_interrupt_count ; i++) {
960 interrupt = &hdev->user_interrupt[i];
961 wake_pending_user_interrupt_threads(interrupt);
964 interrupt = &hdev->common_user_interrupt;
965 wake_pending_user_interrupt_threads(interrupt);
968 static void job_wq_completion(struct work_struct *work)
970 struct hl_cs_job *job = container_of(work, struct hl_cs_job,
972 struct hl_cs *cs = job->cs;
973 struct hl_device *hdev = cs->ctx->hdev;
975 /* job is no longer needed */
976 complete_job(hdev, job);
979 static int validate_queue_index(struct hl_device *hdev,
980 struct hl_cs_chunk *chunk,
981 enum hl_queue_type *queue_type,
982 bool *is_kernel_allocated_cb)
984 struct asic_fixed_properties *asic = &hdev->asic_prop;
985 struct hw_queue_properties *hw_queue_prop;
987 /* This must be checked here to prevent out-of-bounds access to
988 * hw_queues_props array
990 if (chunk->queue_index >= asic->max_queues) {
991 dev_err(hdev->dev, "Queue index %d is invalid\n",
996 hw_queue_prop = &asic->hw_queues_props[chunk->queue_index];
998 if (hw_queue_prop->type == QUEUE_TYPE_NA) {
999 dev_err(hdev->dev, "Queue index %d is invalid\n",
1000 chunk->queue_index);
1004 if (hw_queue_prop->driver_only) {
1006 "Queue index %d is restricted for the kernel driver\n",
1007 chunk->queue_index);
1011 /* When hw queue type isn't QUEUE_TYPE_HW,
1012 * USER_ALLOC_CB flag shall be referred as "don't care".
1014 if (hw_queue_prop->type == QUEUE_TYPE_HW) {
1015 if (chunk->cs_chunk_flags & HL_CS_CHUNK_FLAGS_USER_ALLOC_CB) {
1016 if (!(hw_queue_prop->cb_alloc_flags & CB_ALLOC_USER)) {
1018 "Queue index %d doesn't support user CB\n",
1019 chunk->queue_index);
1023 *is_kernel_allocated_cb = false;
1025 if (!(hw_queue_prop->cb_alloc_flags &
1028 "Queue index %d doesn't support kernel CB\n",
1029 chunk->queue_index);
1033 *is_kernel_allocated_cb = true;
1036 *is_kernel_allocated_cb = !!(hw_queue_prop->cb_alloc_flags
1040 *queue_type = hw_queue_prop->type;
1044 static struct hl_cb *get_cb_from_cs_chunk(struct hl_device *hdev,
1045 struct hl_cb_mgr *cb_mgr,
1046 struct hl_cs_chunk *chunk)
1051 cb_handle = (u32) (chunk->cb_handle >> PAGE_SHIFT);
1053 cb = hl_cb_get(hdev, cb_mgr, cb_handle);
1055 dev_err(hdev->dev, "CB handle 0x%x invalid\n", cb_handle);
1059 if ((chunk->cb_size < 8) || (chunk->cb_size > cb->size)) {
1060 dev_err(hdev->dev, "CB size %u invalid\n", chunk->cb_size);
1064 atomic_inc(&cb->cs_cnt);
1073 struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev,
1074 enum hl_queue_type queue_type, bool is_kernel_allocated_cb)
1076 struct hl_cs_job *job;
1078 job = kzalloc(sizeof(*job), GFP_ATOMIC);
1080 job = kzalloc(sizeof(*job), GFP_KERNEL);
1085 kref_init(&job->refcount);
1086 job->queue_type = queue_type;
1087 job->is_kernel_allocated_cb = is_kernel_allocated_cb;
1089 if (is_cb_patched(hdev, job))
1090 INIT_LIST_HEAD(&job->userptr_list);
1092 if (job->queue_type == QUEUE_TYPE_EXT)
1093 INIT_WORK(&job->finish_work, job_wq_completion);
1098 static enum hl_cs_type hl_cs_get_cs_type(u32 cs_type_flags)
1100 if (cs_type_flags & HL_CS_FLAGS_SIGNAL)
1101 return CS_TYPE_SIGNAL;
1102 else if (cs_type_flags & HL_CS_FLAGS_WAIT)
1103 return CS_TYPE_WAIT;
1104 else if (cs_type_flags & HL_CS_FLAGS_COLLECTIVE_WAIT)
1105 return CS_TYPE_COLLECTIVE_WAIT;
1106 else if (cs_type_flags & HL_CS_FLAGS_RESERVE_SIGNALS_ONLY)
1107 return CS_RESERVE_SIGNALS;
1108 else if (cs_type_flags & HL_CS_FLAGS_UNRESERVE_SIGNALS_ONLY)
1109 return CS_UNRESERVE_SIGNALS;
1111 return CS_TYPE_DEFAULT;
1114 static int hl_cs_sanity_checks(struct hl_fpriv *hpriv, union hl_cs_args *args)
1116 struct hl_device *hdev = hpriv->hdev;
1117 struct hl_ctx *ctx = hpriv->ctx;
1118 u32 cs_type_flags, num_chunks;
1119 enum hl_device_status status;
1120 enum hl_cs_type cs_type;
1122 if (!hl_device_operational(hdev, &status)) {
1123 dev_warn_ratelimited(hdev->dev,
1124 "Device is %s. Can't submit new CS\n",
1125 hdev->status[status]);
1129 if ((args->in.cs_flags & HL_CS_FLAGS_STAGED_SUBMISSION) &&
1130 !hdev->supports_staged_submission) {
1131 dev_err(hdev->dev, "staged submission not supported");
1135 cs_type_flags = args->in.cs_flags & HL_CS_FLAGS_TYPE_MASK;
1137 if (unlikely(cs_type_flags && !is_power_of_2(cs_type_flags))) {
1139 "CS type flags are mutually exclusive, context %d\n",
1144 cs_type = hl_cs_get_cs_type(cs_type_flags);
1145 num_chunks = args->in.num_chunks_execute;
1147 if (unlikely((cs_type != CS_TYPE_DEFAULT) &&
1148 !hdev->supports_sync_stream)) {
1149 dev_err(hdev->dev, "Sync stream CS is not supported\n");
1153 if (cs_type == CS_TYPE_DEFAULT) {
1156 "Got execute CS with 0 chunks, context %d\n",
1160 } else if (num_chunks != 1) {
1162 "Sync stream CS mandates one chunk only, context %d\n",
1170 static int hl_cs_copy_chunk_array(struct hl_device *hdev,
1171 struct hl_cs_chunk **cs_chunk_array,
1172 void __user *chunks, u32 num_chunks,
1177 if (num_chunks > HL_MAX_JOBS_PER_CS) {
1178 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1179 atomic64_inc(&hdev->aggregated_cs_counters.validation_drop_cnt);
1181 "Number of chunks can NOT be larger than %d\n",
1182 HL_MAX_JOBS_PER_CS);
1186 *cs_chunk_array = kmalloc_array(num_chunks, sizeof(**cs_chunk_array),
1188 if (!*cs_chunk_array)
1189 *cs_chunk_array = kmalloc_array(num_chunks,
1190 sizeof(**cs_chunk_array), GFP_KERNEL);
1191 if (!*cs_chunk_array) {
1192 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
1193 atomic64_inc(&hdev->aggregated_cs_counters.out_of_mem_drop_cnt);
1197 size_to_copy = num_chunks * sizeof(struct hl_cs_chunk);
1198 if (copy_from_user(*cs_chunk_array, chunks, size_to_copy)) {
1199 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1200 atomic64_inc(&hdev->aggregated_cs_counters.validation_drop_cnt);
1201 dev_err(hdev->dev, "Failed to copy cs chunk array from user\n");
1202 kfree(*cs_chunk_array);
1209 static int cs_staged_submission(struct hl_device *hdev, struct hl_cs *cs,
1210 u64 sequence, u32 flags,
1211 u32 encaps_signal_handle)
1213 if (!(flags & HL_CS_FLAGS_STAGED_SUBMISSION))
1216 cs->staged_last = !!(flags & HL_CS_FLAGS_STAGED_SUBMISSION_LAST);
1217 cs->staged_first = !!(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST);
1219 if (cs->staged_first) {
1220 /* Staged CS sequence is the first CS sequence */
1221 INIT_LIST_HEAD(&cs->staged_cs_node);
1222 cs->staged_sequence = cs->sequence;
1224 if (cs->encaps_signals)
1225 cs->encaps_sig_hdl_id = encaps_signal_handle;
1227 /* User sequence will be validated in 'hl_hw_queue_schedule_cs'
1228 * under the cs_mirror_lock
1230 cs->staged_sequence = sequence;
1233 /* Increment CS reference if needed */
1234 staged_cs_get(hdev, cs);
1236 cs->staged_cs = true;
1241 static u32 get_stream_master_qid_mask(struct hl_device *hdev, u32 qid)
1245 for (i = 0; i < hdev->stream_master_qid_arr_size; i++)
1246 if (qid == hdev->stream_master_qid_arr[i])
1252 static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks,
1253 u32 num_chunks, u64 *cs_seq, u32 flags,
1254 u32 encaps_signals_handle, u32 timeout)
1256 bool staged_mid, int_queues_only = true;
1257 struct hl_device *hdev = hpriv->hdev;
1258 struct hl_cs_chunk *cs_chunk_array;
1259 struct hl_cs_counters_atomic *cntr;
1260 struct hl_ctx *ctx = hpriv->ctx;
1261 struct hl_cs_job *job;
1265 u8 stream_master_qid_map = 0;
1268 cntr = &hdev->aggregated_cs_counters;
1269 user_sequence = *cs_seq;
1270 *cs_seq = ULLONG_MAX;
1272 rc = hl_cs_copy_chunk_array(hdev, &cs_chunk_array, chunks, num_chunks,
1277 if ((flags & HL_CS_FLAGS_STAGED_SUBMISSION) &&
1278 !(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST))
1283 rc = allocate_cs(hdev, hpriv->ctx, CS_TYPE_DEFAULT,
1284 staged_mid ? user_sequence : ULLONG_MAX, &cs, flags,
1287 goto free_cs_chunk_array;
1289 *cs_seq = cs->sequence;
1291 hl_debugfs_add_cs(cs);
1293 rc = cs_staged_submission(hdev, cs, user_sequence, flags,
1294 encaps_signals_handle);
1296 goto free_cs_object;
1298 /* If this is a staged submission we must return the staged sequence
1299 * rather than the internal CS sequence
1302 *cs_seq = cs->staged_sequence;
1304 /* Validate ALL the CS chunks before submitting the CS */
1305 for (i = 0 ; i < num_chunks ; i++) {
1306 struct hl_cs_chunk *chunk = &cs_chunk_array[i];
1307 enum hl_queue_type queue_type;
1308 bool is_kernel_allocated_cb;
1310 rc = validate_queue_index(hdev, chunk, &queue_type,
1311 &is_kernel_allocated_cb);
1313 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1314 atomic64_inc(&cntr->validation_drop_cnt);
1315 goto free_cs_object;
1318 if (is_kernel_allocated_cb) {
1319 cb = get_cb_from_cs_chunk(hdev, &hpriv->cb_mgr, chunk);
1322 &ctx->cs_counters.validation_drop_cnt);
1323 atomic64_inc(&cntr->validation_drop_cnt);
1325 goto free_cs_object;
1328 cb = (struct hl_cb *) (uintptr_t) chunk->cb_handle;
1331 if (queue_type == QUEUE_TYPE_EXT ||
1332 queue_type == QUEUE_TYPE_HW) {
1333 int_queues_only = false;
1336 * store which stream are being used for external/HW
1339 if (hdev->supports_wait_for_multi_cs)
1340 stream_master_qid_map |=
1341 get_stream_master_qid_mask(hdev,
1342 chunk->queue_index);
1345 job = hl_cs_allocate_job(hdev, queue_type,
1346 is_kernel_allocated_cb);
1348 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
1349 atomic64_inc(&cntr->out_of_mem_drop_cnt);
1350 dev_err(hdev->dev, "Failed to allocate a new job\n");
1352 if (is_kernel_allocated_cb)
1355 goto free_cs_object;
1361 job->user_cb_size = chunk->cb_size;
1362 job->hw_queue_id = chunk->queue_index;
1364 cs->jobs_in_queue_cnt[job->hw_queue_id]++;
1366 list_add_tail(&job->cs_node, &cs->job_list);
1369 * Increment CS reference. When CS reference is 0, CS is
1370 * done and can be signaled to user and free all its resources
1371 * Only increment for JOB on external or H/W queues, because
1372 * only for those JOBs we get completion
1374 if (cs_needs_completion(cs) &&
1375 (job->queue_type == QUEUE_TYPE_EXT ||
1376 job->queue_type == QUEUE_TYPE_HW))
1379 hl_debugfs_add_job(hdev, job);
1381 rc = cs_parser(hpriv, job);
1383 atomic64_inc(&ctx->cs_counters.parsing_drop_cnt);
1384 atomic64_inc(&cntr->parsing_drop_cnt);
1386 "Failed to parse JOB %d.%llu.%d, err %d, rejecting the CS\n",
1387 cs->ctx->asid, cs->sequence, job->id, rc);
1388 goto free_cs_object;
1392 /* We allow a CS with any queue type combination as long as it does
1393 * not get a completion
1395 if (int_queues_only && cs_needs_completion(cs)) {
1396 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1397 atomic64_inc(&cntr->validation_drop_cnt);
1399 "Reject CS %d.%llu since it contains only internal queues jobs and needs completion\n",
1400 cs->ctx->asid, cs->sequence);
1402 goto free_cs_object;
1406 * store the (external/HW queues) streams used by the CS in the
1407 * fence object for multi-CS completion
1409 if (hdev->supports_wait_for_multi_cs)
1410 cs->fence->stream_master_qid_map = stream_master_qid_map;
1412 rc = hl_hw_queue_schedule_cs(cs);
1416 "Failed to submit CS %d.%llu to H/W queues, error %d\n",
1417 cs->ctx->asid, cs->sequence, rc);
1418 goto free_cs_object;
1421 rc = HL_CS_STATUS_SUCCESS;
1425 atomic_dec(&cb->cs_cnt);
1428 cs_rollback(hdev, cs);
1429 *cs_seq = ULLONG_MAX;
1430 /* The path below is both for good and erroneous exits */
1432 /* We finished with the CS in this function, so put the ref */
1434 free_cs_chunk_array:
1435 kfree(cs_chunk_array);
1440 static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args,
1443 struct hl_device *hdev = hpriv->hdev;
1444 struct hl_ctx *ctx = hpriv->ctx;
1445 bool need_soft_reset = false;
1446 int rc = 0, do_ctx_switch;
1447 void __user *chunks;
1448 u32 num_chunks, tmp;
1451 do_ctx_switch = atomic_cmpxchg(&ctx->thread_ctx_switch_token, 1, 0);
1453 if (do_ctx_switch || (args->in.cs_flags & HL_CS_FLAGS_FORCE_RESTORE)) {
1454 mutex_lock(&hpriv->restore_phase_mutex);
1456 if (do_ctx_switch) {
1457 rc = hdev->asic_funcs->context_switch(hdev, ctx->asid);
1459 dev_err_ratelimited(hdev->dev,
1460 "Failed to switch to context %d, rejecting CS! %d\n",
1463 * If we timedout, or if the device is not IDLE
1464 * while we want to do context-switch (-EBUSY),
1465 * we need to soft-reset because QMAN is
1466 * probably stuck. However, we can't call to
1467 * reset here directly because of deadlock, so
1468 * need to do it at the very end of this
1471 if ((rc == -ETIMEDOUT) || (rc == -EBUSY))
1472 need_soft_reset = true;
1473 mutex_unlock(&hpriv->restore_phase_mutex);
1478 hdev->asic_funcs->restore_phase_topology(hdev);
1480 chunks = (void __user *) (uintptr_t) args->in.chunks_restore;
1481 num_chunks = args->in.num_chunks_restore;
1485 "Need to run restore phase but restore CS is empty\n");
1488 rc = cs_ioctl_default(hpriv, chunks, num_chunks,
1489 cs_seq, 0, 0, hdev->timeout_jiffies);
1492 mutex_unlock(&hpriv->restore_phase_mutex);
1496 "Failed to submit restore CS for context %d (%d)\n",
1501 /* Need to wait for restore completion before execution phase */
1503 enum hl_cs_wait_status status;
1505 ret = _hl_cs_wait_ioctl(hdev, ctx,
1506 jiffies_to_usecs(hdev->timeout_jiffies),
1507 *cs_seq, &status, NULL);
1509 if (ret == -ERESTARTSYS) {
1510 usleep_range(100, 200);
1515 "Restore CS for context %d failed to complete %d\n",
1522 ctx->thread_ctx_switch_wait_token = 1;
1524 } else if (!ctx->thread_ctx_switch_wait_token) {
1525 rc = hl_poll_timeout_memory(hdev,
1526 &ctx->thread_ctx_switch_wait_token, tmp, (tmp == 1),
1527 100, jiffies_to_usecs(hdev->timeout_jiffies), false);
1529 if (rc == -ETIMEDOUT) {
1531 "context switch phase timeout (%d)\n", tmp);
1537 if ((rc == -ETIMEDOUT || rc == -EBUSY) && (need_soft_reset))
1538 hl_device_reset(hdev, 0);
1544 * hl_cs_signal_sob_wraparound_handler: handle SOB value wrapaound case.
1545 * if the SOB value reaches the max value move to the other SOB reserved
1547 * @hdev: pointer to device structure
1548 * @q_idx: stream queue index
1549 * @hw_sob: the H/W SOB used in this signal CS.
1550 * @count: signals count
1551 * @encaps_sig: tells whether it's reservation for encaps signals or not.
1553 * Note that this function must be called while hw_queues_lock is taken.
1555 int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx,
1556 struct hl_hw_sob **hw_sob, u32 count, bool encaps_sig)
1559 struct hl_sync_stream_properties *prop;
1560 struct hl_hw_sob *sob = *hw_sob, *other_sob;
1561 u8 other_sob_offset;
1563 prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
1567 /* check for wraparound */
1568 if (prop->next_sob_val + count >= HL_MAX_SOB_VAL) {
1570 * Decrement as we reached the max value.
1571 * The release function won't be called here as we've
1572 * just incremented the refcount right before calling this
1575 hw_sob_put_err(sob);
1578 * check the other sob value, if it still in use then fail
1579 * otherwise make the switch
1581 other_sob_offset = (prop->curr_sob_offset + 1) % HL_RSVD_SOBS;
1582 other_sob = &prop->hw_sob[other_sob_offset];
1584 if (kref_read(&other_sob->kref) != 1) {
1585 dev_err(hdev->dev, "error: Cannot switch SOBs q_idx: %d\n",
1591 * next_sob_val always points to the next available signal
1592 * in the sob, so in encaps signals it will be the next one
1593 * after reserving the required amount.
1596 prop->next_sob_val = count + 1;
1598 prop->next_sob_val = count;
1600 /* only two SOBs are currently in use */
1601 prop->curr_sob_offset = other_sob_offset;
1602 *hw_sob = other_sob;
1605 * check if other_sob needs reset, then do it before using it
1606 * for the reservation or the next signal cs.
1607 * we do it here, and for both encaps and regular signal cs
1608 * cases in order to avoid possible races of two kref_put
1609 * of the sob which can occur at the same time if we move the
1610 * sob reset(kref_put) to cs_do_release function.
1611 * in addition, if we have combination of cs signal and
1612 * encaps, and at the point we need to reset the sob there was
1613 * no more reservations and only signal cs keep coming,
1614 * in such case we need signal_cs to put the refcount and
1617 if (other_sob->need_reset)
1618 hw_sob_put(other_sob);
1621 /* set reset indication for the sob */
1622 sob->need_reset = true;
1623 hw_sob_get(other_sob);
1626 dev_dbg(hdev->dev, "switched to SOB %d, q_idx: %d\n",
1627 prop->curr_sob_offset, q_idx);
1629 prop->next_sob_val += count;
1635 static int cs_ioctl_extract_signal_seq(struct hl_device *hdev,
1636 struct hl_cs_chunk *chunk, u64 *signal_seq, struct hl_ctx *ctx,
1637 bool encaps_signals)
1639 u64 *signal_seq_arr = NULL;
1640 u32 size_to_copy, signal_seq_arr_len;
1643 if (encaps_signals) {
1644 *signal_seq = chunk->encaps_signal_seq;
1648 signal_seq_arr_len = chunk->num_signal_seq_arr;
1650 /* currently only one signal seq is supported */
1651 if (signal_seq_arr_len != 1) {
1652 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1653 atomic64_inc(&hdev->aggregated_cs_counters.validation_drop_cnt);
1655 "Wait for signal CS supports only one signal CS seq\n");
1659 signal_seq_arr = kmalloc_array(signal_seq_arr_len,
1660 sizeof(*signal_seq_arr),
1662 if (!signal_seq_arr)
1663 signal_seq_arr = kmalloc_array(signal_seq_arr_len,
1664 sizeof(*signal_seq_arr),
1666 if (!signal_seq_arr) {
1667 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
1668 atomic64_inc(&hdev->aggregated_cs_counters.out_of_mem_drop_cnt);
1672 size_to_copy = signal_seq_arr_len * sizeof(*signal_seq_arr);
1673 if (copy_from_user(signal_seq_arr,
1674 u64_to_user_ptr(chunk->signal_seq_arr),
1676 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1677 atomic64_inc(&hdev->aggregated_cs_counters.validation_drop_cnt);
1679 "Failed to copy signal seq array from user\n");
1684 /* currently it is guaranteed to have only one signal seq */
1685 *signal_seq = signal_seq_arr[0];
1688 kfree(signal_seq_arr);
1693 static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev,
1694 struct hl_ctx *ctx, struct hl_cs *cs,
1695 enum hl_queue_type q_type, u32 q_idx, u32 encaps_signal_offset)
1697 struct hl_cs_counters_atomic *cntr;
1698 struct hl_cs_job *job;
1702 cntr = &hdev->aggregated_cs_counters;
1704 job = hl_cs_allocate_job(hdev, q_type, true);
1706 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
1707 atomic64_inc(&cntr->out_of_mem_drop_cnt);
1708 dev_err(hdev->dev, "Failed to allocate a new job\n");
1712 if (cs->type == CS_TYPE_WAIT)
1713 cb_size = hdev->asic_funcs->get_wait_cb_size(hdev);
1715 cb_size = hdev->asic_funcs->get_signal_cb_size(hdev);
1717 cb = hl_cb_kernel_create(hdev, cb_size,
1718 q_type == QUEUE_TYPE_HW && hdev->mmu_enable);
1720 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
1721 atomic64_inc(&cntr->out_of_mem_drop_cnt);
1729 atomic_inc(&job->user_cb->cs_cnt);
1730 job->user_cb_size = cb_size;
1731 job->hw_queue_id = q_idx;
1733 if ((cs->type == CS_TYPE_WAIT || cs->type == CS_TYPE_COLLECTIVE_WAIT)
1734 && cs->encaps_signals)
1735 job->encaps_sig_wait_offset = encaps_signal_offset;
1737 * No need in parsing, user CB is the patched CB.
1738 * We call hl_cb_destroy() out of two reasons - we don't need the CB in
1739 * the CB idr anymore and to decrement its refcount as it was
1740 * incremented inside hl_cb_kernel_create().
1742 job->patched_cb = job->user_cb;
1743 job->job_cb_size = job->user_cb_size;
1744 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
1746 /* increment refcount as for external queues we get completion */
1749 cs->jobs_in_queue_cnt[job->hw_queue_id]++;
1751 list_add_tail(&job->cs_node, &cs->job_list);
1753 hl_debugfs_add_job(hdev, job);
1758 static int cs_ioctl_reserve_signals(struct hl_fpriv *hpriv,
1759 u32 q_idx, u32 count,
1760 u32 *handle_id, u32 *sob_addr,
1763 struct hw_queue_properties *hw_queue_prop;
1764 struct hl_sync_stream_properties *prop;
1765 struct hl_device *hdev = hpriv->hdev;
1766 struct hl_cs_encaps_sig_handle *handle;
1767 struct hl_encaps_signals_mgr *mgr;
1768 struct hl_hw_sob *hw_sob;
1772 if (count >= HL_MAX_SOB_VAL) {
1773 dev_err(hdev->dev, "signals count(%u) exceeds the max SOB value\n",
1779 if (q_idx >= hdev->asic_prop.max_queues) {
1780 dev_err(hdev->dev, "Queue index %d is invalid\n",
1786 hw_queue_prop = &hdev->asic_prop.hw_queues_props[q_idx];
1788 if (!hw_queue_prop->supports_sync_stream) {
1790 "Queue index %d does not support sync stream operations\n",
1796 prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
1798 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
1804 handle->count = count;
1805 mgr = &hpriv->ctx->sig_mgr;
1807 spin_lock(&mgr->lock);
1808 hdl_id = idr_alloc(&mgr->handles, handle, 1, 0, GFP_ATOMIC);
1809 spin_unlock(&mgr->lock);
1812 dev_err(hdev->dev, "Failed to allocate IDR for a new signal reservation\n");
1817 handle->id = hdl_id;
1818 handle->q_idx = q_idx;
1819 handle->hdev = hdev;
1820 kref_init(&handle->refcount);
1822 hdev->asic_funcs->hw_queues_lock(hdev);
1824 hw_sob = &prop->hw_sob[prop->curr_sob_offset];
1827 * Increment the SOB value by count by user request
1828 * to reserve those signals
1829 * check if the signals amount to reserve is not exceeding the max sob
1830 * value, if yes then switch sob.
1832 rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, count,
1835 dev_err(hdev->dev, "Failed to switch SOB\n");
1836 hdev->asic_funcs->hw_queues_unlock(hdev);
1840 /* set the hw_sob to the handle after calling the sob wraparound handler
1841 * since sob could have changed.
1843 handle->hw_sob = hw_sob;
1845 /* store the current sob value for unreserve validity check, and
1846 * signal offset support
1848 handle->pre_sob_val = prop->next_sob_val - handle->count;
1850 *signals_count = prop->next_sob_val;
1851 hdev->asic_funcs->hw_queues_unlock(hdev);
1853 *sob_addr = handle->hw_sob->sob_addr;
1854 *handle_id = hdl_id;
1857 "Signals reserved, sob_id: %d, sob addr: 0x%x, last sob_val: %u, q_idx: %d, hdl_id: %d\n",
1858 hw_sob->sob_id, handle->hw_sob->sob_addr,
1859 prop->next_sob_val - 1, q_idx, hdl_id);
1863 spin_lock(&mgr->lock);
1864 idr_remove(&mgr->handles, hdl_id);
1865 spin_unlock(&mgr->lock);
1872 static int cs_ioctl_unreserve_signals(struct hl_fpriv *hpriv, u32 handle_id)
1874 struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
1875 struct hl_sync_stream_properties *prop;
1876 struct hl_device *hdev = hpriv->hdev;
1877 struct hl_encaps_signals_mgr *mgr;
1878 struct hl_hw_sob *hw_sob;
1879 u32 q_idx, sob_addr;
1882 mgr = &hpriv->ctx->sig_mgr;
1884 spin_lock(&mgr->lock);
1885 encaps_sig_hdl = idr_find(&mgr->handles, handle_id);
1886 if (encaps_sig_hdl) {
1887 dev_dbg(hdev->dev, "unreserve signals, handle: %u, SOB:0x%x, count: %u\n",
1888 handle_id, encaps_sig_hdl->hw_sob->sob_addr,
1889 encaps_sig_hdl->count);
1891 hdev->asic_funcs->hw_queues_lock(hdev);
1893 q_idx = encaps_sig_hdl->q_idx;
1894 prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
1895 hw_sob = &prop->hw_sob[prop->curr_sob_offset];
1896 sob_addr = hdev->asic_funcs->get_sob_addr(hdev, hw_sob->sob_id);
1898 /* Check if sob_val got out of sync due to other
1899 * signal submission requests which were handled
1900 * between the reserve-unreserve calls or SOB switch
1901 * upon reaching SOB max value.
1903 if (encaps_sig_hdl->pre_sob_val + encaps_sig_hdl->count
1904 != prop->next_sob_val ||
1905 sob_addr != encaps_sig_hdl->hw_sob->sob_addr) {
1906 dev_err(hdev->dev, "Cannot unreserve signals, SOB val ran out of sync, expected: %u, actual val: %u\n",
1907 encaps_sig_hdl->pre_sob_val,
1908 (prop->next_sob_val - encaps_sig_hdl->count));
1910 hdev->asic_funcs->hw_queues_unlock(hdev);
1916 * Decrement the SOB value by count by user request
1917 * to unreserve those signals
1919 prop->next_sob_val -= encaps_sig_hdl->count;
1921 hdev->asic_funcs->hw_queues_unlock(hdev);
1925 /* Release the id and free allocated memory of the handle */
1926 idr_remove(&mgr->handles, handle_id);
1927 kfree(encaps_sig_hdl);
1930 dev_err(hdev->dev, "failed to unreserve signals, cannot find handler\n");
1933 spin_unlock(&mgr->lock);
1938 static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
1939 void __user *chunks, u32 num_chunks,
1940 u64 *cs_seq, u32 flags, u32 timeout)
1942 struct hl_cs_encaps_sig_handle *encaps_sig_hdl = NULL;
1943 bool handle_found = false, is_wait_cs = false,
1944 wait_cs_submitted = false,
1945 cs_encaps_signals = false;
1946 struct hl_cs_chunk *cs_chunk_array, *chunk;
1947 bool staged_cs_with_encaps_signals = false;
1948 struct hw_queue_properties *hw_queue_prop;
1949 struct hl_device *hdev = hpriv->hdev;
1950 struct hl_cs_compl *sig_waitcs_cmpl;
1951 u32 q_idx, collective_engine_id = 0;
1952 struct hl_cs_counters_atomic *cntr;
1953 struct hl_fence *sig_fence = NULL;
1954 struct hl_ctx *ctx = hpriv->ctx;
1955 enum hl_queue_type q_type;
1960 cntr = &hdev->aggregated_cs_counters;
1961 *cs_seq = ULLONG_MAX;
1963 rc = hl_cs_copy_chunk_array(hdev, &cs_chunk_array, chunks, num_chunks,
1968 /* currently it is guaranteed to have only one chunk */
1969 chunk = &cs_chunk_array[0];
1971 if (chunk->queue_index >= hdev->asic_prop.max_queues) {
1972 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1973 atomic64_inc(&cntr->validation_drop_cnt);
1974 dev_err(hdev->dev, "Queue index %d is invalid\n",
1975 chunk->queue_index);
1977 goto free_cs_chunk_array;
1980 q_idx = chunk->queue_index;
1981 hw_queue_prop = &hdev->asic_prop.hw_queues_props[q_idx];
1982 q_type = hw_queue_prop->type;
1984 if (!hw_queue_prop->supports_sync_stream) {
1985 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1986 atomic64_inc(&cntr->validation_drop_cnt);
1988 "Queue index %d does not support sync stream operations\n",
1991 goto free_cs_chunk_array;
1994 if (cs_type == CS_TYPE_COLLECTIVE_WAIT) {
1995 if (!(hw_queue_prop->collective_mode == HL_COLLECTIVE_MASTER)) {
1996 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1997 atomic64_inc(&cntr->validation_drop_cnt);
1999 "Queue index %d is invalid\n", q_idx);
2001 goto free_cs_chunk_array;
2004 if (!hdev->nic_ports_mask) {
2005 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
2006 atomic64_inc(&cntr->validation_drop_cnt);
2008 "Collective operations not supported when NIC ports are disabled");
2010 goto free_cs_chunk_array;
2013 collective_engine_id = chunk->collective_engine_id;
2016 is_wait_cs = !!(cs_type == CS_TYPE_WAIT ||
2017 cs_type == CS_TYPE_COLLECTIVE_WAIT);
2019 cs_encaps_signals = !!(flags & HL_CS_FLAGS_ENCAP_SIGNALS);
2022 rc = cs_ioctl_extract_signal_seq(hdev, chunk, &signal_seq,
2023 ctx, cs_encaps_signals);
2025 goto free_cs_chunk_array;
2027 if (cs_encaps_signals) {
2028 /* check if cs sequence has encapsulated
2034 spin_lock(&ctx->sig_mgr.lock);
2035 idp = &ctx->sig_mgr.handles;
2036 idr_for_each_entry(idp, encaps_sig_hdl, id) {
2037 if (encaps_sig_hdl->cs_seq == signal_seq) {
2038 handle_found = true;
2039 /* get refcount to protect removing
2040 * this handle from idr, needed when
2041 * multiple wait cs are used with offset
2042 * to wait on reserved encaps signals.
2044 kref_get(&encaps_sig_hdl->refcount);
2048 spin_unlock(&ctx->sig_mgr.lock);
2050 if (!handle_found) {
2051 /* treat as signal CS already finished */
2052 dev_dbg(hdev->dev, "Cannot find encapsulated signals handle for seq 0x%llx\n",
2055 goto free_cs_chunk_array;
2058 /* validate also the signal offset value */
2059 if (chunk->encaps_signal_offset >
2060 encaps_sig_hdl->count) {
2061 dev_err(hdev->dev, "offset(%u) value exceed max reserved signals count(%u)!\n",
2062 chunk->encaps_signal_offset,
2063 encaps_sig_hdl->count);
2065 goto free_cs_chunk_array;
2069 sig_fence = hl_ctx_get_fence(ctx, signal_seq);
2070 if (IS_ERR(sig_fence)) {
2071 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
2072 atomic64_inc(&cntr->validation_drop_cnt);
2074 "Failed to get signal CS with seq 0x%llx\n",
2076 rc = PTR_ERR(sig_fence);
2077 goto free_cs_chunk_array;
2081 /* signal CS already finished */
2083 goto free_cs_chunk_array;
2087 container_of(sig_fence, struct hl_cs_compl, base_fence);
2089 staged_cs_with_encaps_signals = !!
2090 (sig_waitcs_cmpl->type == CS_TYPE_DEFAULT &&
2091 (flags & HL_CS_FLAGS_ENCAP_SIGNALS));
2093 if (sig_waitcs_cmpl->type != CS_TYPE_SIGNAL &&
2094 !staged_cs_with_encaps_signals) {
2095 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
2096 atomic64_inc(&cntr->validation_drop_cnt);
2098 "CS seq 0x%llx is not of a signal/encaps-signal CS\n",
2100 hl_fence_put(sig_fence);
2102 goto free_cs_chunk_array;
2105 if (completion_done(&sig_fence->completion)) {
2106 /* signal CS already finished */
2107 hl_fence_put(sig_fence);
2109 goto free_cs_chunk_array;
2113 rc = allocate_cs(hdev, ctx, cs_type, ULLONG_MAX, &cs, flags, timeout);
2116 hl_fence_put(sig_fence);
2118 goto free_cs_chunk_array;
2122 * Save the signal CS fence for later initialization right before
2123 * hanging the wait CS on the queue.
2124 * for encaps signals case, we save the cs sequence and handle pointer
2125 * for later initialization.
2128 cs->signal_fence = sig_fence;
2129 /* store the handle pointer, so we don't have to
2130 * look for it again, later on the flow
2131 * when we need to set SOB info in hw_queue.
2133 if (cs->encaps_signals)
2134 cs->encaps_sig_hdl = encaps_sig_hdl;
2137 hl_debugfs_add_cs(cs);
2139 *cs_seq = cs->sequence;
2141 if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_SIGNAL)
2142 rc = cs_ioctl_signal_wait_create_jobs(hdev, ctx, cs, q_type,
2143 q_idx, chunk->encaps_signal_offset);
2144 else if (cs_type == CS_TYPE_COLLECTIVE_WAIT)
2145 rc = hdev->asic_funcs->collective_wait_create_jobs(hdev, ctx,
2146 cs, q_idx, collective_engine_id,
2147 chunk->encaps_signal_offset);
2149 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
2150 atomic64_inc(&cntr->validation_drop_cnt);
2155 goto free_cs_object;
2157 rc = hl_hw_queue_schedule_cs(cs);
2159 /* In case wait cs failed here, it means the signal cs
2160 * already completed. we want to free all it's related objects
2161 * but we don't want to fail the ioctl.
2165 else if (rc != -EAGAIN)
2167 "Failed to submit CS %d.%llu to H/W queues, error %d\n",
2168 ctx->asid, cs->sequence, rc);
2169 goto free_cs_object;
2172 rc = HL_CS_STATUS_SUCCESS;
2174 wait_cs_submitted = true;
2178 cs_rollback(hdev, cs);
2179 *cs_seq = ULLONG_MAX;
2180 /* The path below is both for good and erroneous exits */
2182 /* We finished with the CS in this function, so put the ref */
2184 free_cs_chunk_array:
2185 if (!wait_cs_submitted && cs_encaps_signals && handle_found &&
2187 kref_put(&encaps_sig_hdl->refcount,
2188 hl_encaps_handle_do_release);
2189 kfree(cs_chunk_array);
2194 int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
2196 union hl_cs_args *args = data;
2197 enum hl_cs_type cs_type = 0;
2198 u64 cs_seq = ULONG_MAX;
2199 void __user *chunks;
2200 u32 num_chunks, flags, timeout,
2201 signals_count = 0, sob_addr = 0, handle_id = 0;
2204 rc = hl_cs_sanity_checks(hpriv, args);
2208 rc = hl_cs_ctx_switch(hpriv, args, &cs_seq);
2212 cs_type = hl_cs_get_cs_type(args->in.cs_flags &
2213 ~HL_CS_FLAGS_FORCE_RESTORE);
2214 chunks = (void __user *) (uintptr_t) args->in.chunks_execute;
2215 num_chunks = args->in.num_chunks_execute;
2216 flags = args->in.cs_flags;
2218 /* In case this is a staged CS, user should supply the CS sequence */
2219 if ((flags & HL_CS_FLAGS_STAGED_SUBMISSION) &&
2220 !(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST))
2221 cs_seq = args->in.seq;
2223 timeout = flags & HL_CS_FLAGS_CUSTOM_TIMEOUT
2224 ? msecs_to_jiffies(args->in.timeout * 1000)
2225 : hpriv->hdev->timeout_jiffies;
2228 case CS_TYPE_SIGNAL:
2230 case CS_TYPE_COLLECTIVE_WAIT:
2231 rc = cs_ioctl_signal_wait(hpriv, cs_type, chunks, num_chunks,
2232 &cs_seq, args->in.cs_flags, timeout);
2234 case CS_RESERVE_SIGNALS:
2235 rc = cs_ioctl_reserve_signals(hpriv,
2236 args->in.encaps_signals_q_idx,
2237 args->in.encaps_signals_count,
2238 &handle_id, &sob_addr, &signals_count);
2240 case CS_UNRESERVE_SIGNALS:
2241 rc = cs_ioctl_unreserve_signals(hpriv,
2242 args->in.encaps_sig_handle_id);
2245 rc = cs_ioctl_default(hpriv, chunks, num_chunks, &cs_seq,
2247 args->in.encaps_sig_handle_id,
2252 if (rc != -EAGAIN) {
2253 memset(args, 0, sizeof(*args));
2255 if (cs_type == CS_RESERVE_SIGNALS) {
2256 args->out.handle_id = handle_id;
2257 args->out.sob_base_addr_offset = sob_addr;
2258 args->out.count = signals_count;
2260 args->out.seq = cs_seq;
2262 args->out.status = rc;
2268 static int hl_wait_for_fence(struct hl_ctx *ctx, u64 seq, struct hl_fence *fence,
2269 enum hl_cs_wait_status *status, u64 timeout_us,
2272 struct hl_device *hdev = ctx->hdev;
2276 if (IS_ERR(fence)) {
2277 rc = PTR_ERR(fence);
2279 dev_notice_ratelimited(hdev->dev,
2280 "Can't wait on CS %llu because current CS is at seq %llu\n",
2281 seq, ctx->cs_sequence);
2287 "Can't wait on seq %llu because current CS is at seq %llu (Fence is gone)\n",
2288 seq, ctx->cs_sequence);
2290 *status = CS_WAIT_STATUS_GONE;
2295 completion_rc = completion_done(&fence->completion);
2297 unsigned long timeout;
2299 timeout = (timeout_us == MAX_SCHEDULE_TIMEOUT) ?
2300 timeout_us : usecs_to_jiffies(timeout_us);
2302 wait_for_completion_interruptible_timeout(
2303 &fence->completion, timeout);
2306 if (completion_rc > 0) {
2307 *status = CS_WAIT_STATUS_COMPLETED;
2309 *timestamp = ktime_to_ns(fence->timestamp);
2311 *status = CS_WAIT_STATUS_BUSY;
2314 if (fence->error == -ETIMEDOUT)
2316 else if (fence->error == -EIO)
2323 * hl_cs_poll_fences - iterate CS fences to check for CS completion
2325 * @mcs_data: multi-CS internal data
2327 * @return 0 on success, otherwise non 0 error code
2329 * The function iterates on all CS sequence in the list and set bit in
2330 * completion_bitmap for each completed CS.
2331 * while iterating, the function can extracts the stream map to be later
2332 * used by the waiting function.
2333 * this function shall be called after taking context ref
2335 static int hl_cs_poll_fences(struct multi_cs_data *mcs_data)
2337 struct hl_fence **fence_ptr = mcs_data->fence_arr;
2338 struct hl_device *hdev = mcs_data->ctx->hdev;
2339 int i, rc, arr_len = mcs_data->arr_len;
2340 u64 *seq_arr = mcs_data->seq_arr;
2341 ktime_t max_ktime, first_cs_time;
2342 enum hl_cs_wait_status status;
2344 memset(fence_ptr, 0, arr_len * sizeof(*fence_ptr));
2346 /* get all fences under the same lock */
2347 rc = hl_ctx_get_fences(mcs_data->ctx, seq_arr, fence_ptr, arr_len);
2352 * set to maximum time to verify timestamp is valid: if at the end
2353 * this value is maintained- no timestamp was updated
2355 max_ktime = ktime_set(KTIME_SEC_MAX, 0);
2356 first_cs_time = max_ktime;
2358 for (i = 0; i < arr_len; i++, fence_ptr++) {
2359 struct hl_fence *fence = *fence_ptr;
2362 * function won't sleep as it is called with timeout 0 (i.e.
2365 rc = hl_wait_for_fence(mcs_data->ctx, seq_arr[i], fence,
2369 "wait_for_fence error :%d for CS seq %llu\n",
2374 mcs_data->stream_master_qid_map |= fence->stream_master_qid_map;
2376 if (status == CS_WAIT_STATUS_BUSY)
2379 mcs_data->completion_bitmap |= BIT(i);
2382 * best effort to extract timestamp. few notes:
2383 * - if even single fence is gone we cannot extract timestamp
2384 * (as fence not exist anymore)
2385 * - for all completed CSs we take the earliest timestamp.
2386 * for this we have to validate that:
2387 * 1. given timestamp was indeed set
2388 * 2. the timestamp is earliest of all timestamps so far
2391 if (status == CS_WAIT_STATUS_GONE) {
2392 mcs_data->update_ts = false;
2393 mcs_data->gone_cs = true;
2394 } else if (mcs_data->update_ts &&
2395 (ktime_compare(fence->timestamp,
2396 ktime_set(0, 0)) > 0) &&
2397 (ktime_compare(fence->timestamp, first_cs_time) < 0)) {
2398 first_cs_time = fence->timestamp;
2402 hl_fences_put(mcs_data->fence_arr, arr_len);
2404 if (mcs_data->update_ts &&
2405 (ktime_compare(first_cs_time, max_ktime) != 0))
2406 mcs_data->timestamp = ktime_to_ns(first_cs_time);
2411 static int _hl_cs_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
2412 u64 timeout_us, u64 seq,
2413 enum hl_cs_wait_status *status, s64 *timestamp)
2415 struct hl_fence *fence;
2421 hl_ctx_get(hdev, ctx);
2423 fence = hl_ctx_get_fence(ctx, seq);
2425 rc = hl_wait_for_fence(ctx, seq, fence, status, timeout_us, timestamp);
2426 hl_fence_put(fence);
2433 * hl_wait_multi_cs_completion_init - init completion structure
2435 * @hdev: pointer to habanalabs device structure
2436 * @stream_master_bitmap: stream master QIDs map, set bit indicates stream
2437 * master QID to wait on
2439 * @return valid completion struct pointer on success, otherwise error pointer
2441 * up to MULTI_CS_MAX_USER_CTX calls can be done concurrently to the driver.
2442 * the function gets the first available completion (by marking it "used")
2443 * and initialize its values.
2445 static struct multi_cs_completion *hl_wait_multi_cs_completion_init(
2446 struct hl_device *hdev,
2447 u8 stream_master_bitmap)
2449 struct multi_cs_completion *mcs_compl;
2452 /* find free multi_cs completion structure */
2453 for (i = 0; i < MULTI_CS_MAX_USER_CTX; i++) {
2454 mcs_compl = &hdev->multi_cs_completion[i];
2455 spin_lock(&mcs_compl->lock);
2456 if (!mcs_compl->used) {
2457 mcs_compl->used = 1;
2458 mcs_compl->timestamp = 0;
2459 mcs_compl->stream_master_qid_map = stream_master_bitmap;
2460 reinit_completion(&mcs_compl->completion);
2461 spin_unlock(&mcs_compl->lock);
2464 spin_unlock(&mcs_compl->lock);
2467 if (i == MULTI_CS_MAX_USER_CTX) {
2469 "no available multi-CS completion structure\n");
2470 return ERR_PTR(-ENOMEM);
2476 * hl_wait_multi_cs_completion_fini - return completion structure and set as
2479 * @mcs_compl: pointer to the completion structure
2481 static void hl_wait_multi_cs_completion_fini(
2482 struct multi_cs_completion *mcs_compl)
2485 * free completion structure, do it under lock to be in-sync with the
2486 * thread that signals completion
2488 spin_lock(&mcs_compl->lock);
2489 mcs_compl->used = 0;
2490 spin_unlock(&mcs_compl->lock);
2494 * hl_wait_multi_cs_completion - wait for first CS to complete
2496 * @mcs_data: multi-CS internal data
2498 * @return 0 on success, otherwise non 0 error code
2500 static int hl_wait_multi_cs_completion(struct multi_cs_data *mcs_data)
2502 struct hl_device *hdev = mcs_data->ctx->hdev;
2503 struct multi_cs_completion *mcs_compl;
2506 mcs_compl = hl_wait_multi_cs_completion_init(hdev,
2507 mcs_data->stream_master_qid_map);
2508 if (IS_ERR(mcs_compl))
2509 return PTR_ERR(mcs_compl);
2511 completion_rc = wait_for_completion_interruptible_timeout(
2512 &mcs_compl->completion,
2513 usecs_to_jiffies(mcs_data->timeout_us));
2515 /* update timestamp */
2516 if (completion_rc > 0)
2517 mcs_data->timestamp = mcs_compl->timestamp;
2519 hl_wait_multi_cs_completion_fini(mcs_compl);
2521 mcs_data->wait_status = completion_rc;
2527 * hl_multi_cs_completion_init - init array of multi-CS completion structures
2529 * @hdev: pointer to habanalabs device structure
2531 void hl_multi_cs_completion_init(struct hl_device *hdev)
2533 struct multi_cs_completion *mcs_cmpl;
2536 for (i = 0; i < MULTI_CS_MAX_USER_CTX; i++) {
2537 mcs_cmpl = &hdev->multi_cs_completion[i];
2539 spin_lock_init(&mcs_cmpl->lock);
2540 init_completion(&mcs_cmpl->completion);
2545 * hl_multi_cs_wait_ioctl - implementation of the multi-CS wait ioctl
2547 * @hpriv: pointer to the private data of the fd
2548 * @data: pointer to multi-CS wait ioctl in/out args
2551 static int hl_multi_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data)
2553 struct hl_device *hdev = hpriv->hdev;
2554 struct multi_cs_data mcs_data = {0};
2555 union hl_wait_cs_args *args = data;
2556 struct hl_ctx *ctx = hpriv->ctx;
2557 struct hl_fence **fence_arr;
2558 void __user *seq_arr;
2564 if (!hdev->supports_wait_for_multi_cs) {
2565 dev_err(hdev->dev, "Wait for multi CS is not supported\n");
2569 seq_arr_len = args->in.seq_arr_len;
2571 if (seq_arr_len > HL_WAIT_MULTI_CS_LIST_MAX_LEN) {
2572 dev_err(hdev->dev, "Can wait only up to %d CSs, input sequence is of length %u\n",
2573 HL_WAIT_MULTI_CS_LIST_MAX_LEN, seq_arr_len);
2577 /* allocate memory for sequence array */
2579 kmalloc_array(seq_arr_len, sizeof(*cs_seq_arr), GFP_KERNEL);
2583 /* copy CS sequence array from user */
2584 seq_arr = (void __user *) (uintptr_t) args->in.seq;
2585 size_to_copy = seq_arr_len * sizeof(*cs_seq_arr);
2586 if (copy_from_user(cs_seq_arr, seq_arr, size_to_copy)) {
2587 dev_err(hdev->dev, "Failed to copy multi-cs sequence array from user\n");
2592 /* allocate array for the fences */
2593 fence_arr = kmalloc_array(seq_arr_len, sizeof(*fence_arr), GFP_KERNEL);
2599 /* initialize the multi-CS internal data */
2601 mcs_data.seq_arr = cs_seq_arr;
2602 mcs_data.fence_arr = fence_arr;
2603 mcs_data.arr_len = seq_arr_len;
2605 hl_ctx_get(hdev, ctx);
2607 /* poll all CS fences, extract timestamp */
2608 mcs_data.update_ts = true;
2609 rc = hl_cs_poll_fences(&mcs_data);
2611 * skip wait for CS completion when one of the below is true:
2612 * - an error on the poll function
2613 * - one or more CS in the list completed
2614 * - the user called ioctl with timeout 0
2616 if (rc || mcs_data.completion_bitmap || !args->in.timeout_us)
2619 /* wait (with timeout) for the first CS to be completed */
2620 mcs_data.timeout_us = args->in.timeout_us;
2621 rc = hl_wait_multi_cs_completion(&mcs_data);
2625 if (mcs_data.wait_status > 0) {
2627 * poll fences once again to update the CS map.
2628 * no timestamp should be updated this time.
2630 mcs_data.update_ts = false;
2631 rc = hl_cs_poll_fences(&mcs_data);
2634 * if hl_wait_multi_cs_completion returned before timeout (i.e.
2635 * it got a completion) we expect to see at least one CS
2636 * completed after the poll function.
2638 if (!mcs_data.completion_bitmap) {
2639 dev_warn_ratelimited(hdev->dev,
2640 "Multi-CS got completion on wait but no CS completed\n");
2652 /* update output args */
2653 memset(args, 0, sizeof(*args));
2657 if (mcs_data.completion_bitmap) {
2658 args->out.status = HL_WAIT_CS_STATUS_COMPLETED;
2659 args->out.cs_completion_map = mcs_data.completion_bitmap;
2661 /* if timestamp not 0- it's valid */
2662 if (mcs_data.timestamp) {
2663 args->out.timestamp_nsec = mcs_data.timestamp;
2664 args->out.flags |= HL_WAIT_CS_STATUS_FLAG_TIMESTAMP_VLD;
2667 /* update if some CS was gone */
2668 if (mcs_data.timestamp)
2669 args->out.flags |= HL_WAIT_CS_STATUS_FLAG_GONE;
2670 } else if (mcs_data.wait_status == -ERESTARTSYS) {
2671 args->out.status = HL_WAIT_CS_STATUS_INTERRUPTED;
2673 args->out.status = HL_WAIT_CS_STATUS_BUSY;
2679 static int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data)
2681 struct hl_device *hdev = hpriv->hdev;
2682 union hl_wait_cs_args *args = data;
2683 enum hl_cs_wait_status status;
2684 u64 seq = args->in.seq;
2688 rc = _hl_cs_wait_ioctl(hdev, hpriv->ctx, args->in.timeout_us, seq,
2689 &status, ×tamp);
2691 memset(args, 0, sizeof(*args));
2694 if (rc == -ERESTARTSYS) {
2695 dev_err_ratelimited(hdev->dev,
2696 "user process got signal while waiting for CS handle %llu\n",
2698 args->out.status = HL_WAIT_CS_STATUS_INTERRUPTED;
2700 } else if (rc == -ETIMEDOUT) {
2701 dev_err_ratelimited(hdev->dev,
2702 "CS %llu has timed-out while user process is waiting for it\n",
2704 args->out.status = HL_WAIT_CS_STATUS_TIMEDOUT;
2705 } else if (rc == -EIO) {
2706 dev_err_ratelimited(hdev->dev,
2707 "CS %llu has been aborted while user process is waiting for it\n",
2709 args->out.status = HL_WAIT_CS_STATUS_ABORTED;
2715 args->out.flags |= HL_WAIT_CS_STATUS_FLAG_TIMESTAMP_VLD;
2716 args->out.timestamp_nsec = timestamp;
2720 case CS_WAIT_STATUS_GONE:
2721 args->out.flags |= HL_WAIT_CS_STATUS_FLAG_GONE;
2723 case CS_WAIT_STATUS_COMPLETED:
2724 args->out.status = HL_WAIT_CS_STATUS_COMPLETED;
2726 case CS_WAIT_STATUS_BUSY:
2728 args->out.status = HL_WAIT_CS_STATUS_BUSY;
2735 static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
2736 u32 timeout_us, u64 user_address,
2737 u32 target_value, u16 interrupt_offset,
2738 enum hl_cs_wait_status *status)
2740 struct hl_user_pending_interrupt *pend;
2741 struct hl_user_interrupt *interrupt;
2742 unsigned long timeout, flags;
2743 u32 completion_value;
2747 if (timeout_us == U32_MAX)
2748 timeout = timeout_us;
2750 timeout = usecs_to_jiffies(timeout_us);
2752 hl_ctx_get(hdev, ctx);
2754 pend = kmalloc(sizeof(*pend), GFP_KERNEL);
2760 hl_fence_init(&pend->fence, ULONG_MAX);
2762 if (interrupt_offset == HL_COMMON_USER_INTERRUPT_ID)
2763 interrupt = &hdev->common_user_interrupt;
2765 interrupt = &hdev->user_interrupt[interrupt_offset];
2767 /* Add pending user interrupt to relevant list for the interrupt
2768 * handler to monitor
2770 spin_lock_irqsave(&interrupt->wait_list_lock, flags);
2771 list_add_tail(&pend->wait_list_node, &interrupt->wait_list_head);
2772 spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
2774 /* We check for completion value as interrupt could have been received
2775 * before we added the node to the wait list
2777 if (copy_from_user(&completion_value, u64_to_user_ptr(user_address), 4)) {
2778 dev_err(hdev->dev, "Failed to copy completion value from user\n");
2780 goto remove_pending_user_interrupt;
2783 if (completion_value >= target_value)
2784 *status = CS_WAIT_STATUS_COMPLETED;
2786 *status = CS_WAIT_STATUS_BUSY;
2788 if (!timeout_us || (*status == CS_WAIT_STATUS_COMPLETED))
2789 goto remove_pending_user_interrupt;
2792 /* Wait for interrupt handler to signal completion */
2793 completion_rc = wait_for_completion_interruptible_timeout(&pend->fence.completion,
2796 /* If timeout did not expire we need to perform the comparison.
2797 * If comparison fails, keep waiting until timeout expires
2799 if (completion_rc > 0) {
2800 spin_lock_irqsave(&interrupt->wait_list_lock, flags);
2801 /* reinit_completion must be called before we check for user
2802 * completion value, otherwise, if interrupt is received after
2803 * the comparison and before the next wait_for_completion,
2804 * we will reach timeout and fail
2806 reinit_completion(&pend->fence.completion);
2807 spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
2809 if (copy_from_user(&completion_value, u64_to_user_ptr(user_address), 4)) {
2810 dev_err(hdev->dev, "Failed to copy completion value from user\n");
2813 goto remove_pending_user_interrupt;
2816 if (completion_value >= target_value) {
2817 *status = CS_WAIT_STATUS_COMPLETED;
2819 timeout = completion_rc;
2822 } else if (completion_rc == -ERESTARTSYS) {
2823 dev_err_ratelimited(hdev->dev,
2824 "user process got signal while waiting for interrupt ID %d\n",
2825 interrupt->interrupt_id);
2826 *status = HL_WAIT_CS_STATUS_INTERRUPTED;
2829 *status = CS_WAIT_STATUS_BUSY;
2832 remove_pending_user_interrupt:
2833 spin_lock_irqsave(&interrupt->wait_list_lock, flags);
2834 list_del(&pend->wait_list_node);
2835 spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
2843 static int hl_interrupt_wait_ioctl(struct hl_fpriv *hpriv, void *data)
2845 u16 interrupt_id, interrupt_offset, first_interrupt, last_interrupt;
2846 struct hl_device *hdev = hpriv->hdev;
2847 struct asic_fixed_properties *prop;
2848 union hl_wait_cs_args *args = data;
2849 enum hl_cs_wait_status status;
2852 prop = &hdev->asic_prop;
2854 if (!prop->user_interrupt_count) {
2855 dev_err(hdev->dev, "no user interrupts allowed");
2860 FIELD_GET(HL_WAIT_CS_FLAGS_INTERRUPT_MASK, args->in.flags);
2862 first_interrupt = prop->first_available_user_msix_interrupt;
2863 last_interrupt = prop->first_available_user_msix_interrupt +
2864 prop->user_interrupt_count - 1;
2866 if ((interrupt_id < first_interrupt || interrupt_id > last_interrupt) &&
2867 interrupt_id != HL_COMMON_USER_INTERRUPT_ID) {
2868 dev_err(hdev->dev, "invalid user interrupt %u", interrupt_id);
2872 if (interrupt_id == HL_COMMON_USER_INTERRUPT_ID)
2873 interrupt_offset = HL_COMMON_USER_INTERRUPT_ID;
2875 interrupt_offset = interrupt_id - first_interrupt;
2877 rc = _hl_interrupt_wait_ioctl(hdev, hpriv->ctx,
2878 args->in.interrupt_timeout_us, args->in.addr,
2879 args->in.target, interrupt_offset, &status);
2881 memset(args, 0, sizeof(*args));
2885 dev_err_ratelimited(hdev->dev,
2886 "interrupt_wait_ioctl failed (%d)\n", rc);
2892 case CS_WAIT_STATUS_COMPLETED:
2893 args->out.status = HL_WAIT_CS_STATUS_COMPLETED;
2895 case CS_WAIT_STATUS_BUSY:
2897 args->out.status = HL_WAIT_CS_STATUS_BUSY;
2904 int hl_wait_ioctl(struct hl_fpriv *hpriv, void *data)
2906 union hl_wait_cs_args *args = data;
2907 u32 flags = args->in.flags;
2910 /* If the device is not operational, no point in waiting for any command submission or
2913 if (!hl_device_operational(hpriv->hdev, NULL))
2916 if (flags & HL_WAIT_CS_FLAGS_INTERRUPT)
2917 rc = hl_interrupt_wait_ioctl(hpriv, data);
2918 else if (flags & HL_WAIT_CS_FLAGS_MULTI_CS)
2919 rc = hl_multi_cs_wait_ioctl(hpriv, data);
2921 rc = hl_cs_wait_ioctl(hpriv, data);