1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #include "habanalabs.h"
10 #include <linux/slab.h>
13 * struct hl_eqe_work - This structure is used to schedule work of EQ
14 * entry and cpucp_reset event
16 * @eq_work: workqueue object to run when EQ entry is received
17 * @hdev: pointer to device structure
18 * @eq_entry: copy of the EQ entry
21 struct work_struct eq_work;
22 struct hl_device *hdev;
23 struct hl_eq_entry eq_entry;
27 * hl_cq_inc_ptr - increment ci or pi of cq
29 * @ptr: the current ci or pi value of the completion queue
31 * Increment ptr by 1. If it reaches the number of completion queue
32 * entries, set it to 0
34 inline u32 hl_cq_inc_ptr(u32 ptr)
37 if (unlikely(ptr == HL_CQ_LENGTH))
43 * hl_eq_inc_ptr - increment ci of eq
45 * @ptr: the current ci value of the event queue
47 * Increment ptr by 1. If it reaches the number of event queue
48 * entries, set it to 0
50 static inline u32 hl_eq_inc_ptr(u32 ptr)
53 if (unlikely(ptr == HL_EQ_LENGTH))
58 static void irq_handle_eqe(struct work_struct *work)
60 struct hl_eqe_work *eqe_work = container_of(work, struct hl_eqe_work,
62 struct hl_device *hdev = eqe_work->hdev;
64 hdev->asic_funcs->handle_eqe(hdev, &eqe_work->eq_entry);
70 * hl_irq_handler_cq - irq handler for completion queue
73 * @arg: pointer to completion queue structure
76 irqreturn_t hl_irq_handler_cq(int irq, void *arg)
78 struct hl_cq *cq = arg;
79 struct hl_device *hdev = cq->hdev;
80 struct hl_hw_queue *queue;
81 struct hl_cs_job *job;
82 bool shadow_index_valid;
84 struct hl_cq_entry *cq_entry, *cq_base;
88 "Device disabled but received IRQ %d for CQ %d\n",
89 irq, cq->hw_queue_id);
93 cq_base = cq->kernel_address;
96 bool entry_ready = ((le32_to_cpu(cq_base[cq->ci].data) &
98 >> CQ_ENTRY_READY_SHIFT);
103 cq_entry = (struct hl_cq_entry *) &cq_base[cq->ci];
105 /* Make sure we read CQ entry contents after we've
106 * checked the ownership bit.
110 shadow_index_valid = ((le32_to_cpu(cq_entry->data) &
111 CQ_ENTRY_SHADOW_INDEX_VALID_MASK)
112 >> CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT);
114 shadow_index = (u16) ((le32_to_cpu(cq_entry->data) &
115 CQ_ENTRY_SHADOW_INDEX_MASK)
116 >> CQ_ENTRY_SHADOW_INDEX_SHIFT);
118 queue = &hdev->kernel_queues[cq->hw_queue_id];
120 if ((shadow_index_valid) && (!hdev->disabled)) {
121 job = queue->shadow_queue[hl_pi_2_offset(shadow_index)];
122 queue_work(hdev->cq_wq[cq->cq_idx], &job->finish_work);
125 atomic_inc(&queue->ci);
127 /* Clear CQ entry ready bit */
128 cq_entry->data = cpu_to_le32(le32_to_cpu(cq_entry->data) &
129 ~CQ_ENTRY_READY_MASK);
131 cq->ci = hl_cq_inc_ptr(cq->ci);
133 /* Increment free slots */
134 atomic_inc(&cq->free_slots_cnt);
140 static void handle_user_cq(struct hl_device *hdev,
141 struct hl_user_interrupt *user_cq)
143 struct hl_user_pending_interrupt *pend;
144 ktime_t now = ktime_get();
146 spin_lock(&user_cq->wait_list_lock);
147 list_for_each_entry(pend, &user_cq->wait_list_head, wait_list_node) {
148 if ((pend->cq_kernel_addr &&
149 *(pend->cq_kernel_addr) >= pend->cq_target_value) ||
150 !pend->cq_kernel_addr) {
151 pend->fence.timestamp = now;
152 complete_all(&pend->fence.completion);
155 spin_unlock(&user_cq->wait_list_lock);
159 * hl_irq_handler_user_cq - irq handler for user completion queues
162 * @arg: pointer to user interrupt structure
165 irqreturn_t hl_irq_handler_user_cq(int irq, void *arg)
167 struct hl_user_interrupt *user_cq = arg;
168 struct hl_device *hdev = user_cq->hdev;
171 "got user completion interrupt id %u",
172 user_cq->interrupt_id);
174 /* Handle user cq interrupts registered on all interrupts */
175 handle_user_cq(hdev, &hdev->common_user_interrupt);
177 /* Handle user cq interrupts registered on this specific interrupt */
178 handle_user_cq(hdev, user_cq);
184 * hl_irq_handler_default - default irq handler
187 * @arg: pointer to user interrupt structure
190 irqreturn_t hl_irq_handler_default(int irq, void *arg)
192 struct hl_user_interrupt *user_interrupt = arg;
193 struct hl_device *hdev = user_interrupt->hdev;
194 u32 interrupt_id = user_interrupt->interrupt_id;
197 "got invalid user interrupt %u",
204 * hl_irq_handler_eq - irq handler for event queue
207 * @arg: pointer to event queue structure
210 irqreturn_t hl_irq_handler_eq(int irq, void *arg)
212 struct hl_eq *eq = arg;
213 struct hl_device *hdev = eq->hdev;
214 struct hl_eq_entry *eq_entry;
215 struct hl_eq_entry *eq_base;
216 struct hl_eqe_work *handle_eqe_work;
221 eq_base = eq->kernel_address;
224 cur_eqe = le32_to_cpu(eq_base[eq->ci].hdr.ctl);
225 entry_ready = !!FIELD_GET(EQ_CTL_READY_MASK, cur_eqe);
230 cur_eqe_index = FIELD_GET(EQ_CTL_INDEX_MASK, cur_eqe);
231 if ((hdev->event_queue.check_eqe_index) &&
232 (((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK)
235 "EQE 0x%x in queue is ready but index does not match %d!=%d",
236 eq_base[eq->ci].hdr.ctl,
237 ((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK),
242 eq->prev_eqe_index++;
244 eq_entry = &eq_base[eq->ci];
247 * Make sure we read EQ entry contents after we've
248 * checked the ownership bit.
252 if (hdev->disabled && !hdev->reset_info.is_in_soft_reset) {
253 dev_warn(hdev->dev, "Device disabled but received an EQ event\n");
257 handle_eqe_work = kmalloc(sizeof(*handle_eqe_work), GFP_ATOMIC);
258 if (handle_eqe_work) {
259 INIT_WORK(&handle_eqe_work->eq_work, irq_handle_eqe);
260 handle_eqe_work->hdev = hdev;
262 memcpy(&handle_eqe_work->eq_entry, eq_entry,
265 queue_work(hdev->eq_wq, &handle_eqe_work->eq_work);
268 /* Clear EQ entry ready bit */
270 cpu_to_le32(le32_to_cpu(eq_entry->hdr.ctl) &
273 eq->ci = hl_eq_inc_ptr(eq->ci);
275 hdev->asic_funcs->update_eq_ci(hdev, eq->ci);
282 * hl_cq_init - main initialization function for an cq object
284 * @hdev: pointer to device structure
285 * @q: pointer to cq structure
286 * @hw_queue_id: The H/W queue ID this completion queue belongs to
288 * Allocate dma-able memory for the completion queue and initialize fields
289 * Returns 0 on success
291 int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id)
295 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
296 &q->bus_address, GFP_KERNEL | __GFP_ZERO);
301 q->kernel_address = p;
302 q->hw_queue_id = hw_queue_id;
306 atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
312 * hl_cq_fini - destroy completion queue
314 * @hdev: pointer to device structure
315 * @q: pointer to cq structure
317 * Free the completion queue memory
319 void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q)
321 hdev->asic_funcs->asic_dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
326 void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q)
331 atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
334 * It's not enough to just reset the PI/CI because the H/W may have
335 * written valid completion entries before it was halted and therefore
336 * we need to clean the actual queues so we won't process old entries
337 * when the device is operational again
340 memset(q->kernel_address, 0, HL_CQ_SIZE_IN_BYTES);
344 * hl_eq_init - main initialization function for an event queue object
346 * @hdev: pointer to device structure
347 * @q: pointer to eq structure
349 * Allocate dma-able memory for the event queue and initialize fields
350 * Returns 0 on success
352 int hl_eq_init(struct hl_device *hdev, struct hl_eq *q)
356 p = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
363 q->kernel_address = p;
365 q->prev_eqe_index = 0;
371 * hl_eq_fini - destroy event queue
373 * @hdev: pointer to device structure
374 * @q: pointer to eq structure
376 * Free the event queue memory
378 void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q)
380 flush_workqueue(hdev->eq_wq);
382 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
387 void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
390 q->prev_eqe_index = 0;
393 * It's not enough to just reset the PI/CI because the H/W may have
394 * written valid completion entries before it was halted and therefore
395 * we need to clean the actual queues so we won't process old entries
396 * when the device is operational again
399 memset(q->kernel_address, 0, HL_EQ_SIZE_IN_BYTES);