drivers/misc/habanalabs/common/irq.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include "habanalabs.h"
   9
  10 #include <linux/slab.h>
  11
  12 /**
  13  * struct hl_eqe_work - This structure is used to schedule work of EQ
  14  *                      entry and cpucp_reset event
  15  *
  16  * @eq_work:          workqueue object to run when EQ entry is received
  17  * @hdev:             pointer to device structure
  18  * @eq_entry:         copy of the EQ entry
  19  */
  20 struct hl_eqe_work {
  21         struct work_struct      eq_work;
  22         struct hl_device        *hdev;
  23         struct hl_eq_entry      eq_entry;
  24 };
  25
  26 /**
  27  * hl_cq_inc_ptr - increment ci or pi of cq
  28  *
  29  * @ptr: the current ci or pi value of the completion queue
  30  *
  31  * Increment ptr by 1. If it reaches the number of completion queue
  32  * entries, set it to 0
  33  */
  34 inline u32 hl_cq_inc_ptr(u32 ptr)
  35 {
  36         ptr++;
  37         if (unlikely(ptr == HL_CQ_LENGTH))
  38                 ptr = 0;
  39         return ptr;
  40 }
  41
  42 /**
  43  * hl_eq_inc_ptr - increment ci of eq
  44  *
  45  * @ptr: the current ci value of the event queue
  46  *
  47  * Increment ptr by 1. If it reaches the number of event queue
  48  * entries, set it to 0
  49  */
  50 inline u32 hl_eq_inc_ptr(u32 ptr)
  51 {
  52         ptr++;
  53         if (unlikely(ptr == HL_EQ_LENGTH))
  54                 ptr = 0;
  55         return ptr;
  56 }
  57
  58 static void irq_handle_eqe(struct work_struct *work)
  59 {
  60         struct hl_eqe_work *eqe_work = container_of(work, struct hl_eqe_work,
  61                                                         eq_work);
  62         struct hl_device *hdev = eqe_work->hdev;
  63
  64         hdev->asic_funcs->handle_eqe(hdev, &eqe_work->eq_entry);
  65
  66         kfree(eqe_work);
  67 }
  68
  69 /**
  70  * hl_irq_handler_cq - irq handler for completion queue
  71  *
  72  * @irq: irq number
  73  * @arg: pointer to completion queue structure
  74  *
  75  */
  76 irqreturn_t hl_irq_handler_cq(int irq, void *arg)
  77 {
  78         struct hl_cq *cq = arg;
  79         struct hl_device *hdev = cq->hdev;
  80         struct hl_hw_queue *queue;
  81         struct hl_cs_job *job;
  82         bool shadow_index_valid;
  83         u16 shadow_index;
  84         struct hl_cq_entry *cq_entry, *cq_base;
  85
  86         if (hdev->disabled) {
  87                 dev_dbg(hdev->dev,
  88                         "Device disabled but received IRQ %d for CQ %d\n",
  89                         irq, cq->hw_queue_id);
  90                 return IRQ_HANDLED;
  91         }
  92
  93         cq_base = (struct hl_cq_entry *) (uintptr_t) cq->kernel_address;
  94
  95         while (1) {
  96                 bool entry_ready = ((le32_to_cpu(cq_base[cq->ci].data) &
  97                                         CQ_ENTRY_READY_MASK)
  98                                                 >> CQ_ENTRY_READY_SHIFT);
  99
 100                 if (!entry_ready)
 101                         break;
 102
 103                 cq_entry = (struct hl_cq_entry *) &cq_base[cq->ci];
 104
 105                 /* Make sure we read CQ entry contents after we've
 106                  * checked the ownership bit.
 107                  */
 108                 dma_rmb();
 109
 110                 shadow_index_valid = ((le32_to_cpu(cq_entry->data) &
 111                                         CQ_ENTRY_SHADOW_INDEX_VALID_MASK)
 112                                         >> CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT);
 113
 114                 shadow_index = (u16) ((le32_to_cpu(cq_entry->data) &
 115                                         CQ_ENTRY_SHADOW_INDEX_MASK)
 116                                         >> CQ_ENTRY_SHADOW_INDEX_SHIFT);
 117
 118                 queue = &hdev->kernel_queues[cq->hw_queue_id];
 119
 120                 if ((shadow_index_valid) && (!hdev->disabled)) {
 121                         job = queue->shadow_queue[hl_pi_2_offset(shadow_index)];
 122                         queue_work(hdev->cq_wq[cq->cq_idx], &job->finish_work);
 123                 }
 124
 125                 atomic_inc(&queue->ci);
 126
 127                 /* Clear CQ entry ready bit */
 128                 cq_entry->data = cpu_to_le32(le32_to_cpu(cq_entry->data) &
 129                                                 ~CQ_ENTRY_READY_MASK);
 130
 131                 cq->ci = hl_cq_inc_ptr(cq->ci);
 132
 133                 /* Increment free slots */
 134                 atomic_inc(&cq->free_slots_cnt);
 135         }
 136
 137         return IRQ_HANDLED;
 138 }
 139
 140 /**
 141  * hl_irq_handler_eq - irq handler for event queue
 142  *
 143  * @irq: irq number
 144  * @arg: pointer to event queue structure
 145  *
 146  */
 147 irqreturn_t hl_irq_handler_eq(int irq, void *arg)
 148 {
 149         struct hl_eq *eq = arg;
 150         struct hl_device *hdev = eq->hdev;
 151         struct hl_eq_entry *eq_entry;
 152         struct hl_eq_entry *eq_base;
 153         struct hl_eqe_work *handle_eqe_work;
 154
 155         eq_base = (struct hl_eq_entry *) (uintptr_t) eq->kernel_address;
 156
 157         while (1) {
 158                 bool entry_ready =
 159                         ((le32_to_cpu(eq_base[eq->ci].hdr.ctl) &
 160                                 EQ_CTL_READY_MASK) >> EQ_CTL_READY_SHIFT);
 161
 162                 if (!entry_ready)
 163                         break;
 164
 165                 eq_entry = &eq_base[eq->ci];
 166
 167                 /*
 168                  * Make sure we read EQ entry contents after we've
 169                  * checked the ownership bit.
 170                  */
 171                 dma_rmb();
 172
 173                 if (hdev->disabled) {
 174                         dev_warn(hdev->dev,
 175                                 "Device disabled but received IRQ %d for EQ\n",
 176                                         irq);
 177                         goto skip_irq;
 178                 }
 179
 180                 handle_eqe_work = kmalloc(sizeof(*handle_eqe_work), GFP_ATOMIC);
 181                 if (handle_eqe_work) {
 182                         INIT_WORK(&handle_eqe_work->eq_work, irq_handle_eqe);
 183                         handle_eqe_work->hdev = hdev;
 184
 185                         memcpy(&handle_eqe_work->eq_entry, eq_entry,
 186                                         sizeof(*eq_entry));
 187
 188                         queue_work(hdev->eq_wq, &handle_eqe_work->eq_work);
 189                 }
 190 skip_irq:
 191                 /* Clear EQ entry ready bit */
 192                 eq_entry->hdr.ctl =
 193                         cpu_to_le32(le32_to_cpu(eq_entry->hdr.ctl) &
 194                                                         ~EQ_CTL_READY_MASK);
 195
 196                 eq->ci = hl_eq_inc_ptr(eq->ci);
 197
 198                 hdev->asic_funcs->update_eq_ci(hdev, eq->ci);
 199         }
 200
 201         return IRQ_HANDLED;
 202 }
 203
 204 /**
 205  * hl_cq_init - main initialization function for an cq object
 206  *
 207  * @hdev: pointer to device structure
 208  * @q: pointer to cq structure
 209  * @hw_queue_id: The H/W queue ID this completion queue belongs to
 210  *
 211  * Allocate dma-able memory for the completion queue and initialize fields
 212  * Returns 0 on success
 213  */
 214 int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id)
 215 {
 216         void *p;
 217
 218         p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
 219                                 &q->bus_address, GFP_KERNEL | __GFP_ZERO);
 220         if (!p)
 221                 return -ENOMEM;
 222
 223         q->hdev = hdev;
 224         q->kernel_address = (u64) (uintptr_t) p;
 225         q->hw_queue_id = hw_queue_id;
 226         q->ci = 0;
 227         q->pi = 0;
 228
 229         atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
 230
 231         return 0;
 232 }
 233
 234 /**
 235  * hl_cq_fini - destroy completion queue
 236  *
 237  * @hdev: pointer to device structure
 238  * @q: pointer to cq structure
 239  *
 240  * Free the completion queue memory
 241  */
 242 void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q)
 243 {
 244         hdev->asic_funcs->asic_dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
 245                         (void *) (uintptr_t) q->kernel_address, q->bus_address);
 246 }
 247
 248 void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q)
 249 {
 250         q->ci = 0;
 251         q->pi = 0;
 252
 253         atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
 254
 255         /*
 256          * It's not enough to just reset the PI/CI because the H/W may have
 257          * written valid completion entries before it was halted and therefore
 258          * we need to clean the actual queues so we won't process old entries
 259          * when the device is operational again
 260          */
 261
 262         memset((void *) (uintptr_t) q->kernel_address, 0, HL_CQ_SIZE_IN_BYTES);
 263 }
 264
 265 /**
 266  * hl_eq_init - main initialization function for an event queue object
 267  *
 268  * @hdev: pointer to device structure
 269  * @q: pointer to eq structure
 270  *
 271  * Allocate dma-able memory for the event queue and initialize fields
 272  * Returns 0 on success
 273  */
 274 int hl_eq_init(struct hl_device *hdev, struct hl_eq *q)
 275 {
 276         void *p;
 277
 278         p = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
 279                                                         HL_EQ_SIZE_IN_BYTES,
 280                                                         &q->bus_address);
 281         if (!p)
 282                 return -ENOMEM;
 283
 284         q->hdev = hdev;
 285         q->kernel_address = (u64) (uintptr_t) p;
 286         q->ci = 0;
 287
 288         return 0;
 289 }
 290
 291 /**
 292  * hl_eq_fini - destroy event queue
 293  *
 294  * @hdev: pointer to device structure
 295  * @q: pointer to eq structure
 296  *
 297  * Free the event queue memory
 298  */
 299 void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q)
 300 {
 301         flush_workqueue(hdev->eq_wq);
 302
 303         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
 304                                         HL_EQ_SIZE_IN_BYTES,
 305                                         (void *) (uintptr_t) q->kernel_address);
 306 }
 307
 308 void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
 309 {
 310         q->ci = 0;
 311
 312         /*
 313          * It's not enough to just reset the PI/CI because the H/W may have
 314          * written valid completion entries before it was halted and therefore
 315          * we need to clean the actual queues so we won't process old entries
 316          * when the device is operational again
 317          */
 318
 319         memset((void *) (uintptr_t) q->kernel_address, 0, HL_EQ_SIZE_IN_BYTES);
 320 }