drivers/misc/habanalabs/irq.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include "habanalabs.h"
   9
  10 #include <linux/slab.h>
  11
  12 /**
  13  * This structure is used to schedule work of EQ entry and armcp_reset event
  14  *
  15  * @eq_work          - workqueue object to run when EQ entry is received
  16  * @hdev             - pointer to device structure
  17  * @eq_entry         - copy of the EQ entry
  18  */
  19 struct hl_eqe_work {
  20         struct work_struct      eq_work;
  21         struct hl_device        *hdev;
  22         struct hl_eq_entry      eq_entry;
  23 };
  24
  25 /*
  26  * hl_cq_inc_ptr - increment ci or pi of cq
  27  *
  28  * @ptr: the current ci or pi value of the completion queue
  29  *
  30  * Increment ptr by 1. If it reaches the number of completion queue
  31  * entries, set it to 0
  32  */
  33 inline u32 hl_cq_inc_ptr(u32 ptr)
  34 {
  35         ptr++;
  36         if (unlikely(ptr == HL_CQ_LENGTH))
  37                 ptr = 0;
  38         return ptr;
  39 }
  40
  41 /*
  42  * hl_eq_inc_ptr - increment ci of eq
  43  *
  44  * @ptr: the current ci value of the event queue
  45  *
  46  * Increment ptr by 1. If it reaches the number of event queue
  47  * entries, set it to 0
  48  */
  49 inline u32 hl_eq_inc_ptr(u32 ptr)
  50 {
  51         ptr++;
  52         if (unlikely(ptr == HL_EQ_LENGTH))
  53                 ptr = 0;
  54         return ptr;
  55 }
  56
  57 static void irq_handle_eqe(struct work_struct *work)
  58 {
  59         struct hl_eqe_work *eqe_work = container_of(work, struct hl_eqe_work,
  60                                                         eq_work);
  61         struct hl_device *hdev = eqe_work->hdev;
  62
  63         hdev->asic_funcs->handle_eqe(hdev, &eqe_work->eq_entry);
  64
  65         kfree(eqe_work);
  66 }
  67
  68 /*
  69  * hl_irq_handler_cq - irq handler for completion queue
  70  *
  71  * @irq: irq number
  72  * @arg: pointer to completion queue structure
  73  *
  74  */
  75 irqreturn_t hl_irq_handler_cq(int irq, void *arg)
  76 {
  77         struct hl_cq *cq = arg;
  78         struct hl_device *hdev = cq->hdev;
  79         struct hl_hw_queue *queue;
  80         struct hl_cs_job *job;
  81         bool shadow_index_valid;
  82         u16 shadow_index;
  83         u32 *cq_entry;
  84         u32 *cq_base;
  85
  86         if (hdev->disabled) {
  87                 dev_dbg(hdev->dev,
  88                         "Device disabled but received IRQ %d for CQ %d\n",
  89                         irq, cq->hw_queue_id);
  90                 return IRQ_HANDLED;
  91         }
  92
  93         cq_base = (u32 *) (uintptr_t) cq->kernel_address;
  94
  95         while (1) {
  96                 bool entry_ready = ((cq_base[cq->ci] & CQ_ENTRY_READY_MASK)
  97                                                 >> CQ_ENTRY_READY_SHIFT);
  98
  99                 if (!entry_ready)
 100                         break;
 101
 102                 cq_entry = (u32 *) &cq_base[cq->ci];
 103
 104                 /*
 105                  * Make sure we read CQ entry contents after we've
 106                  * checked the ownership bit.
 107                  */
 108                 dma_rmb();
 109
 110                 shadow_index_valid =
 111                         ((*cq_entry & CQ_ENTRY_SHADOW_INDEX_VALID_MASK)
 112                                         >> CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT);
 113
 114                 shadow_index = (u16)
 115                         ((*cq_entry & CQ_ENTRY_SHADOW_INDEX_MASK)
 116                                         >> CQ_ENTRY_SHADOW_INDEX_SHIFT);
 117
 118                 queue = &hdev->kernel_queues[cq->hw_queue_id];
 119
 120                 if ((shadow_index_valid) && (!hdev->disabled)) {
 121                         job = queue->shadow_queue[hl_pi_2_offset(shadow_index)];
 122                         queue_work(hdev->cq_wq, &job->finish_work);
 123                 }
 124
 125                 /*
 126                  * Update ci of the context's queue. There is no
 127                  * need to protect it with spinlock because this update is
 128                  * done only inside IRQ and there is a different IRQ per
 129                  * queue
 130                  */
 131                 queue->ci = hl_queue_inc_ptr(queue->ci);
 132
 133                 /* Clear CQ entry ready bit */
 134                 cq_base[cq->ci] &= ~CQ_ENTRY_READY_MASK;
 135
 136                 cq->ci = hl_cq_inc_ptr(cq->ci);
 137
 138                 /* Increment free slots */
 139                 atomic_inc(&cq->free_slots_cnt);
 140         }
 141
 142         return IRQ_HANDLED;
 143 }
 144
 145 /*
 146  * hl_irq_handler_eq - irq handler for event queue
 147  *
 148  * @irq: irq number
 149  * @arg: pointer to event queue structure
 150  *
 151  */
 152 irqreturn_t hl_irq_handler_eq(int irq, void *arg)
 153 {
 154         struct hl_eq *eq = arg;
 155         struct hl_device *hdev = eq->hdev;
 156         struct hl_eq_entry *eq_entry;
 157         struct hl_eq_entry *eq_base;
 158         struct hl_eqe_work *handle_eqe_work;
 159
 160         eq_base = (struct hl_eq_entry *) (uintptr_t) eq->kernel_address;
 161
 162         while (1) {
 163                 bool entry_ready =
 164                         ((__le32_to_cpu(eq_base[eq->ci].hdr.ctl) &
 165                                 EQ_CTL_READY_MASK) >> EQ_CTL_READY_SHIFT);
 166
 167                 if (!entry_ready)
 168                         break;
 169
 170                 eq_entry = &eq_base[eq->ci];
 171
 172                 /*
 173                  * Make sure we read EQ entry contents after we've
 174                  * checked the ownership bit.
 175                  */
 176                 dma_rmb();
 177
 178                 if (hdev->disabled) {
 179                         dev_warn(hdev->dev,
 180                                 "Device disabled but received IRQ %d for EQ\n",
 181                                         irq);
 182                         goto skip_irq;
 183                 }
 184
 185                 handle_eqe_work = kmalloc(sizeof(*handle_eqe_work), GFP_ATOMIC);
 186                 if (handle_eqe_work) {
 187                         INIT_WORK(&handle_eqe_work->eq_work, irq_handle_eqe);
 188                         handle_eqe_work->hdev = hdev;
 189
 190                         memcpy(&handle_eqe_work->eq_entry, eq_entry,
 191                                         sizeof(*eq_entry));
 192
 193                         queue_work(hdev->eq_wq, &handle_eqe_work->eq_work);
 194                 }
 195 skip_irq:
 196                 /* Clear EQ entry ready bit */
 197                 eq_entry->hdr.ctl =
 198                         __cpu_to_le32(__le32_to_cpu(eq_entry->hdr.ctl) &
 199                                                         ~EQ_CTL_READY_MASK);
 200
 201                 eq->ci = hl_eq_inc_ptr(eq->ci);
 202
 203                 hdev->asic_funcs->update_eq_ci(hdev, eq->ci);
 204         }
 205
 206         return IRQ_HANDLED;
 207 }
 208
 209 /*
 210  * hl_cq_init - main initialization function for an cq object
 211  *
 212  * @hdev: pointer to device structure
 213  * @q: pointer to cq structure
 214  * @hw_queue_id: The H/W queue ID this completion queue belongs to
 215  *
 216  * Allocate dma-able memory for the completion queue and initialize fields
 217  * Returns 0 on success
 218  */
 219 int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id)
 220 {
 221         void *p;
 222
 223         BUILD_BUG_ON(HL_CQ_SIZE_IN_BYTES > HL_PAGE_SIZE);
 224
 225         p = hdev->asic_funcs->dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
 226                                 &q->bus_address, GFP_KERNEL | __GFP_ZERO);
 227         if (!p)
 228                 return -ENOMEM;
 229
 230         q->hdev = hdev;
 231         q->kernel_address = (u64) (uintptr_t) p;
 232         q->hw_queue_id = hw_queue_id;
 233         q->ci = 0;
 234         q->pi = 0;
 235
 236         atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
 237
 238         return 0;
 239 }
 240
 241 /*
 242  * hl_cq_fini - destroy completion queue
 243  *
 244  * @hdev: pointer to device structure
 245  * @q: pointer to cq structure
 246  *
 247  * Free the completion queue memory
 248  */
 249 void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q)
 250 {
 251         hdev->asic_funcs->dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
 252                         (void *) (uintptr_t) q->kernel_address, q->bus_address);
 253 }
 254
 255 void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q)
 256 {
 257         q->ci = 0;
 258         q->pi = 0;
 259
 260         atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
 261
 262         /*
 263          * It's not enough to just reset the PI/CI because the H/W may have
 264          * written valid completion entries before it was halted and therefore
 265          * we need to clean the actual queues so we won't process old entries
 266          * when the device is operational again
 267          */
 268
 269         memset((void *) (uintptr_t) q->kernel_address, 0, HL_CQ_SIZE_IN_BYTES);
 270 }
 271
 272 /*
 273  * hl_eq_init - main initialization function for an event queue object
 274  *
 275  * @hdev: pointer to device structure
 276  * @q: pointer to eq structure
 277  *
 278  * Allocate dma-able memory for the event queue and initialize fields
 279  * Returns 0 on success
 280  */
 281 int hl_eq_init(struct hl_device *hdev, struct hl_eq *q)
 282 {
 283         void *p;
 284
 285         BUILD_BUG_ON(HL_EQ_SIZE_IN_BYTES > HL_PAGE_SIZE);
 286
 287         p = hdev->asic_funcs->dma_alloc_coherent(hdev, HL_EQ_SIZE_IN_BYTES,
 288                                 &q->bus_address, GFP_KERNEL | __GFP_ZERO);
 289         if (!p)
 290                 return -ENOMEM;
 291
 292         q->hdev = hdev;
 293         q->kernel_address = (u64) (uintptr_t) p;
 294         q->ci = 0;
 295
 296         return 0;
 297 }
 298
 299 /*
 300  * hl_eq_fini - destroy event queue
 301  *
 302  * @hdev: pointer to device structure
 303  * @q: pointer to eq structure
 304  *
 305  * Free the event queue memory
 306  */
 307 void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q)
 308 {
 309         flush_workqueue(hdev->eq_wq);
 310
 311         hdev->asic_funcs->dma_free_coherent(hdev, HL_EQ_SIZE_IN_BYTES,
 312                         (void *) (uintptr_t) q->kernel_address, q->bus_address);
 313 }
 314
 315 void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
 316 {
 317         q->ci = 0;
 318
 319         /*
 320          * It's not enough to just reset the PI/CI because the H/W may have
 321          * written valid completion entries before it was halted and therefore
 322          * we need to clean the actual queues so we won't process old entries
 323          * when the device is operational again
 324          */
 325
 326         memset((void *) (uintptr_t) q->kernel_address, 0, HL_EQ_SIZE_IN_BYTES);
 327 }