drivers/misc/habanalabs/common/irq.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include "habanalabs.h"
   9
  10 #include <linux/slab.h>
  11
  12 /**
  13  * struct hl_eqe_work - This structure is used to schedule work of EQ
  14  *                      entry and cpucp_reset event
  15  *
  16  * @eq_work:          workqueue object to run when EQ entry is received
  17  * @hdev:             pointer to device structure
  18  * @eq_entry:         copy of the EQ entry
  19  */
  20 struct hl_eqe_work {
  21         struct work_struct      eq_work;
  22         struct hl_device        *hdev;
  23         struct hl_eq_entry      eq_entry;
  24 };
  25
  26 /**
  27  * hl_cq_inc_ptr - increment ci or pi of cq
  28  *
  29  * @ptr: the current ci or pi value of the completion queue
  30  *
  31  * Increment ptr by 1. If it reaches the number of completion queue
  32  * entries, set it to 0
  33  */
  34 inline u32 hl_cq_inc_ptr(u32 ptr)
  35 {
  36         ptr++;
  37         if (unlikely(ptr == HL_CQ_LENGTH))
  38                 ptr = 0;
  39         return ptr;
  40 }
  41
  42 /**
  43  * hl_eq_inc_ptr - increment ci of eq
  44  *
  45  * @ptr: the current ci value of the event queue
  46  *
  47  * Increment ptr by 1. If it reaches the number of event queue
  48  * entries, set it to 0
  49  */
  50 static inline u32 hl_eq_inc_ptr(u32 ptr)
  51 {
  52         ptr++;
  53         if (unlikely(ptr == HL_EQ_LENGTH))
  54                 ptr = 0;
  55         return ptr;
  56 }
  57
  58 static void irq_handle_eqe(struct work_struct *work)
  59 {
  60         struct hl_eqe_work *eqe_work = container_of(work, struct hl_eqe_work,
  61                                                         eq_work);
  62         struct hl_device *hdev = eqe_work->hdev;
  63
  64         hdev->asic_funcs->handle_eqe(hdev, &eqe_work->eq_entry);
  65
  66         kfree(eqe_work);
  67 }
  68
  69 /**
  70  * hl_irq_handler_cq - irq handler for completion queue
  71  *
  72  * @irq: irq number
  73  * @arg: pointer to completion queue structure
  74  *
  75  */
  76 irqreturn_t hl_irq_handler_cq(int irq, void *arg)
  77 {
  78         struct hl_cq *cq = arg;
  79         struct hl_device *hdev = cq->hdev;
  80         struct hl_hw_queue *queue;
  81         struct hl_cs_job *job;
  82         bool shadow_index_valid;
  83         u16 shadow_index;
  84         struct hl_cq_entry *cq_entry, *cq_base;
  85
  86         if (hdev->disabled) {
  87                 dev_dbg(hdev->dev,
  88                         "Device disabled but received IRQ %d for CQ %d\n",
  89                         irq, cq->hw_queue_id);
  90                 return IRQ_HANDLED;
  91         }
  92
  93         cq_base = cq->kernel_address;
  94
  95         while (1) {
  96                 bool entry_ready = ((le32_to_cpu(cq_base[cq->ci].data) &
  97                                         CQ_ENTRY_READY_MASK)
  98                                                 >> CQ_ENTRY_READY_SHIFT);
  99
 100                 if (!entry_ready)
 101                         break;
 102
 103                 cq_entry = (struct hl_cq_entry *) &cq_base[cq->ci];
 104
 105                 /* Make sure we read CQ entry contents after we've
 106                  * checked the ownership bit.
 107                  */
 108                 dma_rmb();
 109
 110                 shadow_index_valid = ((le32_to_cpu(cq_entry->data) &
 111                                         CQ_ENTRY_SHADOW_INDEX_VALID_MASK)
 112                                         >> CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT);
 113
 114                 shadow_index = (u16) ((le32_to_cpu(cq_entry->data) &
 115                                         CQ_ENTRY_SHADOW_INDEX_MASK)
 116                                         >> CQ_ENTRY_SHADOW_INDEX_SHIFT);
 117
 118                 queue = &hdev->kernel_queues[cq->hw_queue_id];
 119
 120                 if ((shadow_index_valid) && (!hdev->disabled)) {
 121                         job = queue->shadow_queue[hl_pi_2_offset(shadow_index)];
 122                         queue_work(hdev->cq_wq[cq->cq_idx], &job->finish_work);
 123                 }
 124
 125                 atomic_inc(&queue->ci);
 126
 127                 /* Clear CQ entry ready bit */
 128                 cq_entry->data = cpu_to_le32(le32_to_cpu(cq_entry->data) &
 129                                                 ~CQ_ENTRY_READY_MASK);
 130
 131                 cq->ci = hl_cq_inc_ptr(cq->ci);
 132
 133                 /* Increment free slots */
 134                 atomic_inc(&cq->free_slots_cnt);
 135         }
 136
 137         return IRQ_HANDLED;
 138 }
 139
 140 static void handle_user_cq(struct hl_device *hdev,
 141                         struct hl_user_interrupt *user_cq)
 142 {
 143         struct hl_user_pending_interrupt *pend;
 144
 145         spin_lock(&user_cq->wait_list_lock);
 146         list_for_each_entry(pend, &user_cq->wait_list_head, wait_list_node)
 147                 complete_all(&pend->fence.completion);
 148         spin_unlock(&user_cq->wait_list_lock);
 149 }
 150
 151 /**
 152  * hl_irq_handler_user_cq - irq handler for user completion queues
 153  *
 154  * @irq: irq number
 155  * @arg: pointer to user interrupt structure
 156  *
 157  */
 158 irqreturn_t hl_irq_handler_user_cq(int irq, void *arg)
 159 {
 160         struct hl_user_interrupt *user_cq = arg;
 161         struct hl_device *hdev = user_cq->hdev;
 162
 163         dev_dbg(hdev->dev,
 164                 "got user completion interrupt id %u",
 165                 user_cq->interrupt_id);
 166
 167         /* Handle user cq interrupts registered on all interrupts */
 168         handle_user_cq(hdev, &hdev->common_user_interrupt);
 169
 170         /* Handle user cq interrupts registered on this specific interrupt */
 171         handle_user_cq(hdev, user_cq);
 172
 173         return IRQ_HANDLED;
 174 }
 175
 176 /**
 177  * hl_irq_handler_default - default irq handler
 178  *
 179  * @irq: irq number
 180  * @arg: pointer to user interrupt structure
 181  *
 182  */
 183 irqreturn_t hl_irq_handler_default(int irq, void *arg)
 184 {
 185         struct hl_user_interrupt *user_interrupt = arg;
 186         struct hl_device *hdev = user_interrupt->hdev;
 187         u32 interrupt_id = user_interrupt->interrupt_id;
 188
 189         dev_err(hdev->dev,
 190                 "got invalid user interrupt %u",
 191                 interrupt_id);
 192
 193         return IRQ_HANDLED;
 194 }
 195
 196 /**
 197  * hl_irq_handler_eq - irq handler for event queue
 198  *
 199  * @irq: irq number
 200  * @arg: pointer to event queue structure
 201  *
 202  */
 203 irqreturn_t hl_irq_handler_eq(int irq, void *arg)
 204 {
 205         struct hl_eq *eq = arg;
 206         struct hl_device *hdev = eq->hdev;
 207         struct hl_eq_entry *eq_entry;
 208         struct hl_eq_entry *eq_base;
 209         struct hl_eqe_work *handle_eqe_work;
 210
 211         eq_base = eq->kernel_address;
 212
 213         while (1) {
 214                 bool entry_ready =
 215                         ((le32_to_cpu(eq_base[eq->ci].hdr.ctl) &
 216                                 EQ_CTL_READY_MASK) >> EQ_CTL_READY_SHIFT);
 217
 218                 if (!entry_ready)
 219                         break;
 220
 221                 eq_entry = &eq_base[eq->ci];
 222
 223                 /*
 224                  * Make sure we read EQ entry contents after we've
 225                  * checked the ownership bit.
 226                  */
 227                 dma_rmb();
 228
 229                 if (hdev->disabled) {
 230                         dev_warn(hdev->dev,
 231                                 "Device disabled but received IRQ %d for EQ\n",
 232                                         irq);
 233                         goto skip_irq;
 234                 }
 235
 236                 handle_eqe_work = kmalloc(sizeof(*handle_eqe_work), GFP_ATOMIC);
 237                 if (handle_eqe_work) {
 238                         INIT_WORK(&handle_eqe_work->eq_work, irq_handle_eqe);
 239                         handle_eqe_work->hdev = hdev;
 240
 241                         memcpy(&handle_eqe_work->eq_entry, eq_entry,
 242                                         sizeof(*eq_entry));
 243
 244                         queue_work(hdev->eq_wq, &handle_eqe_work->eq_work);
 245                 }
 246 skip_irq:
 247                 /* Clear EQ entry ready bit */
 248                 eq_entry->hdr.ctl =
 249                         cpu_to_le32(le32_to_cpu(eq_entry->hdr.ctl) &
 250                                                         ~EQ_CTL_READY_MASK);
 251
 252                 eq->ci = hl_eq_inc_ptr(eq->ci);
 253
 254                 hdev->asic_funcs->update_eq_ci(hdev, eq->ci);
 255         }
 256
 257         return IRQ_HANDLED;
 258 }
 259
 260 /**
 261  * hl_cq_init - main initialization function for an cq object
 262  *
 263  * @hdev: pointer to device structure
 264  * @q: pointer to cq structure
 265  * @hw_queue_id: The H/W queue ID this completion queue belongs to
 266  *
 267  * Allocate dma-able memory for the completion queue and initialize fields
 268  * Returns 0 on success
 269  */
 270 int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id)
 271 {
 272         void *p;
 273
 274         p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
 275                                 &q->bus_address, GFP_KERNEL | __GFP_ZERO);
 276         if (!p)
 277                 return -ENOMEM;
 278
 279         q->hdev = hdev;
 280         q->kernel_address = p;
 281         q->hw_queue_id = hw_queue_id;
 282         q->ci = 0;
 283         q->pi = 0;
 284
 285         atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
 286
 287         return 0;
 288 }
 289
 290 /**
 291  * hl_cq_fini - destroy completion queue
 292  *
 293  * @hdev: pointer to device structure
 294  * @q: pointer to cq structure
 295  *
 296  * Free the completion queue memory
 297  */
 298 void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q)
 299 {
 300         hdev->asic_funcs->asic_dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
 301                                                  q->kernel_address,
 302                                                  q->bus_address);
 303 }
 304
 305 void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q)
 306 {
 307         q->ci = 0;
 308         q->pi = 0;
 309
 310         atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
 311
 312         /*
 313          * It's not enough to just reset the PI/CI because the H/W may have
 314          * written valid completion entries before it was halted and therefore
 315          * we need to clean the actual queues so we won't process old entries
 316          * when the device is operational again
 317          */
 318
 319         memset(q->kernel_address, 0, HL_CQ_SIZE_IN_BYTES);
 320 }
 321
 322 /**
 323  * hl_eq_init - main initialization function for an event queue object
 324  *
 325  * @hdev: pointer to device structure
 326  * @q: pointer to eq structure
 327  *
 328  * Allocate dma-able memory for the event queue and initialize fields
 329  * Returns 0 on success
 330  */
 331 int hl_eq_init(struct hl_device *hdev, struct hl_eq *q)
 332 {
 333         void *p;
 334
 335         p = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
 336                                                         HL_EQ_SIZE_IN_BYTES,
 337                                                         &q->bus_address);
 338         if (!p)
 339                 return -ENOMEM;
 340
 341         q->hdev = hdev;
 342         q->kernel_address = p;
 343         q->ci = 0;
 344
 345         return 0;
 346 }
 347
 348 /**
 349  * hl_eq_fini - destroy event queue
 350  *
 351  * @hdev: pointer to device structure
 352  * @q: pointer to eq structure
 353  *
 354  * Free the event queue memory
 355  */
 356 void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q)
 357 {
 358         flush_workqueue(hdev->eq_wq);
 359
 360         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
 361                                         HL_EQ_SIZE_IN_BYTES,
 362                                         q->kernel_address);
 363 }
 364
 365 void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
 366 {
 367         q->ci = 0;
 368
 369         /*
 370          * It's not enough to just reset the PI/CI because the H/W may have
 371          * written valid completion entries before it was halted and therefore
 372          * we need to clean the actual queues so we won't process old entries
 373          * when the device is operational again
 374          */
 375
 376         memset(q->kernel_address, 0, HL_EQ_SIZE_IN_BYTES);
 377 }