drivers/misc/habanalabs/common/irq.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include "habanalabs.h"
   9
  10 #include <linux/slab.h>
  11
  12 /**
  13  * struct hl_eqe_work - This structure is used to schedule work of EQ
  14  *                      entry and cpucp_reset event
  15  *
  16  * @eq_work:          workqueue object to run when EQ entry is received
  17  * @hdev:             pointer to device structure
  18  * @eq_entry:         copy of the EQ entry
  19  */
  20 struct hl_eqe_work {
  21         struct work_struct      eq_work;
  22         struct hl_device        *hdev;
  23         struct hl_eq_entry      eq_entry;
  24 };
  25
  26 /**
  27  * hl_cq_inc_ptr - increment ci or pi of cq
  28  *
  29  * @ptr: the current ci or pi value of the completion queue
  30  *
  31  * Increment ptr by 1. If it reaches the number of completion queue
  32  * entries, set it to 0
  33  */
  34 inline u32 hl_cq_inc_ptr(u32 ptr)
  35 {
  36         ptr++;
  37         if (unlikely(ptr == HL_CQ_LENGTH))
  38                 ptr = 0;
  39         return ptr;
  40 }
  41
  42 /**
  43  * hl_eq_inc_ptr - increment ci of eq
  44  *
  45  * @ptr: the current ci value of the event queue
  46  *
  47  * Increment ptr by 1. If it reaches the number of event queue
  48  * entries, set it to 0
  49  */
  50 static inline u32 hl_eq_inc_ptr(u32 ptr)
  51 {
  52         ptr++;
  53         if (unlikely(ptr == HL_EQ_LENGTH))
  54                 ptr = 0;
  55         return ptr;
  56 }
  57
  58 static void irq_handle_eqe(struct work_struct *work)
  59 {
  60         struct hl_eqe_work *eqe_work = container_of(work, struct hl_eqe_work,
  61                                                         eq_work);
  62         struct hl_device *hdev = eqe_work->hdev;
  63
  64         hdev->asic_funcs->handle_eqe(hdev, &eqe_work->eq_entry);
  65
  66         kfree(eqe_work);
  67 }
  68
  69 /**
  70  * hl_irq_handler_cq - irq handler for completion queue
  71  *
  72  * @irq: irq number
  73  * @arg: pointer to completion queue structure
  74  *
  75  */
  76 irqreturn_t hl_irq_handler_cq(int irq, void *arg)
  77 {
  78         struct hl_cq *cq = arg;
  79         struct hl_device *hdev = cq->hdev;
  80         struct hl_hw_queue *queue;
  81         struct hl_cs_job *job;
  82         bool shadow_index_valid;
  83         u16 shadow_index;
  84         struct hl_cq_entry *cq_entry, *cq_base;
  85
  86         if (hdev->disabled) {
  87                 dev_dbg(hdev->dev,
  88                         "Device disabled but received IRQ %d for CQ %d\n",
  89                         irq, cq->hw_queue_id);
  90                 return IRQ_HANDLED;
  91         }
  92
  93         cq_base = cq->kernel_address;
  94
  95         while (1) {
  96                 bool entry_ready = ((le32_to_cpu(cq_base[cq->ci].data) &
  97                                         CQ_ENTRY_READY_MASK)
  98                                                 >> CQ_ENTRY_READY_SHIFT);
  99
 100                 if (!entry_ready)
 101                         break;
 102
 103                 cq_entry = (struct hl_cq_entry *) &cq_base[cq->ci];
 104
 105                 /* Make sure we read CQ entry contents after we've
 106                  * checked the ownership bit.
 107                  */
 108                 dma_rmb();
 109
 110                 shadow_index_valid = ((le32_to_cpu(cq_entry->data) &
 111                                         CQ_ENTRY_SHADOW_INDEX_VALID_MASK)
 112                                         >> CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT);
 113
 114                 shadow_index = (u16) ((le32_to_cpu(cq_entry->data) &
 115                                         CQ_ENTRY_SHADOW_INDEX_MASK)
 116                                         >> CQ_ENTRY_SHADOW_INDEX_SHIFT);
 117
 118                 queue = &hdev->kernel_queues[cq->hw_queue_id];
 119
 120                 if ((shadow_index_valid) && (!hdev->disabled)) {
 121                         job = queue->shadow_queue[hl_pi_2_offset(shadow_index)];
 122                         queue_work(hdev->cq_wq[cq->cq_idx], &job->finish_work);
 123                 }
 124
 125                 atomic_inc(&queue->ci);
 126
 127                 /* Clear CQ entry ready bit */
 128                 cq_entry->data = cpu_to_le32(le32_to_cpu(cq_entry->data) &
 129                                                 ~CQ_ENTRY_READY_MASK);
 130
 131                 cq->ci = hl_cq_inc_ptr(cq->ci);
 132
 133                 /* Increment free slots */
 134                 atomic_inc(&cq->free_slots_cnt);
 135         }
 136
 137         return IRQ_HANDLED;
 138 }
 139
 140 static void handle_user_cq(struct hl_device *hdev,
 141                         struct hl_user_interrupt *user_cq)
 142 {
 143         struct hl_user_pending_interrupt *pend;
 144         ktime_t now = ktime_get();
 145
 146         spin_lock(&user_cq->wait_list_lock);
 147         list_for_each_entry(pend, &user_cq->wait_list_head, wait_list_node) {
 148                 if ((pend->cq_kernel_addr &&
 149                                 *(pend->cq_kernel_addr) >= pend->cq_target_value) ||
 150                                 !pend->cq_kernel_addr) {
 151                         pend->fence.timestamp = now;
 152                         complete_all(&pend->fence.completion);
 153                 }
 154         }
 155         spin_unlock(&user_cq->wait_list_lock);
 156 }
 157
 158 /**
 159  * hl_irq_handler_user_cq - irq handler for user completion queues
 160  *
 161  * @irq: irq number
 162  * @arg: pointer to user interrupt structure
 163  *
 164  */
 165 irqreturn_t hl_irq_handler_user_cq(int irq, void *arg)
 166 {
 167         struct hl_user_interrupt *user_cq = arg;
 168         struct hl_device *hdev = user_cq->hdev;
 169
 170         dev_dbg(hdev->dev,
 171                 "got user completion interrupt id %u",
 172                 user_cq->interrupt_id);
 173
 174         /* Handle user cq interrupts registered on all interrupts */
 175         handle_user_cq(hdev, &hdev->common_user_interrupt);
 176
 177         /* Handle user cq interrupts registered on this specific interrupt */
 178         handle_user_cq(hdev, user_cq);
 179
 180         return IRQ_HANDLED;
 181 }
 182
 183 /**
 184  * hl_irq_handler_default - default irq handler
 185  *
 186  * @irq: irq number
 187  * @arg: pointer to user interrupt structure
 188  *
 189  */
 190 irqreturn_t hl_irq_handler_default(int irq, void *arg)
 191 {
 192         struct hl_user_interrupt *user_interrupt = arg;
 193         struct hl_device *hdev = user_interrupt->hdev;
 194         u32 interrupt_id = user_interrupt->interrupt_id;
 195
 196         dev_err(hdev->dev,
 197                 "got invalid user interrupt %u",
 198                 interrupt_id);
 199
 200         return IRQ_HANDLED;
 201 }
 202
 203 /**
 204  * hl_irq_handler_eq - irq handler for event queue
 205  *
 206  * @irq: irq number
 207  * @arg: pointer to event queue structure
 208  *
 209  */
 210 irqreturn_t hl_irq_handler_eq(int irq, void *arg)
 211 {
 212         struct hl_eq *eq = arg;
 213         struct hl_device *hdev = eq->hdev;
 214         struct hl_eq_entry *eq_entry;
 215         struct hl_eq_entry *eq_base;
 216         struct hl_eqe_work *handle_eqe_work;
 217         bool entry_ready;
 218         u32 cur_eqe;
 219         u16 cur_eqe_index;
 220
 221         eq_base = eq->kernel_address;
 222
 223         while (1) {
 224                 cur_eqe = le32_to_cpu(eq_base[eq->ci].hdr.ctl);
 225                 entry_ready = !!FIELD_GET(EQ_CTL_READY_MASK, cur_eqe);
 226
 227                 if (!entry_ready)
 228                         break;
 229
 230                 cur_eqe_index = FIELD_GET(EQ_CTL_INDEX_MASK, cur_eqe);
 231                 if ((hdev->event_queue.check_eqe_index) &&
 232                                 (((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK)
 233                                                         != cur_eqe_index)) {
 234                         dev_dbg(hdev->dev,
 235                                 "EQE 0x%x in queue is ready but index does not match %d!=%d",
 236                                 eq_base[eq->ci].hdr.ctl,
 237                                 ((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK),
 238                                 cur_eqe_index);
 239                         break;
 240                 }
 241
 242                 eq->prev_eqe_index++;
 243
 244                 eq_entry = &eq_base[eq->ci];
 245
 246                 /*
 247                  * Make sure we read EQ entry contents after we've
 248                  * checked the ownership bit.
 249                  */
 250                 dma_rmb();
 251
 252                 if (hdev->disabled && !hdev->reset_info.is_in_soft_reset) {
 253                         dev_warn(hdev->dev, "Device disabled but received an EQ event\n");
 254                         goto skip_irq;
 255                 }
 256
 257                 handle_eqe_work = kmalloc(sizeof(*handle_eqe_work), GFP_ATOMIC);
 258                 if (handle_eqe_work) {
 259                         INIT_WORK(&handle_eqe_work->eq_work, irq_handle_eqe);
 260                         handle_eqe_work->hdev = hdev;
 261
 262                         memcpy(&handle_eqe_work->eq_entry, eq_entry,
 263                                         sizeof(*eq_entry));
 264
 265                         queue_work(hdev->eq_wq, &handle_eqe_work->eq_work);
 266                 }
 267 skip_irq:
 268                 /* Clear EQ entry ready bit */
 269                 eq_entry->hdr.ctl =
 270                         cpu_to_le32(le32_to_cpu(eq_entry->hdr.ctl) &
 271                                                         ~EQ_CTL_READY_MASK);
 272
 273                 eq->ci = hl_eq_inc_ptr(eq->ci);
 274
 275                 hdev->asic_funcs->update_eq_ci(hdev, eq->ci);
 276         }
 277
 278         return IRQ_HANDLED;
 279 }
 280
 281 /**
 282  * hl_cq_init - main initialization function for an cq object
 283  *
 284  * @hdev: pointer to device structure
 285  * @q: pointer to cq structure
 286  * @hw_queue_id: The H/W queue ID this completion queue belongs to
 287  *
 288  * Allocate dma-able memory for the completion queue and initialize fields
 289  * Returns 0 on success
 290  */
 291 int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id)
 292 {
 293         void *p;
 294
 295         p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
 296                                 &q->bus_address, GFP_KERNEL | __GFP_ZERO);
 297         if (!p)
 298                 return -ENOMEM;
 299
 300         q->hdev = hdev;
 301         q->kernel_address = p;
 302         q->hw_queue_id = hw_queue_id;
 303         q->ci = 0;
 304         q->pi = 0;
 305
 306         atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
 307
 308         return 0;
 309 }
 310
 311 /**
 312  * hl_cq_fini - destroy completion queue
 313  *
 314  * @hdev: pointer to device structure
 315  * @q: pointer to cq structure
 316  *
 317  * Free the completion queue memory
 318  */
 319 void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q)
 320 {
 321         hdev->asic_funcs->asic_dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
 322                                                  q->kernel_address,
 323                                                  q->bus_address);
 324 }
 325
 326 void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q)
 327 {
 328         q->ci = 0;
 329         q->pi = 0;
 330
 331         atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
 332
 333         /*
 334          * It's not enough to just reset the PI/CI because the H/W may have
 335          * written valid completion entries before it was halted and therefore
 336          * we need to clean the actual queues so we won't process old entries
 337          * when the device is operational again
 338          */
 339
 340         memset(q->kernel_address, 0, HL_CQ_SIZE_IN_BYTES);
 341 }
 342
 343 /**
 344  * hl_eq_init - main initialization function for an event queue object
 345  *
 346  * @hdev: pointer to device structure
 347  * @q: pointer to eq structure
 348  *
 349  * Allocate dma-able memory for the event queue and initialize fields
 350  * Returns 0 on success
 351  */
 352 int hl_eq_init(struct hl_device *hdev, struct hl_eq *q)
 353 {
 354         void *p;
 355
 356         p = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
 357                                                         HL_EQ_SIZE_IN_BYTES,
 358                                                         &q->bus_address);
 359         if (!p)
 360                 return -ENOMEM;
 361
 362         q->hdev = hdev;
 363         q->kernel_address = p;
 364         q->ci = 0;
 365         q->prev_eqe_index = 0;
 366
 367         return 0;
 368 }
 369
 370 /**
 371  * hl_eq_fini - destroy event queue
 372  *
 373  * @hdev: pointer to device structure
 374  * @q: pointer to eq structure
 375  *
 376  * Free the event queue memory
 377  */
 378 void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q)
 379 {
 380         flush_workqueue(hdev->eq_wq);
 381
 382         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
 383                                         HL_EQ_SIZE_IN_BYTES,
 384                                         q->kernel_address);
 385 }
 386
 387 void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
 388 {
 389         q->ci = 0;
 390         q->prev_eqe_index = 0;
 391
 392         /*
 393          * It's not enough to just reset the PI/CI because the H/W may have
 394          * written valid completion entries before it was halted and therefore
 395          * we need to clean the actual queues so we won't process old entries
 396          * when the device is operational again
 397          */
 398
 399         memset(q->kernel_address, 0, HL_EQ_SIZE_IN_BYTES);
 400 }