drivers/misc/habanalabs/common/hw_queue.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include "habanalabs.h"
   9
  10 #include <linux/slab.h>
  11
  12 /*
  13  * hl_queue_add_ptr - add to pi or ci and checks if it wraps around
  14  *
  15  * @ptr: the current pi/ci value
  16  * @val: the amount to add
  17  *
  18  * Add val to ptr. It can go until twice the queue length.
  19  */
  20 inline u32 hl_hw_queue_add_ptr(u32 ptr, u16 val)
  21 {
  22         ptr += val;
  23         ptr &= ((HL_QUEUE_LENGTH << 1) - 1);
  24         return ptr;
  25 }
  26 static inline int queue_ci_get(atomic_t *ci, u32 queue_len)
  27 {
  28         return atomic_read(ci) & ((queue_len << 1) - 1);
  29 }
  30
  31 static inline int queue_free_slots(struct hl_hw_queue *q, u32 queue_len)
  32 {
  33         int delta = (q->pi - queue_ci_get(&q->ci, queue_len));
  34
  35         if (delta >= 0)
  36                 return (queue_len - delta);
  37         else
  38                 return (abs(delta) - queue_len);
  39 }
  40
  41 void hl_hw_queue_update_ci(struct hl_cs *cs)
  42 {
  43         struct hl_device *hdev = cs->ctx->hdev;
  44         struct hl_hw_queue *q;
  45         int i;
  46
  47         if (hdev->disabled)
  48                 return;
  49
  50         q = &hdev->kernel_queues[0];
  51
  52         /* There are no internal queues if H/W queues are being used */
  53         if (!hdev->asic_prop.max_queues || q->queue_type == QUEUE_TYPE_HW)
  54                 return;
  55
  56         /* We must increment CI for every queue that will never get a
  57          * completion, there are 2 scenarios this can happen:
  58          * 1. All queues of a non completion CS will never get a completion.
  59          * 2. Internal queues never gets completion.
  60          */
  61         for (i = 0 ; i < hdev->asic_prop.max_queues ; i++, q++) {
  62                 if (!cs_needs_completion(cs) || q->queue_type == QUEUE_TYPE_INT)
  63                         atomic_add(cs->jobs_in_queue_cnt[i], &q->ci);
  64         }
  65 }
  66
  67 /*
  68  * ext_and_hw_queue_submit_bd() - Submit a buffer descriptor to an external or a
  69  *                                H/W queue.
  70  * @hdev: pointer to habanalabs device structure
  71  * @q: pointer to habanalabs queue structure
  72  * @ctl: BD's control word
  73  * @len: BD's length
  74  * @ptr: BD's pointer
  75  *
  76  * This function assumes there is enough space on the queue to submit a new
  77  * BD to it. It initializes the next BD and calls the device specific
  78  * function to set the pi (and doorbell)
  79  *
  80  * This function must be called when the scheduler mutex is taken
  81  *
  82  */
  83 static void ext_and_hw_queue_submit_bd(struct hl_device *hdev,
  84                         struct hl_hw_queue *q, u32 ctl, u32 len, u64 ptr)
  85 {
  86         struct hl_bd *bd;
  87
  88         bd = q->kernel_address;
  89         bd += hl_pi_2_offset(q->pi);
  90         bd->ctl = cpu_to_le32(ctl);
  91         bd->len = cpu_to_le32(len);
  92         bd->ptr = cpu_to_le64(ptr);
  93
  94         q->pi = hl_queue_inc_ptr(q->pi);
  95         hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);
  96 }
  97
  98 /*
  99  * ext_queue_sanity_checks - perform some sanity checks on external queue
 100  *
 101  * @hdev              : pointer to hl_device structure
 102  * @q                 : pointer to hl_hw_queue structure
 103  * @num_of_entries    : how many entries to check for space
 104  * @reserve_cq_entry  : whether to reserve an entry in the cq
 105  *
 106  * H/W queues spinlock should be taken before calling this function
 107  *
 108  * Perform the following:
 109  * - Make sure we have enough space in the h/w queue
 110  * - Make sure we have enough space in the completion queue
 111  * - Reserve space in the completion queue (needs to be reversed if there
 112  *   is a failure down the road before the actual submission of work). Only
 113  *   do this action if reserve_cq_entry is true
 114  *
 115  */
 116 static int ext_queue_sanity_checks(struct hl_device *hdev,
 117                                 struct hl_hw_queue *q, int num_of_entries,
 118                                 bool reserve_cq_entry)
 119 {
 120         atomic_t *free_slots =
 121                         &hdev->completion_queue[q->cq_id].free_slots_cnt;
 122         int free_slots_cnt;
 123
 124         /* Check we have enough space in the queue */
 125         free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH);
 126
 127         if (free_slots_cnt < num_of_entries) {
 128                 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",
 129                         q->hw_queue_id, num_of_entries);
 130                 return -EAGAIN;
 131         }
 132
 133         if (reserve_cq_entry) {
 134                 /*
 135                  * Check we have enough space in the completion queue
 136                  * Add -1 to counter (decrement) unless counter was already 0
 137                  * In that case, CQ is full so we can't submit a new CB because
 138                  * we won't get ack on its completion
 139                  * atomic_add_unless will return 0 if counter was already 0
 140                  */
 141                 if (atomic_add_negative(num_of_entries * -1, free_slots)) {
 142                         dev_dbg(hdev->dev, "No space for %d on CQ %d\n",
 143                                 num_of_entries, q->hw_queue_id);
 144                         atomic_add(num_of_entries, free_slots);
 145                         return -EAGAIN;
 146                 }
 147         }
 148
 149         return 0;
 150 }
 151
 152 /*
 153  * int_queue_sanity_checks - perform some sanity checks on internal queue
 154  *
 155  * @hdev              : pointer to hl_device structure
 156  * @q                 : pointer to hl_hw_queue structure
 157  * @num_of_entries    : how many entries to check for space
 158  *
 159  * H/W queues spinlock should be taken before calling this function
 160  *
 161  * Perform the following:
 162  * - Make sure we have enough space in the h/w queue
 163  *
 164  */
 165 static int int_queue_sanity_checks(struct hl_device *hdev,
 166                                         struct hl_hw_queue *q,
 167                                         int num_of_entries)
 168 {
 169         int free_slots_cnt;
 170
 171         if (num_of_entries > q->int_queue_len) {
 172                 dev_err(hdev->dev,
 173                         "Cannot populate queue %u with %u jobs\n",
 174                         q->hw_queue_id, num_of_entries);
 175                 return -ENOMEM;
 176         }
 177
 178         /* Check we have enough space in the queue */
 179         free_slots_cnt = queue_free_slots(q, q->int_queue_len);
 180
 181         if (free_slots_cnt < num_of_entries) {
 182                 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",
 183                         q->hw_queue_id, num_of_entries);
 184                 return -EAGAIN;
 185         }
 186
 187         return 0;
 188 }
 189
 190 /*
 191  * hw_queue_sanity_checks() - Make sure we have enough space in the h/w queue
 192  * @hdev: Pointer to hl_device structure.
 193  * @q: Pointer to hl_hw_queue structure.
 194  * @num_of_entries: How many entries to check for space.
 195  *
 196  * Notice: We do not reserve queue entries so this function mustn't be called
 197  *         more than once per CS for the same queue
 198  *
 199  */
 200 static int hw_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q,
 201                                         int num_of_entries)
 202 {
 203         int free_slots_cnt;
 204
 205         /* Check we have enough space in the queue */
 206         free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH);
 207
 208         if (free_slots_cnt < num_of_entries) {
 209                 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",
 210                         q->hw_queue_id, num_of_entries);
 211                 return -EAGAIN;
 212         }
 213
 214         return 0;
 215 }
 216
 217 /*
 218  * hl_hw_queue_send_cb_no_cmpl - send a single CB (not a JOB) without completion
 219  *
 220  * @hdev: pointer to hl_device structure
 221  * @hw_queue_id: Queue's type
 222  * @cb_size: size of CB
 223  * @cb_ptr: pointer to CB location
 224  *
 225  * This function sends a single CB, that must NOT generate a completion entry
 226  *
 227  */
 228 int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
 229                                 u32 cb_size, u64 cb_ptr)
 230 {
 231         struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];
 232         int rc = 0;
 233
 234         /*
 235          * The CPU queue is a synchronous queue with an effective depth of
 236          * a single entry (although it is allocated with room for multiple
 237          * entries). Therefore, there is a different lock, called
 238          * send_cpu_message_lock, that serializes accesses to the CPU queue.
 239          * As a result, we don't need to lock the access to the entire H/W
 240          * queues module when submitting a JOB to the CPU queue
 241          */
 242         if (q->queue_type != QUEUE_TYPE_CPU)
 243                 hdev->asic_funcs->hw_queues_lock(hdev);
 244
 245         if (hdev->disabled) {
 246                 rc = -EPERM;
 247                 goto out;
 248         }
 249
 250         /*
 251          * hl_hw_queue_send_cb_no_cmpl() is called for queues of a H/W queue
 252          * type only on init phase, when the queues are empty and being tested,
 253          * so there is no need for sanity checks.
 254          */
 255         if (q->queue_type != QUEUE_TYPE_HW) {
 256                 rc = ext_queue_sanity_checks(hdev, q, 1, false);
 257                 if (rc)
 258                         goto out;
 259         }
 260
 261         ext_and_hw_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);
 262
 263 out:
 264         if (q->queue_type != QUEUE_TYPE_CPU)
 265                 hdev->asic_funcs->hw_queues_unlock(hdev);
 266
 267         return rc;
 268 }
 269
 270 /*
 271  * ext_queue_schedule_job - submit a JOB to an external queue
 272  *
 273  * @job: pointer to the job that needs to be submitted to the queue
 274  *
 275  * This function must be called when the scheduler mutex is taken
 276  *
 277  */
 278 static void ext_queue_schedule_job(struct hl_cs_job *job)
 279 {
 280         struct hl_device *hdev = job->cs->ctx->hdev;
 281         struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];
 282         struct hl_cq_entry cq_pkt;
 283         struct hl_cq *cq;
 284         u64 cq_addr;
 285         struct hl_cb *cb;
 286         u32 ctl;
 287         u32 len;
 288         u64 ptr;
 289
 290         /*
 291          * Update the JOB ID inside the BD CTL so the device would know what
 292          * to write in the completion queue
 293          */
 294         ctl = ((q->pi << BD_CTL_SHADOW_INDEX_SHIFT) & BD_CTL_SHADOW_INDEX_MASK);
 295
 296         cb = job->patched_cb;
 297         len = job->job_cb_size;
 298         ptr = cb->bus_address;
 299
 300         /* Skip completion flow in case this is a non completion CS */
 301         if (!cs_needs_completion(job->cs))
 302                 goto submit_bd;
 303
 304         cq_pkt.data = cpu_to_le32(
 305                         ((q->pi << CQ_ENTRY_SHADOW_INDEX_SHIFT)
 306                                 & CQ_ENTRY_SHADOW_INDEX_MASK) |
 307                         FIELD_PREP(CQ_ENTRY_SHADOW_INDEX_VALID_MASK, 1) |
 308                         FIELD_PREP(CQ_ENTRY_READY_MASK, 1));
 309
 310         /*
 311          * No need to protect pi_offset because scheduling to the
 312          * H/W queues is done under the scheduler mutex
 313          *
 314          * No need to check if CQ is full because it was already
 315          * checked in ext_queue_sanity_checks
 316          */
 317         cq = &hdev->completion_queue[q->cq_id];
 318         cq_addr = cq->bus_address + cq->pi * sizeof(struct hl_cq_entry);
 319
 320         hdev->asic_funcs->add_end_of_cb_packets(hdev, cb->kernel_address, len,
 321                                                 cq_addr,
 322                                                 le32_to_cpu(cq_pkt.data),
 323                                                 q->msi_vec,
 324                                                 job->contains_dma_pkt);
 325
 326         q->shadow_queue[hl_pi_2_offset(q->pi)] = job;
 327
 328         cq->pi = hl_cq_inc_ptr(cq->pi);
 329
 330 submit_bd:
 331         ext_and_hw_queue_submit_bd(hdev, q, ctl, len, ptr);
 332 }
 333
 334 /*
 335  * int_queue_schedule_job - submit a JOB to an internal queue
 336  *
 337  * @job: pointer to the job that needs to be submitted to the queue
 338  *
 339  * This function must be called when the scheduler mutex is taken
 340  *
 341  */
 342 static void int_queue_schedule_job(struct hl_cs_job *job)
 343 {
 344         struct hl_device *hdev = job->cs->ctx->hdev;
 345         struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];
 346         struct hl_bd bd;
 347         __le64 *pi;
 348
 349         bd.ctl = 0;
 350         bd.len = cpu_to_le32(job->job_cb_size);
 351
 352         if (job->is_kernel_allocated_cb)
 353                 /* bus_address is actually a mmu mapped address
 354                  * allocated from an internal pool
 355                  */
 356                 bd.ptr = cpu_to_le64(job->user_cb->bus_address);
 357         else
 358                 bd.ptr = cpu_to_le64((u64) (uintptr_t) job->user_cb);
 359
 360         pi = q->kernel_address + (q->pi & (q->int_queue_len - 1)) * sizeof(bd);
 361
 362         q->pi++;
 363         q->pi &= ((q->int_queue_len << 1) - 1);
 364
 365         hdev->asic_funcs->pqe_write(hdev, pi, &bd);
 366
 367         hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);
 368 }
 369
 370 /*
 371  * hw_queue_schedule_job - submit a JOB to a H/W queue
 372  *
 373  * @job: pointer to the job that needs to be submitted to the queue
 374  *
 375  * This function must be called when the scheduler mutex is taken
 376  *
 377  */
 378 static void hw_queue_schedule_job(struct hl_cs_job *job)
 379 {
 380         struct hl_device *hdev = job->cs->ctx->hdev;
 381         struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];
 382         u64 ptr;
 383         u32 offset, ctl, len;
 384
 385         /*
 386          * Upon PQE completion, COMP_DATA is used as the write data to the
 387          * completion queue (QMAN HBW message), and COMP_OFFSET is used as the
 388          * write address offset in the SM block (QMAN LBW message).
 389          * The write address offset is calculated as "COMP_OFFSET << 2".
 390          */
 391         offset = job->cs->sequence & (hdev->asic_prop.max_pending_cs - 1);
 392         ctl = ((offset << BD_CTL_COMP_OFFSET_SHIFT) & BD_CTL_COMP_OFFSET_MASK) |
 393                 ((q->pi << BD_CTL_COMP_DATA_SHIFT) & BD_CTL_COMP_DATA_MASK);
 394
 395         len = job->job_cb_size;
 396
 397         /*
 398          * A patched CB is created only if a user CB was allocated by driver and
 399          * MMU is disabled. If MMU is enabled, the user CB should be used
 400          * instead. If the user CB wasn't allocated by driver, assume that it
 401          * holds an address.
 402          */
 403         if (job->patched_cb)
 404                 ptr = job->patched_cb->bus_address;
 405         else if (job->is_kernel_allocated_cb)
 406                 ptr = job->user_cb->bus_address;
 407         else
 408                 ptr = (u64) (uintptr_t) job->user_cb;
 409
 410         ext_and_hw_queue_submit_bd(hdev, q, ctl, len, ptr);
 411 }
 412
 413 static void init_signal_cs(struct hl_device *hdev,
 414                 struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl)
 415 {
 416         struct hl_sync_stream_properties *prop;
 417         struct hl_hw_sob *hw_sob;
 418         u32 q_idx;
 419
 420         q_idx = job->hw_queue_id;
 421         prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
 422         hw_sob = &prop->hw_sob[prop->curr_sob_offset];
 423
 424         cs_cmpl->hw_sob = hw_sob;
 425         cs_cmpl->sob_val = prop->next_sob_val++;
 426
 427         dev_dbg(hdev->dev,
 428                 "generate signal CB, sob_id: %d, sob val: 0x%x, q_idx: %d\n",
 429                 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val, q_idx);
 430
 431         /* we set an EB since we must make sure all oeprations are done
 432          * when sending the signal
 433          */
 434         hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb,
 435                                 cs_cmpl->hw_sob->sob_id, 0, true);
 436
 437         kref_get(&hw_sob->kref);
 438
 439         /* check for wraparound */
 440         if (prop->next_sob_val == HL_MAX_SOB_VAL) {
 441                 /*
 442                  * Decrement as we reached the max value.
 443                  * The release function won't be called here as we've
 444                  * just incremented the refcount.
 445                  */
 446                 kref_put(&hw_sob->kref, hl_sob_reset_error);
 447                 prop->next_sob_val = 1;
 448                 /* only two SOBs are currently in use */
 449                 prop->curr_sob_offset =
 450                         (prop->curr_sob_offset + 1) % HL_RSVD_SOBS;
 451
 452                 dev_dbg(hdev->dev, "switched to SOB %d, q_idx: %d\n",
 453                                 prop->curr_sob_offset, q_idx);
 454         }
 455 }
 456
 457 static void init_wait_cs(struct hl_device *hdev, struct hl_cs *cs,
 458                 struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl)
 459 {
 460         struct hl_cs_compl *signal_cs_cmpl;
 461         struct hl_sync_stream_properties *prop;
 462         struct hl_gen_wait_properties wait_prop;
 463         u32 q_idx;
 464
 465         q_idx = job->hw_queue_id;
 466         prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
 467
 468         signal_cs_cmpl = container_of(cs->signal_fence,
 469                                         struct hl_cs_compl,
 470                                         base_fence);
 471
 472         /* copy the SOB id and value of the signal CS */
 473         cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;
 474         cs_cmpl->sob_val = signal_cs_cmpl->sob_val;
 475
 476         dev_dbg(hdev->dev,
 477                 "generate wait CB, sob_id: %d, sob_val: 0x%x, mon_id: %d, q_idx: %d\n",
 478                 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val,
 479                 prop->base_mon_id, q_idx);
 480
 481         wait_prop.data = (void *) job->patched_cb;
 482         wait_prop.sob_base = cs_cmpl->hw_sob->sob_id;
 483         wait_prop.sob_mask = 0x1;
 484         wait_prop.sob_val = cs_cmpl->sob_val;
 485         wait_prop.mon_id = prop->base_mon_id;
 486         wait_prop.q_idx = q_idx;
 487         wait_prop.size = 0;
 488         hdev->asic_funcs->gen_wait_cb(hdev, &wait_prop);
 489
 490         kref_get(&cs_cmpl->hw_sob->kref);
 491         /*
 492          * Must put the signal fence after the SOB refcnt increment so
 493          * the SOB refcnt won't turn 0 and reset the SOB before the
 494          * wait CS was submitted.
 495          */
 496         mb();
 497         hl_fence_put(cs->signal_fence);
 498         cs->signal_fence = NULL;
 499 }
 500
 501 /*
 502  * init_signal_wait_cs - initialize a signal/wait CS
 503  * @cs: pointer to the signal/wait CS
 504  *
 505  * H/W queues spinlock should be taken before calling this function
 506  */
 507 static void init_signal_wait_cs(struct hl_cs *cs)
 508 {
 509         struct hl_ctx *ctx = cs->ctx;
 510         struct hl_device *hdev = ctx->hdev;
 511         struct hl_cs_job *job;
 512         struct hl_cs_compl *cs_cmpl =
 513                         container_of(cs->fence, struct hl_cs_compl, base_fence);
 514
 515         /* There is only one job in a signal/wait CS */
 516         job = list_first_entry(&cs->job_list, struct hl_cs_job,
 517                                 cs_node);
 518
 519         if (cs->type & CS_TYPE_SIGNAL)
 520                 init_signal_cs(hdev, job, cs_cmpl);
 521         else if (cs->type & CS_TYPE_WAIT)
 522                 init_wait_cs(hdev, cs, job, cs_cmpl);
 523 }
 524
 525 /*
 526  * hl_hw_queue_schedule_cs - schedule a command submission
 527  * @cs: pointer to the CS
 528  */
 529 int hl_hw_queue_schedule_cs(struct hl_cs *cs)
 530 {
 531         enum hl_device_status status;
 532         struct hl_cs_counters_atomic *cntr;
 533         struct hl_ctx *ctx = cs->ctx;
 534         struct hl_device *hdev = ctx->hdev;
 535         struct hl_cs_job *job, *tmp;
 536         struct hl_hw_queue *q;
 537         int rc = 0, i, cq_cnt;
 538         bool first_entry;
 539         u32 max_queues;
 540
 541         cntr = &hdev->aggregated_cs_counters;
 542
 543         hdev->asic_funcs->hw_queues_lock(hdev);
 544
 545         if (!hl_device_operational(hdev, &status)) {
 546                 atomic64_inc(&cntr->device_in_reset_drop_cnt);
 547                 atomic64_inc(&ctx->cs_counters.device_in_reset_drop_cnt);
 548                 dev_err(hdev->dev,
 549                         "device is %s, CS rejected!\n", hdev->status[status]);
 550                 rc = -EPERM;
 551                 goto out;
 552         }
 553
 554         max_queues = hdev->asic_prop.max_queues;
 555
 556         q = &hdev->kernel_queues[0];
 557         for (i = 0, cq_cnt = 0 ; i < max_queues ; i++, q++) {
 558                 if (cs->jobs_in_queue_cnt[i]) {
 559                         switch (q->queue_type) {
 560                         case QUEUE_TYPE_EXT:
 561                                 rc = ext_queue_sanity_checks(hdev, q,
 562                                                 cs->jobs_in_queue_cnt[i],
 563                                                 cs_needs_completion(cs) ?
 564                                                                 true : false);
 565                                 break;
 566                         case QUEUE_TYPE_INT:
 567                                 rc = int_queue_sanity_checks(hdev, q,
 568                                                 cs->jobs_in_queue_cnt[i]);
 569                                 break;
 570                         case QUEUE_TYPE_HW:
 571                                 rc = hw_queue_sanity_checks(hdev, q,
 572                                                 cs->jobs_in_queue_cnt[i]);
 573                                 break;
 574                         default:
 575                                 dev_err(hdev->dev, "Queue type %d is invalid\n",
 576                                         q->queue_type);
 577                                 rc = -EINVAL;
 578                                 break;
 579                         }
 580
 581                         if (rc) {
 582                                 atomic64_inc(
 583                                         &ctx->cs_counters.queue_full_drop_cnt);
 584                                 atomic64_inc(&cntr->queue_full_drop_cnt);
 585                                 goto unroll_cq_resv;
 586                         }
 587
 588                         if (q->queue_type == QUEUE_TYPE_EXT)
 589                                 cq_cnt++;
 590                 }
 591         }
 592
 593         if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT))
 594                 init_signal_wait_cs(cs);
 595         else if (cs->type == CS_TYPE_COLLECTIVE_WAIT)
 596                 hdev->asic_funcs->collective_wait_init_cs(cs);
 597
 598         spin_lock(&hdev->cs_mirror_lock);
 599
 600         /* Verify staged CS exists and add to the staged list */
 601         if (cs->staged_cs && !cs->staged_first) {
 602                 struct hl_cs *staged_cs;
 603
 604                 staged_cs = hl_staged_cs_find_first(hdev, cs->staged_sequence);
 605                 if (!staged_cs) {
 606                         dev_err(hdev->dev,
 607                                 "Cannot find staged submission sequence %llu",
 608                                 cs->staged_sequence);
 609                         rc = -EINVAL;
 610                         goto unlock_cs_mirror;
 611                 }
 612
 613                 if (is_staged_cs_last_exists(hdev, staged_cs)) {
 614                         dev_err(hdev->dev,
 615                                 "Staged submission sequence %llu already submitted",
 616                                 cs->staged_sequence);
 617                         rc = -EINVAL;
 618                         goto unlock_cs_mirror;
 619                 }
 620
 621                 list_add_tail(&cs->staged_cs_node, &staged_cs->staged_cs_node);
 622         }
 623
 624         list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list);
 625
 626         /* Queue TDR if the CS is the first entry and if timeout is wanted */
 627         first_entry = list_first_entry(&hdev->cs_mirror_list,
 628                                         struct hl_cs, mirror_node) == cs;
 629         if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) &&
 630                                 first_entry && cs_needs_timeout(cs)) {
 631                 cs->tdr_active = true;
 632                 schedule_delayed_work(&cs->work_tdr, cs->timeout_jiffies);
 633
 634         }
 635
 636         spin_unlock(&hdev->cs_mirror_lock);
 637
 638         list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)
 639                 switch (job->queue_type) {
 640                 case QUEUE_TYPE_EXT:
 641                         ext_queue_schedule_job(job);
 642                         break;
 643                 case QUEUE_TYPE_INT:
 644                         int_queue_schedule_job(job);
 645                         break;
 646                 case QUEUE_TYPE_HW:
 647                         hw_queue_schedule_job(job);
 648                         break;
 649                 default:
 650                         break;
 651                 }
 652
 653         cs->submitted = true;
 654
 655         goto out;
 656
 657 unlock_cs_mirror:
 658         spin_unlock(&hdev->cs_mirror_lock);
 659 unroll_cq_resv:
 660         q = &hdev->kernel_queues[0];
 661         for (i = 0 ; (i < max_queues) && (cq_cnt > 0) ; i++, q++) {
 662                 if ((q->queue_type == QUEUE_TYPE_EXT) &&
 663                                                 (cs->jobs_in_queue_cnt[i])) {
 664                         atomic_t *free_slots =
 665                                 &hdev->completion_queue[i].free_slots_cnt;
 666                         atomic_add(cs->jobs_in_queue_cnt[i], free_slots);
 667                         cq_cnt--;
 668                 }
 669         }
 670
 671 out:
 672         hdev->asic_funcs->hw_queues_unlock(hdev);
 673
 674         return rc;
 675 }
 676
 677 /*
 678  * hl_hw_queue_inc_ci_kernel - increment ci for kernel's queue
 679  *
 680  * @hdev: pointer to hl_device structure
 681  * @hw_queue_id: which queue to increment its ci
 682  */
 683 void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id)
 684 {
 685         struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];
 686
 687         atomic_inc(&q->ci);
 688 }
 689
 690 static int ext_and_cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q,
 691                                         bool is_cpu_queue)
 692 {
 693         void *p;
 694         int rc;
 695
 696         if (is_cpu_queue)
 697                 p = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
 698                                                         HL_QUEUE_SIZE_IN_BYTES,
 699                                                         &q->bus_address);
 700         else
 701                 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
 702                                                 HL_QUEUE_SIZE_IN_BYTES,
 703                                                 &q->bus_address,
 704                                                 GFP_KERNEL | __GFP_ZERO);
 705         if (!p)
 706                 return -ENOMEM;
 707
 708         q->kernel_address = p;
 709
 710         q->shadow_queue = kmalloc_array(HL_QUEUE_LENGTH,
 711                                         sizeof(*q->shadow_queue),
 712                                         GFP_KERNEL);
 713         if (!q->shadow_queue) {
 714                 dev_err(hdev->dev,
 715                         "Failed to allocate shadow queue for H/W queue %d\n",
 716                         q->hw_queue_id);
 717                 rc = -ENOMEM;
 718                 goto free_queue;
 719         }
 720
 721         /* Make sure read/write pointers are initialized to start of queue */
 722         atomic_set(&q->ci, 0);
 723         q->pi = 0;
 724
 725         return 0;
 726
 727 free_queue:
 728         if (is_cpu_queue)
 729                 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
 730                                         HL_QUEUE_SIZE_IN_BYTES,
 731                                         q->kernel_address);
 732         else
 733                 hdev->asic_funcs->asic_dma_free_coherent(hdev,
 734                                         HL_QUEUE_SIZE_IN_BYTES,
 735                                         q->kernel_address,
 736                                         q->bus_address);
 737
 738         return rc;
 739 }
 740
 741 static int int_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
 742 {
 743         void *p;
 744
 745         p = hdev->asic_funcs->get_int_queue_base(hdev, q->hw_queue_id,
 746                                         &q->bus_address, &q->int_queue_len);
 747         if (!p) {
 748                 dev_err(hdev->dev,
 749                         "Failed to get base address for internal queue %d\n",
 750                         q->hw_queue_id);
 751                 return -EFAULT;
 752         }
 753
 754         q->kernel_address = p;
 755         q->pi = 0;
 756         atomic_set(&q->ci, 0);
 757
 758         return 0;
 759 }
 760
 761 static int cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
 762 {
 763         return ext_and_cpu_queue_init(hdev, q, true);
 764 }
 765
 766 static int ext_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
 767 {
 768         return ext_and_cpu_queue_init(hdev, q, false);
 769 }
 770
 771 static int hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
 772 {
 773         void *p;
 774
 775         p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
 776                                                 HL_QUEUE_SIZE_IN_BYTES,
 777                                                 &q->bus_address,
 778                                                 GFP_KERNEL | __GFP_ZERO);
 779         if (!p)
 780                 return -ENOMEM;
 781
 782         q->kernel_address = p;
 783
 784         /* Make sure read/write pointers are initialized to start of queue */
 785         atomic_set(&q->ci, 0);
 786         q->pi = 0;
 787
 788         return 0;
 789 }
 790
 791 static void sync_stream_queue_init(struct hl_device *hdev, u32 q_idx)
 792 {
 793         struct hl_sync_stream_properties *sync_stream_prop;
 794         struct asic_fixed_properties *prop = &hdev->asic_prop;
 795         struct hl_hw_sob *hw_sob;
 796         int sob, reserved_mon_idx, queue_idx;
 797
 798         sync_stream_prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
 799
 800         /* We use 'collective_mon_idx' as a running index in order to reserve
 801          * monitors for collective master/slave queues.
 802          * collective master queue gets 2 reserved monitors
 803          * collective slave queue gets 1 reserved monitor
 804          */
 805         if (hdev->kernel_queues[q_idx].collective_mode ==
 806                         HL_COLLECTIVE_MASTER) {
 807                 reserved_mon_idx = hdev->collective_mon_idx;
 808
 809                 /* reserve the first monitor for collective master queue */
 810                 sync_stream_prop->collective_mstr_mon_id[0] =
 811                         prop->collective_first_mon + reserved_mon_idx;
 812
 813                 /* reserve the second monitor for collective master queue */
 814                 sync_stream_prop->collective_mstr_mon_id[1] =
 815                         prop->collective_first_mon + reserved_mon_idx + 1;
 816
 817                 hdev->collective_mon_idx += HL_COLLECTIVE_RSVD_MSTR_MONS;
 818         } else if (hdev->kernel_queues[q_idx].collective_mode ==
 819                         HL_COLLECTIVE_SLAVE) {
 820                 reserved_mon_idx = hdev->collective_mon_idx++;
 821
 822                 /* reserve a monitor for collective slave queue */
 823                 sync_stream_prop->collective_slave_mon_id =
 824                         prop->collective_first_mon + reserved_mon_idx;
 825         }
 826
 827         if (!hdev->kernel_queues[q_idx].supports_sync_stream)
 828                 return;
 829
 830         queue_idx = hdev->sync_stream_queue_idx++;
 831
 832         sync_stream_prop->base_sob_id = prop->sync_stream_first_sob +
 833                         (queue_idx * HL_RSVD_SOBS);
 834         sync_stream_prop->base_mon_id = prop->sync_stream_first_mon +
 835                         (queue_idx * HL_RSVD_MONS);
 836         sync_stream_prop->next_sob_val = 1;
 837         sync_stream_prop->curr_sob_offset = 0;
 838
 839         for (sob = 0 ; sob < HL_RSVD_SOBS ; sob++) {
 840                 hw_sob = &sync_stream_prop->hw_sob[sob];
 841                 hw_sob->hdev = hdev;
 842                 hw_sob->sob_id = sync_stream_prop->base_sob_id + sob;
 843                 hw_sob->q_idx = q_idx;
 844                 kref_init(&hw_sob->kref);
 845         }
 846 }
 847
 848 static void sync_stream_queue_reset(struct hl_device *hdev, u32 q_idx)
 849 {
 850         struct hl_sync_stream_properties *prop =
 851                         &hdev->kernel_queues[q_idx].sync_stream_prop;
 852
 853         /*
 854          * In case we got here due to a stuck CS, the refcnt might be bigger
 855          * than 1 and therefore we reset it.
 856          */
 857         kref_init(&prop->hw_sob[prop->curr_sob_offset].kref);
 858         prop->curr_sob_offset = 0;
 859         prop->next_sob_val = 1;
 860 }
 861
 862 /*
 863  * queue_init - main initialization function for H/W queue object
 864  *
 865  * @hdev: pointer to hl_device device structure
 866  * @q: pointer to hl_hw_queue queue structure
 867  * @hw_queue_id: The id of the H/W queue
 868  *
 869  * Allocate dma-able memory for the queue and initialize fields
 870  * Returns 0 on success
 871  */
 872 static int queue_init(struct hl_device *hdev, struct hl_hw_queue *q,
 873                         u32 hw_queue_id)
 874 {
 875         int rc;
 876
 877         q->hw_queue_id = hw_queue_id;
 878
 879         switch (q->queue_type) {
 880         case QUEUE_TYPE_EXT:
 881                 rc = ext_queue_init(hdev, q);
 882                 break;
 883         case QUEUE_TYPE_INT:
 884                 rc = int_queue_init(hdev, q);
 885                 break;
 886         case QUEUE_TYPE_CPU:
 887                 rc = cpu_queue_init(hdev, q);
 888                 break;
 889         case QUEUE_TYPE_HW:
 890                 rc = hw_queue_init(hdev, q);
 891                 break;
 892         case QUEUE_TYPE_NA:
 893                 q->valid = 0;
 894                 return 0;
 895         default:
 896                 dev_crit(hdev->dev, "wrong queue type %d during init\n",
 897                         q->queue_type);
 898                 rc = -EINVAL;
 899                 break;
 900         }
 901
 902         sync_stream_queue_init(hdev, q->hw_queue_id);
 903
 904         if (rc)
 905                 return rc;
 906
 907         q->valid = 1;
 908
 909         return 0;
 910 }
 911
 912 /*
 913  * hw_queue_fini - destroy queue
 914  *
 915  * @hdev: pointer to hl_device device structure
 916  * @q: pointer to hl_hw_queue queue structure
 917  *
 918  * Free the queue memory
 919  */
 920 static void queue_fini(struct hl_device *hdev, struct hl_hw_queue *q)
 921 {
 922         if (!q->valid)
 923                 return;
 924
 925         /*
 926          * If we arrived here, there are no jobs waiting on this queue
 927          * so we can safely remove it.
 928          * This is because this function can only called when:
 929          * 1. Either a context is deleted, which only can occur if all its
 930          *    jobs were finished
 931          * 2. A context wasn't able to be created due to failure or timeout,
 932          *    which means there are no jobs on the queue yet
 933          *
 934          * The only exception are the queues of the kernel context, but
 935          * if they are being destroyed, it means that the entire module is
 936          * being removed. If the module is removed, it means there is no open
 937          * user context. It also means that if a job was submitted by
 938          * the kernel driver (e.g. context creation), the job itself was
 939          * released by the kernel driver when a timeout occurred on its
 940          * Completion. Thus, we don't need to release it again.
 941          */
 942
 943         if (q->queue_type == QUEUE_TYPE_INT)
 944                 return;
 945
 946         kfree(q->shadow_queue);
 947
 948         if (q->queue_type == QUEUE_TYPE_CPU)
 949                 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
 950                                         HL_QUEUE_SIZE_IN_BYTES,
 951                                         q->kernel_address);
 952         else
 953                 hdev->asic_funcs->asic_dma_free_coherent(hdev,
 954                                         HL_QUEUE_SIZE_IN_BYTES,
 955                                         q->kernel_address,
 956                                         q->bus_address);
 957 }
 958
 959 int hl_hw_queues_create(struct hl_device *hdev)
 960 {
 961         struct asic_fixed_properties *asic = &hdev->asic_prop;
 962         struct hl_hw_queue *q;
 963         int i, rc, q_ready_cnt;
 964
 965         hdev->kernel_queues = kcalloc(asic->max_queues,
 966                                 sizeof(*hdev->kernel_queues), GFP_KERNEL);
 967
 968         if (!hdev->kernel_queues) {
 969                 dev_err(hdev->dev, "Not enough memory for H/W queues\n");
 970                 return -ENOMEM;
 971         }
 972
 973         /* Initialize the H/W queues */
 974         for (i = 0, q_ready_cnt = 0, q = hdev->kernel_queues;
 975                         i < asic->max_queues ; i++, q_ready_cnt++, q++) {
 976
 977                 q->queue_type = asic->hw_queues_props[i].type;
 978                 q->supports_sync_stream =
 979                                 asic->hw_queues_props[i].supports_sync_stream;
 980                 q->collective_mode = asic->hw_queues_props[i].collective_mode;
 981                 rc = queue_init(hdev, q, i);
 982                 if (rc) {
 983                         dev_err(hdev->dev,
 984                                 "failed to initialize queue %d\n", i);
 985                         goto release_queues;
 986                 }
 987         }
 988
 989         return 0;
 990
 991 release_queues:
 992         for (i = 0, q = hdev->kernel_queues ; i < q_ready_cnt ; i++, q++)
 993                 queue_fini(hdev, q);
 994
 995         kfree(hdev->kernel_queues);
 996
 997         return rc;
 998 }
 999
1000 void hl_hw_queues_destroy(struct hl_device *hdev)
1001 {
1002         struct hl_hw_queue *q;
1003         u32 max_queues = hdev->asic_prop.max_queues;
1004         int i;
1005
1006         for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++)
1007                 queue_fini(hdev, q);
1008
1009         kfree(hdev->kernel_queues);
1010 }
1011
1012 void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset)
1013 {
1014         struct hl_hw_queue *q;
1015         u32 max_queues = hdev->asic_prop.max_queues;
1016         int i;
1017
1018         for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++) {
1019                 if ((!q->valid) ||
1020                         ((!hard_reset) && (q->queue_type == QUEUE_TYPE_CPU)))
1021                         continue;
1022                 q->pi = 0;
1023                 atomic_set(&q->ci, 0);
1024
1025                 if (q->supports_sync_stream)
1026                         sync_stream_queue_reset(hdev, q->hw_queue_id);
1027         }
1028 }