drivers/net/ethernet/pensando/ionic/ionic_dev.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/types.h>
   6 #include <linux/errno.h>
   7 #include <linux/io.h>
   8 #include <linux/slab.h>
   9 #include <linux/etherdevice.h>
  10 #include "ionic.h"
  11 #include "ionic_dev.h"
  12 #include "ionic_lif.h"
  13
  14 static void ionic_watchdog_cb(struct timer_list *t)
  15 {
  16         struct ionic *ionic = from_timer(ionic, t, watchdog_timer);
  17         int hb;
  18
  19         mod_timer(&ionic->watchdog_timer,
  20                   round_jiffies(jiffies + ionic->watchdog_period));
  21
  22         if (!ionic->lif)
  23                 return;
  24
  25         hb = ionic_heartbeat_check(ionic);
  26
  27         if (hb >= 0)
  28                 ionic_link_status_check_request(ionic->lif, false);
  29 }
  30
  31 void ionic_init_devinfo(struct ionic *ionic)
  32 {
  33         struct ionic_dev *idev = &ionic->idev;
  34
  35         idev->dev_info.asic_type = ioread8(&idev->dev_info_regs->asic_type);
  36         idev->dev_info.asic_rev = ioread8(&idev->dev_info_regs->asic_rev);
  37
  38         memcpy_fromio(idev->dev_info.fw_version,
  39                       idev->dev_info_regs->fw_version,
  40                       IONIC_DEVINFO_FWVERS_BUFLEN);
  41
  42         memcpy_fromio(idev->dev_info.serial_num,
  43                       idev->dev_info_regs->serial_num,
  44                       IONIC_DEVINFO_SERIAL_BUFLEN);
  45
  46         idev->dev_info.fw_version[IONIC_DEVINFO_FWVERS_BUFLEN] = 0;
  47         idev->dev_info.serial_num[IONIC_DEVINFO_SERIAL_BUFLEN] = 0;
  48
  49         dev_dbg(ionic->dev, "fw_version %s\n", idev->dev_info.fw_version);
  50 }
  51
  52 int ionic_dev_setup(struct ionic *ionic)
  53 {
  54         struct ionic_dev_bar *bar = ionic->bars;
  55         unsigned int num_bars = ionic->num_bars;
  56         struct ionic_dev *idev = &ionic->idev;
  57         struct device *dev = ionic->dev;
  58         u32 sig;
  59
  60         /* BAR0: dev_cmd and interrupts */
  61         if (num_bars < 1) {
  62                 dev_err(dev, "No bars found, aborting\n");
  63                 return -EFAULT;
  64         }
  65
  66         if (bar->len < IONIC_BAR0_SIZE) {
  67                 dev_err(dev, "Resource bar size %lu too small, aborting\n",
  68                         bar->len);
  69                 return -EFAULT;
  70         }
  71
  72         idev->dev_info_regs = bar->vaddr + IONIC_BAR0_DEV_INFO_REGS_OFFSET;
  73         idev->dev_cmd_regs = bar->vaddr + IONIC_BAR0_DEV_CMD_REGS_OFFSET;
  74         idev->intr_status = bar->vaddr + IONIC_BAR0_INTR_STATUS_OFFSET;
  75         idev->intr_ctrl = bar->vaddr + IONIC_BAR0_INTR_CTRL_OFFSET;
  76
  77         sig = ioread32(&idev->dev_info_regs->signature);
  78         if (sig != IONIC_DEV_INFO_SIGNATURE) {
  79                 dev_err(dev, "Incompatible firmware signature %x", sig);
  80                 return -EFAULT;
  81         }
  82
  83         ionic_init_devinfo(ionic);
  84
  85         /* BAR1: doorbells */
  86         bar++;
  87         if (num_bars < 2) {
  88                 dev_err(dev, "Doorbell bar missing, aborting\n");
  89                 return -EFAULT;
  90         }
  91
  92         idev->last_fw_status = 0xff;
  93         timer_setup(&ionic->watchdog_timer, ionic_watchdog_cb, 0);
  94         ionic->watchdog_period = IONIC_WATCHDOG_SECS * HZ;
  95         mod_timer(&ionic->watchdog_timer,
  96                   round_jiffies(jiffies + ionic->watchdog_period));
  97
  98         idev->db_pages = bar->vaddr;
  99         idev->phy_db_pages = bar->bus_addr;
 100
 101         return 0;
 102 }
 103
 104 /* Devcmd Interface */
 105 int ionic_heartbeat_check(struct ionic *ionic)
 106 {
 107         struct ionic_dev *idev = &ionic->idev;
 108         unsigned long hb_time;
 109         u8 fw_status;
 110         u32 hb;
 111
 112         /* wait a little more than one second before testing again */
 113         hb_time = jiffies;
 114         if (time_before(hb_time, (idev->last_hb_time + ionic->watchdog_period)))
 115                 return 0;
 116
 117         /* firmware is useful only if the running bit is set and
 118          * fw_status != 0xff (bad PCI read)
 119          */
 120         fw_status = ioread8(&idev->dev_info_regs->fw_status);
 121         if (fw_status != 0xff)
 122                 fw_status &= IONIC_FW_STS_F_RUNNING;  /* use only the run bit */
 123
 124         /* is this a transition? */
 125         if (fw_status != idev->last_fw_status &&
 126             idev->last_fw_status != 0xff) {
 127                 struct ionic_lif *lif = ionic->lif;
 128                 bool trigger = false;
 129
 130                 if (!fw_status || fw_status == 0xff) {
 131                         dev_info(ionic->dev, "FW stopped %u\n", fw_status);
 132                         if (lif && !test_bit(IONIC_LIF_F_FW_RESET, lif->state))
 133                                 trigger = true;
 134                 } else {
 135                         dev_info(ionic->dev, "FW running %u\n", fw_status);
 136                         if (lif && test_bit(IONIC_LIF_F_FW_RESET, lif->state))
 137                                 trigger = true;
 138                 }
 139
 140                 if (trigger) {
 141                         struct ionic_deferred_work *work;
 142
 143                         work = kzalloc(sizeof(*work), GFP_ATOMIC);
 144                         if (!work) {
 145                                 dev_err(ionic->dev, "%s OOM\n", __func__);
 146                         } else {
 147                                 work->type = IONIC_DW_TYPE_LIF_RESET;
 148                                 if (fw_status & IONIC_FW_STS_F_RUNNING &&
 149                                     fw_status != 0xff)
 150                                         work->fw_status = 1;
 151                                 ionic_lif_deferred_enqueue(&lif->deferred, work);
 152                         }
 153                 }
 154         }
 155         idev->last_fw_status = fw_status;
 156
 157         if (!fw_status || fw_status == 0xff)
 158                 return -ENXIO;
 159
 160         /* early FW has no heartbeat, else FW will return non-zero */
 161         hb = ioread32(&idev->dev_info_regs->fw_heartbeat);
 162         if (!hb)
 163                 return 0;
 164
 165         /* are we stalled? */
 166         if (hb == idev->last_hb) {
 167                 /* only complain once for each stall seen */
 168                 if (idev->last_hb_time != 1) {
 169                         dev_info(ionic->dev, "FW heartbeat stalled at %d\n",
 170                                  idev->last_hb);
 171                         idev->last_hb_time = 1;
 172                 }
 173
 174                 return -ENXIO;
 175         }
 176
 177         if (idev->last_hb_time == 1)
 178                 dev_info(ionic->dev, "FW heartbeat restored at %d\n", hb);
 179
 180         idev->last_hb = hb;
 181         idev->last_hb_time = hb_time;
 182
 183         return 0;
 184 }
 185
 186 u8 ionic_dev_cmd_status(struct ionic_dev *idev)
 187 {
 188         return ioread8(&idev->dev_cmd_regs->comp.comp.status);
 189 }
 190
 191 bool ionic_dev_cmd_done(struct ionic_dev *idev)
 192 {
 193         return ioread32(&idev->dev_cmd_regs->done) & IONIC_DEV_CMD_DONE;
 194 }
 195
 196 void ionic_dev_cmd_comp(struct ionic_dev *idev, union ionic_dev_cmd_comp *comp)
 197 {
 198         memcpy_fromio(comp, &idev->dev_cmd_regs->comp, sizeof(*comp));
 199 }
 200
 201 void ionic_dev_cmd_go(struct ionic_dev *idev, union ionic_dev_cmd *cmd)
 202 {
 203         memcpy_toio(&idev->dev_cmd_regs->cmd, cmd, sizeof(*cmd));
 204         iowrite32(0, &idev->dev_cmd_regs->done);
 205         iowrite32(1, &idev->dev_cmd_regs->doorbell);
 206 }
 207
 208 /* Device commands */
 209 void ionic_dev_cmd_identify(struct ionic_dev *idev, u8 ver)
 210 {
 211         union ionic_dev_cmd cmd = {
 212                 .identify.opcode = IONIC_CMD_IDENTIFY,
 213                 .identify.ver = ver,
 214         };
 215
 216         ionic_dev_cmd_go(idev, &cmd);
 217 }
 218
 219 void ionic_dev_cmd_init(struct ionic_dev *idev)
 220 {
 221         union ionic_dev_cmd cmd = {
 222                 .init.opcode = IONIC_CMD_INIT,
 223                 .init.type = 0,
 224         };
 225
 226         ionic_dev_cmd_go(idev, &cmd);
 227 }
 228
 229 void ionic_dev_cmd_reset(struct ionic_dev *idev)
 230 {
 231         union ionic_dev_cmd cmd = {
 232                 .reset.opcode = IONIC_CMD_RESET,
 233         };
 234
 235         ionic_dev_cmd_go(idev, &cmd);
 236 }
 237
 238 /* Port commands */
 239 void ionic_dev_cmd_port_identify(struct ionic_dev *idev)
 240 {
 241         union ionic_dev_cmd cmd = {
 242                 .port_init.opcode = IONIC_CMD_PORT_IDENTIFY,
 243                 .port_init.index = 0,
 244         };
 245
 246         ionic_dev_cmd_go(idev, &cmd);
 247 }
 248
 249 void ionic_dev_cmd_port_init(struct ionic_dev *idev)
 250 {
 251         union ionic_dev_cmd cmd = {
 252                 .port_init.opcode = IONIC_CMD_PORT_INIT,
 253                 .port_init.index = 0,
 254                 .port_init.info_pa = cpu_to_le64(idev->port_info_pa),
 255         };
 256
 257         ionic_dev_cmd_go(idev, &cmd);
 258 }
 259
 260 void ionic_dev_cmd_port_reset(struct ionic_dev *idev)
 261 {
 262         union ionic_dev_cmd cmd = {
 263                 .port_reset.opcode = IONIC_CMD_PORT_RESET,
 264                 .port_reset.index = 0,
 265         };
 266
 267         ionic_dev_cmd_go(idev, &cmd);
 268 }
 269
 270 void ionic_dev_cmd_port_state(struct ionic_dev *idev, u8 state)
 271 {
 272         union ionic_dev_cmd cmd = {
 273                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 274                 .port_setattr.index = 0,
 275                 .port_setattr.attr = IONIC_PORT_ATTR_STATE,
 276                 .port_setattr.state = state,
 277         };
 278
 279         ionic_dev_cmd_go(idev, &cmd);
 280 }
 281
 282 void ionic_dev_cmd_port_speed(struct ionic_dev *idev, u32 speed)
 283 {
 284         union ionic_dev_cmd cmd = {
 285                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 286                 .port_setattr.index = 0,
 287                 .port_setattr.attr = IONIC_PORT_ATTR_SPEED,
 288                 .port_setattr.speed = cpu_to_le32(speed),
 289         };
 290
 291         ionic_dev_cmd_go(idev, &cmd);
 292 }
 293
 294 void ionic_dev_cmd_port_autoneg(struct ionic_dev *idev, u8 an_enable)
 295 {
 296         union ionic_dev_cmd cmd = {
 297                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 298                 .port_setattr.index = 0,
 299                 .port_setattr.attr = IONIC_PORT_ATTR_AUTONEG,
 300                 .port_setattr.an_enable = an_enable,
 301         };
 302
 303         ionic_dev_cmd_go(idev, &cmd);
 304 }
 305
 306 void ionic_dev_cmd_port_fec(struct ionic_dev *idev, u8 fec_type)
 307 {
 308         union ionic_dev_cmd cmd = {
 309                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 310                 .port_setattr.index = 0,
 311                 .port_setattr.attr = IONIC_PORT_ATTR_FEC,
 312                 .port_setattr.fec_type = fec_type,
 313         };
 314
 315         ionic_dev_cmd_go(idev, &cmd);
 316 }
 317
 318 void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type)
 319 {
 320         union ionic_dev_cmd cmd = {
 321                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 322                 .port_setattr.index = 0,
 323                 .port_setattr.attr = IONIC_PORT_ATTR_PAUSE,
 324                 .port_setattr.pause_type = pause_type,
 325         };
 326
 327         ionic_dev_cmd_go(idev, &cmd);
 328 }
 329
 330 /* VF commands */
 331 int ionic_set_vf_config(struct ionic *ionic, int vf, u8 attr, u8 *data)
 332 {
 333         union ionic_dev_cmd cmd = {
 334                 .vf_setattr.opcode = IONIC_CMD_VF_SETATTR,
 335                 .vf_setattr.attr = attr,
 336                 .vf_setattr.vf_index = cpu_to_le16(vf),
 337         };
 338         int err;
 339
 340         switch (attr) {
 341         case IONIC_VF_ATTR_SPOOFCHK:
 342                 cmd.vf_setattr.spoofchk = *data;
 343                 dev_dbg(ionic->dev, "%s: vf %d spoof %d\n",
 344                         __func__, vf, *data);
 345                 break;
 346         case IONIC_VF_ATTR_TRUST:
 347                 cmd.vf_setattr.trust = *data;
 348                 dev_dbg(ionic->dev, "%s: vf %d trust %d\n",
 349                         __func__, vf, *data);
 350                 break;
 351         case IONIC_VF_ATTR_LINKSTATE:
 352                 cmd.vf_setattr.linkstate = *data;
 353                 dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n",
 354                         __func__, vf, *data);
 355                 break;
 356         case IONIC_VF_ATTR_MAC:
 357                 ether_addr_copy(cmd.vf_setattr.macaddr, data);
 358                 dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n",
 359                         __func__, vf, data);
 360                 break;
 361         case IONIC_VF_ATTR_VLAN:
 362                 cmd.vf_setattr.vlanid = cpu_to_le16(*(u16 *)data);
 363                 dev_dbg(ionic->dev, "%s: vf %d vlan %d\n",
 364                         __func__, vf, *(u16 *)data);
 365                 break;
 366         case IONIC_VF_ATTR_RATE:
 367                 cmd.vf_setattr.maxrate = cpu_to_le32(*(u32 *)data);
 368                 dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n",
 369                         __func__, vf, *(u32 *)data);
 370                 break;
 371         case IONIC_VF_ATTR_STATSADDR:
 372                 cmd.vf_setattr.stats_pa = cpu_to_le64(*(u64 *)data);
 373                 dev_dbg(ionic->dev, "%s: vf %d stats_pa 0x%08llx\n",
 374                         __func__, vf, *(u64 *)data);
 375                 break;
 376         default:
 377                 return -EINVAL;
 378         }
 379
 380         mutex_lock(&ionic->dev_cmd_lock);
 381         ionic_dev_cmd_go(&ionic->idev, &cmd);
 382         err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
 383         mutex_unlock(&ionic->dev_cmd_lock);
 384
 385         return err;
 386 }
 387
 388 /* LIF commands */
 389 void ionic_dev_cmd_queue_identify(struct ionic_dev *idev,
 390                                   u16 lif_type, u8 qtype, u8 qver)
 391 {
 392         union ionic_dev_cmd cmd = {
 393                 .q_identify.opcode = IONIC_CMD_Q_IDENTIFY,
 394                 .q_identify.lif_type = cpu_to_le16(lif_type),
 395                 .q_identify.type = qtype,
 396                 .q_identify.ver = qver,
 397         };
 398
 399         ionic_dev_cmd_go(idev, &cmd);
 400 }
 401
 402 void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, u8 type, u8 ver)
 403 {
 404         union ionic_dev_cmd cmd = {
 405                 .lif_identify.opcode = IONIC_CMD_LIF_IDENTIFY,
 406                 .lif_identify.type = type,
 407                 .lif_identify.ver = ver,
 408         };
 409
 410         ionic_dev_cmd_go(idev, &cmd);
 411 }
 412
 413 void ionic_dev_cmd_lif_init(struct ionic_dev *idev, u16 lif_index,
 414                             dma_addr_t info_pa)
 415 {
 416         union ionic_dev_cmd cmd = {
 417                 .lif_init.opcode = IONIC_CMD_LIF_INIT,
 418                 .lif_init.index = cpu_to_le16(lif_index),
 419                 .lif_init.info_pa = cpu_to_le64(info_pa),
 420         };
 421
 422         ionic_dev_cmd_go(idev, &cmd);
 423 }
 424
 425 void ionic_dev_cmd_lif_reset(struct ionic_dev *idev, u16 lif_index)
 426 {
 427         union ionic_dev_cmd cmd = {
 428                 .lif_init.opcode = IONIC_CMD_LIF_RESET,
 429                 .lif_init.index = cpu_to_le16(lif_index),
 430         };
 431
 432         ionic_dev_cmd_go(idev, &cmd);
 433 }
 434
 435 void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq,
 436                                u16 lif_index, u16 intr_index)
 437 {
 438         struct ionic_queue *q = &qcq->q;
 439         struct ionic_cq *cq = &qcq->cq;
 440
 441         union ionic_dev_cmd cmd = {
 442                 .q_init.opcode = IONIC_CMD_Q_INIT,
 443                 .q_init.lif_index = cpu_to_le16(lif_index),
 444                 .q_init.type = q->type,
 445                 .q_init.ver = qcq->q.lif->qtype_info[q->type].version,
 446                 .q_init.index = cpu_to_le32(q->index),
 447                 .q_init.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 448                                             IONIC_QINIT_F_ENA),
 449                 .q_init.pid = cpu_to_le16(q->pid),
 450                 .q_init.intr_index = cpu_to_le16(intr_index),
 451                 .q_init.ring_size = ilog2(q->num_descs),
 452                 .q_init.ring_base = cpu_to_le64(q->base_pa),
 453                 .q_init.cq_ring_base = cpu_to_le64(cq->base_pa),
 454         };
 455
 456         ionic_dev_cmd_go(idev, &cmd);
 457 }
 458
 459 int ionic_db_page_num(struct ionic_lif *lif, int pid)
 460 {
 461         return (lif->hw_index * lif->dbid_count) + pid;
 462 }
 463
 464 int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq,
 465                   struct ionic_intr_info *intr,
 466                   unsigned int num_descs, size_t desc_size)
 467 {
 468         unsigned int ring_size;
 469
 470         if (desc_size == 0 || !is_power_of_2(num_descs))
 471                 return -EINVAL;
 472
 473         ring_size = ilog2(num_descs);
 474         if (ring_size < 2 || ring_size > 16)
 475                 return -EINVAL;
 476
 477         cq->lif = lif;
 478         cq->bound_intr = intr;
 479         cq->num_descs = num_descs;
 480         cq->desc_size = desc_size;
 481         cq->tail_idx = 0;
 482         cq->done_color = 1;
 483
 484         return 0;
 485 }
 486
 487 void ionic_cq_map(struct ionic_cq *cq, void *base, dma_addr_t base_pa)
 488 {
 489         struct ionic_cq_info *cur;
 490         unsigned int i;
 491
 492         cq->base = base;
 493         cq->base_pa = base_pa;
 494
 495         for (i = 0, cur = cq->info; i < cq->num_descs; i++, cur++)
 496                 cur->cq_desc = base + (i * cq->desc_size);
 497 }
 498
 499 void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q)
 500 {
 501         cq->bound_q = q;
 502 }
 503
 504 unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do,
 505                               ionic_cq_cb cb, ionic_cq_done_cb done_cb,
 506                               void *done_arg)
 507 {
 508         struct ionic_cq_info *cq_info;
 509         unsigned int work_done = 0;
 510
 511         if (work_to_do == 0)
 512                 return 0;
 513
 514         cq_info = &cq->info[cq->tail_idx];
 515         while (cb(cq, cq_info)) {
 516                 if (cq->tail_idx == cq->num_descs - 1)
 517                         cq->done_color = !cq->done_color;
 518                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
 519                 cq_info = &cq->info[cq->tail_idx];
 520                 DEBUG_STATS_CQE_CNT(cq);
 521
 522                 if (++work_done >= work_to_do)
 523                         break;
 524         }
 525
 526         if (work_done && done_cb)
 527                 done_cb(done_arg);
 528
 529         return work_done;
 530 }
 531
 532 int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev,
 533                  struct ionic_queue *q, unsigned int index, const char *name,
 534                  unsigned int num_descs, size_t desc_size,
 535                  size_t sg_desc_size, unsigned int pid)
 536 {
 537         unsigned int ring_size;
 538
 539         if (desc_size == 0 || !is_power_of_2(num_descs))
 540                 return -EINVAL;
 541
 542         ring_size = ilog2(num_descs);
 543         if (ring_size < 2 || ring_size > 16)
 544                 return -EINVAL;
 545
 546         q->lif = lif;
 547         q->idev = idev;
 548         q->index = index;
 549         q->num_descs = num_descs;
 550         q->desc_size = desc_size;
 551         q->sg_desc_size = sg_desc_size;
 552         q->tail_idx = 0;
 553         q->head_idx = 0;
 554         q->pid = pid;
 555
 556         snprintf(q->name, sizeof(q->name), "L%d-%s%u", lif->index, name, index);
 557
 558         return 0;
 559 }
 560
 561 void ionic_q_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
 562 {
 563         struct ionic_desc_info *cur;
 564         unsigned int i;
 565
 566         q->base = base;
 567         q->base_pa = base_pa;
 568
 569         for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
 570                 cur->desc = base + (i * q->desc_size);
 571 }
 572
 573 void ionic_q_sg_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
 574 {
 575         struct ionic_desc_info *cur;
 576         unsigned int i;
 577
 578         q->sg_base = base;
 579         q->sg_base_pa = base_pa;
 580
 581         for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
 582                 cur->sg_desc = base + (i * q->sg_desc_size);
 583 }
 584
 585 void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb,
 586                   void *cb_arg)
 587 {
 588         struct device *dev = q->lif->ionic->dev;
 589         struct ionic_desc_info *desc_info;
 590         struct ionic_lif *lif = q->lif;
 591
 592         desc_info = &q->info[q->head_idx];
 593         desc_info->cb = cb;
 594         desc_info->cb_arg = cb_arg;
 595
 596         q->head_idx = (q->head_idx + 1) & (q->num_descs - 1);
 597
 598         dev_dbg(dev, "lif=%d qname=%s qid=%d qtype=%d p_index=%d ringdb=%d\n",
 599                 q->lif->index, q->name, q->hw_type, q->hw_index,
 600                 q->head_idx, ring_doorbell);
 601
 602         if (ring_doorbell)
 603                 ionic_dbell_ring(lif->kern_dbpage, q->hw_type,
 604                                  q->dbval | q->head_idx);
 605 }
 606
 607 static bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos)
 608 {
 609         unsigned int mask, tail, head;
 610
 611         mask = q->num_descs - 1;
 612         tail = q->tail_idx;
 613         head = q->head_idx;
 614
 615         return ((pos - tail) & mask) < ((head - tail) & mask);
 616 }
 617
 618 void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info,
 619                      unsigned int stop_index)
 620 {
 621         struct ionic_desc_info *desc_info;
 622         ionic_desc_cb cb;
 623         void *cb_arg;
 624         u16 index;
 625
 626         /* check for empty queue */
 627         if (q->tail_idx == q->head_idx)
 628                 return;
 629
 630         /* stop index must be for a descriptor that is not yet completed */
 631         if (unlikely(!ionic_q_is_posted(q, stop_index)))
 632                 dev_err(q->lif->ionic->dev,
 633                         "ionic stop is not posted %s stop %u tail %u head %u\n",
 634                         q->name, stop_index, q->tail_idx, q->head_idx);
 635
 636         do {
 637                 desc_info = &q->info[q->tail_idx];
 638                 index = q->tail_idx;
 639                 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 640
 641                 cb = desc_info->cb;
 642                 cb_arg = desc_info->cb_arg;
 643
 644                 desc_info->cb = NULL;
 645                 desc_info->cb_arg = NULL;
 646
 647                 if (cb)
 648                         cb(q, desc_info, cq_info, cb_arg);
 649         } while (index != stop_index);
 650 }