drivers/net/ethernet/pensando/ionic/ionic_lif.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4 #include <linux/ethtool.h>
   5 #include <linux/printk.h>
   6 #include <linux/dynamic_debug.h>
   7 #include <linux/netdevice.h>
   8 #include <linux/etherdevice.h>
   9 #include <linux/if_vlan.h>
  10 #include <linux/rtnetlink.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/pci.h>
  13 #include <linux/cpumask.h>
  14
  15 #include "ionic.h"
  16 #include "ionic_bus.h"
  17 #include "ionic_lif.h"
  18 #include "ionic_txrx.h"
  19 #include "ionic_ethtool.h"
  20 #include "ionic_debugfs.h"
  21
  22 /* queuetype support level */
  23 static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
  24         [IONIC_QTYPE_ADMINQ]  = 0,   /* 0 = Base version with CQ support */
  25         [IONIC_QTYPE_NOTIFYQ] = 0,   /* 0 = Base version */
  26         [IONIC_QTYPE_RXQ]     = 0,   /* 0 = Base version with CQ+SG support */
  27         [IONIC_QTYPE_TXQ]     = 1,   /* 0 = Base version with CQ+SG support
  28                                       * 1 =   ... with Tx SG version 1
  29                                       */
  30 };
  31
  32 static void ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode);
  33 static int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr);
  34 static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr);
  35 static void ionic_link_status_check(struct ionic_lif *lif);
  36 static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
  37 static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
  38 static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
  39
  40 static void ionic_txrx_deinit(struct ionic_lif *lif);
  41 static int ionic_txrx_init(struct ionic_lif *lif);
  42 static int ionic_start_queues(struct ionic_lif *lif);
  43 static void ionic_stop_queues(struct ionic_lif *lif);
  44 static void ionic_lif_queue_identify(struct ionic_lif *lif);
  45
  46 static void ionic_dim_work(struct work_struct *work)
  47 {
  48         struct dim *dim = container_of(work, struct dim, work);
  49         struct dim_cq_moder cur_moder;
  50         struct ionic_qcq *qcq;
  51         u32 new_coal;
  52
  53         cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
  54         qcq = container_of(dim, struct ionic_qcq, dim);
  55         new_coal = ionic_coal_usec_to_hw(qcq->q.lif->ionic, cur_moder.usec);
  56         qcq->intr.dim_coal_hw = new_coal ? new_coal : 1;
  57         dim->state = DIM_START_MEASURE;
  58 }
  59
  60 static void ionic_lif_deferred_work(struct work_struct *work)
  61 {
  62         struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
  63         struct ionic_deferred *def = &lif->deferred;
  64         struct ionic_deferred_work *w = NULL;
  65
  66         do {
  67                 spin_lock_bh(&def->lock);
  68                 if (!list_empty(&def->list)) {
  69                         w = list_first_entry(&def->list,
  70                                              struct ionic_deferred_work, list);
  71                         list_del(&w->list);
  72                 }
  73                 spin_unlock_bh(&def->lock);
  74
  75                 if (!w)
  76                         break;
  77
  78                 switch (w->type) {
  79                 case IONIC_DW_TYPE_RX_MODE:
  80                         ionic_lif_rx_mode(lif, w->rx_mode);
  81                         break;
  82                 case IONIC_DW_TYPE_RX_ADDR_ADD:
  83                         ionic_lif_addr_add(lif, w->addr);
  84                         break;
  85                 case IONIC_DW_TYPE_RX_ADDR_DEL:
  86                         ionic_lif_addr_del(lif, w->addr);
  87                         break;
  88                 case IONIC_DW_TYPE_LINK_STATUS:
  89                         ionic_link_status_check(lif);
  90                         break;
  91                 case IONIC_DW_TYPE_LIF_RESET:
  92                         if (w->fw_status)
  93                                 ionic_lif_handle_fw_up(lif);
  94                         else
  95                                 ionic_lif_handle_fw_down(lif);
  96                         break;
  97                 default:
  98                         break;
  99                 }
 100                 kfree(w);
 101                 w = NULL;
 102         } while (true);
 103 }
 104
 105 void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
 106                                 struct ionic_deferred_work *work)
 107 {
 108         spin_lock_bh(&def->lock);
 109         list_add_tail(&work->list, &def->list);
 110         spin_unlock_bh(&def->lock);
 111         schedule_work(&def->work);
 112 }
 113
 114 static void ionic_link_status_check(struct ionic_lif *lif)
 115 {
 116         struct net_device *netdev = lif->netdev;
 117         u16 link_status;
 118         bool link_up;
 119
 120         if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 121                 return;
 122
 123         /* Don't put carrier back up if we're in a broken state */
 124         if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
 125                 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 126                 return;
 127         }
 128
 129         link_status = le16_to_cpu(lif->info->status.link_status);
 130         link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
 131
 132         if (link_up) {
 133                 int err = 0;
 134
 135                 if (netdev->flags & IFF_UP && netif_running(netdev)) {
 136                         mutex_lock(&lif->queue_lock);
 137                         err = ionic_start_queues(lif);
 138                         if (err && err != -EBUSY) {
 139                                 netdev_err(lif->netdev,
 140                                            "Failed to start queues: %d\n", err);
 141                                 set_bit(IONIC_LIF_F_BROKEN, lif->state);
 142                                 netif_carrier_off(lif->netdev);
 143                         }
 144                         mutex_unlock(&lif->queue_lock);
 145                 }
 146
 147                 if (!err && !netif_carrier_ok(netdev)) {
 148                         ionic_port_identify(lif->ionic);
 149                         netdev_info(netdev, "Link up - %d Gbps\n",
 150                                     le32_to_cpu(lif->info->status.link_speed) / 1000);
 151                         netif_carrier_on(netdev);
 152                 }
 153         } else {
 154                 if (netif_carrier_ok(netdev)) {
 155                         netdev_info(netdev, "Link down\n");
 156                         netif_carrier_off(netdev);
 157                 }
 158
 159                 if (netdev->flags & IFF_UP && netif_running(netdev)) {
 160                         mutex_lock(&lif->queue_lock);
 161                         ionic_stop_queues(lif);
 162                         mutex_unlock(&lif->queue_lock);
 163                 }
 164         }
 165
 166         clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 167 }
 168
 169 void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
 170 {
 171         struct ionic_deferred_work *work;
 172
 173         /* we only need one request outstanding at a time */
 174         if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 175                 return;
 176
 177         if (!can_sleep) {
 178                 work = kzalloc(sizeof(*work), GFP_ATOMIC);
 179                 if (!work) {
 180                         clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 181                         return;
 182                 }
 183
 184                 work->type = IONIC_DW_TYPE_LINK_STATUS;
 185                 ionic_lif_deferred_enqueue(&lif->deferred, work);
 186         } else {
 187                 ionic_link_status_check(lif);
 188         }
 189 }
 190
 191 static irqreturn_t ionic_isr(int irq, void *data)
 192 {
 193         struct napi_struct *napi = data;
 194
 195         napi_schedule_irqoff(napi);
 196
 197         return IRQ_HANDLED;
 198 }
 199
 200 static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
 201 {
 202         struct ionic_intr_info *intr = &qcq->intr;
 203         struct device *dev = lif->ionic->dev;
 204         struct ionic_queue *q = &qcq->q;
 205         const char *name;
 206
 207         if (lif->registered)
 208                 name = lif->netdev->name;
 209         else
 210                 name = dev_name(dev);
 211
 212         snprintf(intr->name, sizeof(intr->name),
 213                  "%s-%s-%s", IONIC_DRV_NAME, name, q->name);
 214
 215         return devm_request_irq(dev, intr->vector, ionic_isr,
 216                                 0, intr->name, &qcq->napi);
 217 }
 218
 219 static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 220 {
 221         struct ionic *ionic = lif->ionic;
 222         int index;
 223
 224         index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
 225         if (index == ionic->nintrs) {
 226                 netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
 227                             __func__, index, ionic->nintrs);
 228                 return -ENOSPC;
 229         }
 230
 231         set_bit(index, ionic->intrs);
 232         ionic_intr_init(&ionic->idev, intr, index);
 233
 234         return 0;
 235 }
 236
 237 static void ionic_intr_free(struct ionic *ionic, int index)
 238 {
 239         if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
 240                 clear_bit(index, ionic->intrs);
 241 }
 242
 243 static int ionic_qcq_enable(struct ionic_qcq *qcq)
 244 {
 245         struct ionic_queue *q = &qcq->q;
 246         struct ionic_lif *lif = q->lif;
 247         struct ionic_dev *idev;
 248         struct device *dev;
 249
 250         struct ionic_admin_ctx ctx = {
 251                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 252                 .cmd.q_control = {
 253                         .opcode = IONIC_CMD_Q_CONTROL,
 254                         .lif_index = cpu_to_le16(lif->index),
 255                         .type = q->type,
 256                         .index = cpu_to_le32(q->index),
 257                         .oper = IONIC_Q_ENABLE,
 258                 },
 259         };
 260
 261         idev = &lif->ionic->idev;
 262         dev = lif->ionic->dev;
 263
 264         dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
 265                 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
 266
 267         if (qcq->flags & IONIC_QCQ_F_INTR) {
 268                 irq_set_affinity_hint(qcq->intr.vector,
 269                                       &qcq->intr.affinity_mask);
 270                 napi_enable(&qcq->napi);
 271                 ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
 272                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 273                                 IONIC_INTR_MASK_CLEAR);
 274         }
 275
 276         return ionic_adminq_post_wait(lif, &ctx);
 277 }
 278
 279 static int ionic_qcq_disable(struct ionic_qcq *qcq, bool send_to_hw)
 280 {
 281         struct ionic_queue *q;
 282         struct ionic_lif *lif;
 283         int err = 0;
 284
 285         struct ionic_admin_ctx ctx = {
 286                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 287                 .cmd.q_control = {
 288                         .opcode = IONIC_CMD_Q_CONTROL,
 289                         .oper = IONIC_Q_DISABLE,
 290                 },
 291         };
 292
 293         if (!qcq)
 294                 return -ENXIO;
 295
 296         q = &qcq->q;
 297         lif = q->lif;
 298
 299         if (qcq->flags & IONIC_QCQ_F_INTR) {
 300                 struct ionic_dev *idev = &lif->ionic->idev;
 301
 302                 cancel_work_sync(&qcq->dim.work);
 303                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 304                                 IONIC_INTR_MASK_SET);
 305                 synchronize_irq(qcq->intr.vector);
 306                 irq_set_affinity_hint(qcq->intr.vector, NULL);
 307                 napi_disable(&qcq->napi);
 308         }
 309
 310         if (send_to_hw) {
 311                 ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
 312                 ctx.cmd.q_control.type = q->type;
 313                 ctx.cmd.q_control.index = cpu_to_le32(q->index);
 314                 dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
 315                         ctx.cmd.q_control.index, ctx.cmd.q_control.type);
 316
 317                 err = ionic_adminq_post_wait(lif, &ctx);
 318         }
 319
 320         return err;
 321 }
 322
 323 static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
 324 {
 325         struct ionic_dev *idev = &lif->ionic->idev;
 326
 327         if (!qcq)
 328                 return;
 329
 330         if (!(qcq->flags & IONIC_QCQ_F_INITED))
 331                 return;
 332
 333         if (qcq->flags & IONIC_QCQ_F_INTR) {
 334                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 335                                 IONIC_INTR_MASK_SET);
 336                 netif_napi_del(&qcq->napi);
 337         }
 338
 339         qcq->flags &= ~IONIC_QCQ_F_INITED;
 340 }
 341
 342 static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
 343 {
 344         if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
 345                 return;
 346
 347         irq_set_affinity_hint(qcq->intr.vector, NULL);
 348         devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
 349         qcq->intr.vector = 0;
 350         ionic_intr_free(lif->ionic, qcq->intr.index);
 351         qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 352 }
 353
 354 static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
 355 {
 356         struct device *dev = lif->ionic->dev;
 357
 358         if (!qcq)
 359                 return;
 360
 361         ionic_debugfs_del_qcq(qcq);
 362
 363         if (qcq->q_base) {
 364                 dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
 365                 qcq->q_base = NULL;
 366                 qcq->q_base_pa = 0;
 367         }
 368
 369         if (qcq->cq_base) {
 370                 dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
 371                 qcq->cq_base = NULL;
 372                 qcq->cq_base_pa = 0;
 373         }
 374
 375         if (qcq->sg_base) {
 376                 dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
 377                 qcq->sg_base = NULL;
 378                 qcq->sg_base_pa = 0;
 379         }
 380
 381         ionic_qcq_intr_free(lif, qcq);
 382
 383         if (qcq->cq.info) {
 384                 devm_kfree(dev, qcq->cq.info);
 385                 qcq->cq.info = NULL;
 386         }
 387         if (qcq->q.info) {
 388                 devm_kfree(dev, qcq->q.info);
 389                 qcq->q.info = NULL;
 390         }
 391 }
 392
 393 static void ionic_qcqs_free(struct ionic_lif *lif)
 394 {
 395         struct device *dev = lif->ionic->dev;
 396         struct ionic_qcq *adminqcq;
 397         unsigned long irqflags;
 398
 399         if (lif->notifyqcq) {
 400                 ionic_qcq_free(lif, lif->notifyqcq);
 401                 devm_kfree(dev, lif->notifyqcq);
 402                 lif->notifyqcq = NULL;
 403         }
 404
 405         if (lif->adminqcq) {
 406                 spin_lock_irqsave(&lif->adminq_lock, irqflags);
 407                 adminqcq = READ_ONCE(lif->adminqcq);
 408                 lif->adminqcq = NULL;
 409                 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
 410                 if (adminqcq) {
 411                         ionic_qcq_free(lif, adminqcq);
 412                         devm_kfree(dev, adminqcq);
 413                 }
 414         }
 415
 416         if (lif->rxqcqs) {
 417                 devm_kfree(dev, lif->rxqstats);
 418                 lif->rxqstats = NULL;
 419                 devm_kfree(dev, lif->rxqcqs);
 420                 lif->rxqcqs = NULL;
 421         }
 422
 423         if (lif->txqcqs) {
 424                 devm_kfree(dev, lif->txqstats);
 425                 lif->txqstats = NULL;
 426                 devm_kfree(dev, lif->txqcqs);
 427                 lif->txqcqs = NULL;
 428         }
 429 }
 430
 431 static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
 432                                       struct ionic_qcq *n_qcq)
 433 {
 434         if (WARN_ON(n_qcq->flags & IONIC_QCQ_F_INTR)) {
 435                 ionic_intr_free(n_qcq->cq.lif->ionic, n_qcq->intr.index);
 436                 n_qcq->flags &= ~IONIC_QCQ_F_INTR;
 437         }
 438
 439         n_qcq->intr.vector = src_qcq->intr.vector;
 440         n_qcq->intr.index = src_qcq->intr.index;
 441 }
 442
 443 static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
 444 {
 445         int err;
 446
 447         if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
 448                 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 449                 return 0;
 450         }
 451
 452         err = ionic_intr_alloc(lif, &qcq->intr);
 453         if (err) {
 454                 netdev_warn(lif->netdev, "no intr for %s: %d\n",
 455                             qcq->q.name, err);
 456                 goto err_out;
 457         }
 458
 459         err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
 460         if (err < 0) {
 461                 netdev_warn(lif->netdev, "no vector for %s: %d\n",
 462                             qcq->q.name, err);
 463                 goto err_out_free_intr;
 464         }
 465         qcq->intr.vector = err;
 466         ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
 467                                IONIC_INTR_MASK_SET);
 468
 469         err = ionic_request_irq(lif, qcq);
 470         if (err) {
 471                 netdev_warn(lif->netdev, "irq request failed %d\n", err);
 472                 goto err_out_free_intr;
 473         }
 474
 475         /* try to get the irq on the local numa node first */
 476         qcq->intr.cpu = cpumask_local_spread(qcq->intr.index,
 477                                              dev_to_node(lif->ionic->dev));
 478         if (qcq->intr.cpu != -1)
 479                 cpumask_set_cpu(qcq->intr.cpu, &qcq->intr.affinity_mask);
 480
 481         netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
 482         return 0;
 483
 484 err_out_free_intr:
 485         ionic_intr_free(lif->ionic, qcq->intr.index);
 486 err_out:
 487         return err;
 488 }
 489
 490 static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
 491                            unsigned int index,
 492                            const char *name, unsigned int flags,
 493                            unsigned int num_descs, unsigned int desc_size,
 494                            unsigned int cq_desc_size,
 495                            unsigned int sg_desc_size,
 496                            unsigned int pid, struct ionic_qcq **qcq)
 497 {
 498         struct ionic_dev *idev = &lif->ionic->idev;
 499         struct device *dev = lif->ionic->dev;
 500         void *q_base, *cq_base, *sg_base;
 501         dma_addr_t cq_base_pa = 0;
 502         dma_addr_t sg_base_pa = 0;
 503         dma_addr_t q_base_pa = 0;
 504         struct ionic_qcq *new;
 505         int err;
 506
 507         *qcq = NULL;
 508
 509         new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
 510         if (!new) {
 511                 netdev_err(lif->netdev, "Cannot allocate queue structure\n");
 512                 err = -ENOMEM;
 513                 goto err_out;
 514         }
 515
 516         new->q.dev = dev;
 517         new->flags = flags;
 518
 519         new->q.info = devm_kcalloc(dev, num_descs, sizeof(*new->q.info),
 520                                    GFP_KERNEL);
 521         if (!new->q.info) {
 522                 netdev_err(lif->netdev, "Cannot allocate queue info\n");
 523                 err = -ENOMEM;
 524                 goto err_out_free_qcq;
 525         }
 526
 527         new->q.type = type;
 528         new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
 529
 530         err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
 531                            desc_size, sg_desc_size, pid);
 532         if (err) {
 533                 netdev_err(lif->netdev, "Cannot initialize queue\n");
 534                 goto err_out_free_q_info;
 535         }
 536
 537         err = ionic_alloc_qcq_interrupt(lif, new);
 538         if (err)
 539                 goto err_out;
 540
 541         new->cq.info = devm_kcalloc(dev, num_descs, sizeof(*new->cq.info),
 542                                     GFP_KERNEL);
 543         if (!new->cq.info) {
 544                 netdev_err(lif->netdev, "Cannot allocate completion queue info\n");
 545                 err = -ENOMEM;
 546                 goto err_out_free_irq;
 547         }
 548
 549         err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
 550         if (err) {
 551                 netdev_err(lif->netdev, "Cannot initialize completion queue\n");
 552                 goto err_out_free_cq_info;
 553         }
 554
 555         if (flags & IONIC_QCQ_F_NOTIFYQ) {
 556                 int q_size, cq_size;
 557
 558                 /* q & cq need to be contiguous in case of notifyq */
 559                 q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
 560                 cq_size = ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
 561
 562                 new->q_size = PAGE_SIZE + q_size + cq_size;
 563                 new->q_base = dma_alloc_coherent(dev, new->q_size,
 564                                                  &new->q_base_pa, GFP_KERNEL);
 565                 if (!new->q_base) {
 566                         netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
 567                         err = -ENOMEM;
 568                         goto err_out_free_cq_info;
 569                 }
 570                 q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
 571                 q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
 572                 ionic_q_map(&new->q, q_base, q_base_pa);
 573
 574                 cq_base = PTR_ALIGN(q_base + q_size, PAGE_SIZE);
 575                 cq_base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
 576                 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 577                 ionic_cq_bind(&new->cq, &new->q);
 578         } else {
 579                 new->q_size = PAGE_SIZE + (num_descs * desc_size);
 580                 new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
 581                                                  GFP_KERNEL);
 582                 if (!new->q_base) {
 583                         netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
 584                         err = -ENOMEM;
 585                         goto err_out_free_cq_info;
 586                 }
 587                 q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
 588                 q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
 589                 ionic_q_map(&new->q, q_base, q_base_pa);
 590
 591                 new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
 592                 new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
 593                                                   GFP_KERNEL);
 594                 if (!new->cq_base) {
 595                         netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
 596                         err = -ENOMEM;
 597                         goto err_out_free_q;
 598                 }
 599                 cq_base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
 600                 cq_base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
 601                 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 602                 ionic_cq_bind(&new->cq, &new->q);
 603         }
 604
 605         if (flags & IONIC_QCQ_F_SG) {
 606                 new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
 607                 new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
 608                                                   GFP_KERNEL);
 609                 if (!new->sg_base) {
 610                         netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
 611                         err = -ENOMEM;
 612                         goto err_out_free_cq;
 613                 }
 614                 sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
 615                 sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
 616                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 617         }
 618
 619         INIT_WORK(&new->dim.work, ionic_dim_work);
 620         new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 621
 622         *qcq = new;
 623
 624         return 0;
 625
 626 err_out_free_cq:
 627         dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
 628 err_out_free_q:
 629         dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
 630 err_out_free_cq_info:
 631         devm_kfree(dev, new->cq.info);
 632 err_out_free_irq:
 633         if (flags & IONIC_QCQ_F_INTR) {
 634                 devm_free_irq(dev, new->intr.vector, &new->napi);
 635                 ionic_intr_free(lif->ionic, new->intr.index);
 636         }
 637 err_out_free_q_info:
 638         devm_kfree(dev, new->q.info);
 639 err_out_free_qcq:
 640         devm_kfree(dev, new);
 641 err_out:
 642         dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
 643         return err;
 644 }
 645
 646 static int ionic_qcqs_alloc(struct ionic_lif *lif)
 647 {
 648         struct device *dev = lif->ionic->dev;
 649         unsigned int flags;
 650         int err;
 651
 652         flags = IONIC_QCQ_F_INTR;
 653         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 654                               IONIC_ADMINQ_LENGTH,
 655                               sizeof(struct ionic_admin_cmd),
 656                               sizeof(struct ionic_admin_comp),
 657                               0, lif->kern_pid, &lif->adminqcq);
 658         if (err)
 659                 return err;
 660         ionic_debugfs_add_qcq(lif, lif->adminqcq);
 661
 662         if (lif->ionic->nnqs_per_lif) {
 663                 flags = IONIC_QCQ_F_NOTIFYQ;
 664                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
 665                                       flags, IONIC_NOTIFYQ_LENGTH,
 666                                       sizeof(struct ionic_notifyq_cmd),
 667                                       sizeof(union ionic_notifyq_comp),
 668                                       0, lif->kern_pid, &lif->notifyqcq);
 669                 if (err)
 670                         goto err_out;
 671                 ionic_debugfs_add_qcq(lif, lif->notifyqcq);
 672
 673                 /* Let the notifyq ride on the adminq interrupt */
 674                 ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
 675         }
 676
 677         err = -ENOMEM;
 678         lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
 679                                    sizeof(*lif->txqcqs), GFP_KERNEL);
 680         if (!lif->txqcqs)
 681                 goto err_out;
 682         lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
 683                                    sizeof(*lif->rxqcqs), GFP_KERNEL);
 684         if (!lif->rxqcqs)
 685                 goto err_out;
 686
 687         lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
 688                                      sizeof(*lif->txqstats), GFP_KERNEL);
 689         if (!lif->txqstats)
 690                 goto err_out;
 691         lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
 692                                      sizeof(*lif->rxqstats), GFP_KERNEL);
 693         if (!lif->rxqstats)
 694                 goto err_out;
 695
 696         return 0;
 697
 698 err_out:
 699         ionic_qcqs_free(lif);
 700         return err;
 701 }
 702
 703 static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
 704 {
 705         qcq->q.tail_idx = 0;
 706         qcq->q.head_idx = 0;
 707         qcq->cq.tail_idx = 0;
 708         qcq->cq.done_color = 1;
 709         memset(qcq->q_base, 0, qcq->q_size);
 710         memset(qcq->cq_base, 0, qcq->cq_size);
 711         memset(qcq->sg_base, 0, qcq->sg_size);
 712 }
 713
 714 static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 715 {
 716         struct device *dev = lif->ionic->dev;
 717         struct ionic_queue *q = &qcq->q;
 718         struct ionic_cq *cq = &qcq->cq;
 719         struct ionic_admin_ctx ctx = {
 720                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 721                 .cmd.q_init = {
 722                         .opcode = IONIC_CMD_Q_INIT,
 723                         .lif_index = cpu_to_le16(lif->index),
 724                         .type = q->type,
 725                         .ver = lif->qtype_info[q->type].version,
 726                         .index = cpu_to_le32(q->index),
 727                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 728                                              IONIC_QINIT_F_SG),
 729                         .pid = cpu_to_le16(q->pid),
 730                         .ring_size = ilog2(q->num_descs),
 731                         .ring_base = cpu_to_le64(q->base_pa),
 732                         .cq_ring_base = cpu_to_le64(cq->base_pa),
 733                         .sg_ring_base = cpu_to_le64(q->sg_base_pa),
 734                         .features = cpu_to_le64(q->features),
 735                 },
 736         };
 737         unsigned int intr_index;
 738         int err;
 739
 740         intr_index = qcq->intr.index;
 741
 742         ctx.cmd.q_init.intr_index = cpu_to_le16(intr_index);
 743
 744         dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
 745         dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
 746         dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
 747         dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
 748         dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
 749         dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
 750         dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
 751
 752         ionic_qcq_sanitize(qcq);
 753
 754         err = ionic_adminq_post_wait(lif, &ctx);
 755         if (err)
 756                 return err;
 757
 758         q->hw_type = ctx.comp.q_init.hw_type;
 759         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
 760         q->dbval = IONIC_DBELL_QID(q->hw_index);
 761
 762         dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
 763         dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
 764
 765         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
 766                 netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi,
 767                                NAPI_POLL_WEIGHT);
 768
 769         qcq->flags |= IONIC_QCQ_F_INITED;
 770
 771         return 0;
 772 }
 773
 774 static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 775 {
 776         struct device *dev = lif->ionic->dev;
 777         struct ionic_queue *q = &qcq->q;
 778         struct ionic_cq *cq = &qcq->cq;
 779         struct ionic_admin_ctx ctx = {
 780                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 781                 .cmd.q_init = {
 782                         .opcode = IONIC_CMD_Q_INIT,
 783                         .lif_index = cpu_to_le16(lif->index),
 784                         .type = q->type,
 785                         .ver = lif->qtype_info[q->type].version,
 786                         .index = cpu_to_le32(q->index),
 787                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 788                                              IONIC_QINIT_F_SG),
 789                         .intr_index = cpu_to_le16(cq->bound_intr->index),
 790                         .pid = cpu_to_le16(q->pid),
 791                         .ring_size = ilog2(q->num_descs),
 792                         .ring_base = cpu_to_le64(q->base_pa),
 793                         .cq_ring_base = cpu_to_le64(cq->base_pa),
 794                         .sg_ring_base = cpu_to_le64(q->sg_base_pa),
 795                         .features = cpu_to_le64(q->features),
 796                 },
 797         };
 798         int err;
 799
 800         dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
 801         dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
 802         dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
 803         dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
 804         dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
 805         dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
 806         dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
 807
 808         ionic_qcq_sanitize(qcq);
 809
 810         err = ionic_adminq_post_wait(lif, &ctx);
 811         if (err)
 812                 return err;
 813
 814         q->hw_type = ctx.comp.q_init.hw_type;
 815         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
 816         q->dbval = IONIC_DBELL_QID(q->hw_index);
 817
 818         dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
 819         dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
 820
 821         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
 822                 netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi,
 823                                NAPI_POLL_WEIGHT);
 824         else
 825                 netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi,
 826                                NAPI_POLL_WEIGHT);
 827
 828         qcq->flags |= IONIC_QCQ_F_INITED;
 829
 830         return 0;
 831 }
 832
 833 int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
 834 {
 835         unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
 836         unsigned int txq_i, flags;
 837         struct ionic_qcq *txq;
 838         u64 features;
 839         int err;
 840
 841         mutex_lock(&lif->queue_lock);
 842
 843         if (lif->hwstamp_txq)
 844                 goto out;
 845
 846         features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
 847
 848         num_desc = IONIC_MIN_TXRX_DESC;
 849         desc_sz = sizeof(struct ionic_txq_desc);
 850         comp_sz = 2 * sizeof(struct ionic_txq_comp);
 851
 852         if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
 853             lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
 854                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
 855         else
 856                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
 857
 858         txq_i = lif->ionic->ntxqs_per_lif;
 859         flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
 860
 861         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
 862                               num_desc, desc_sz, comp_sz, sg_desc_sz,
 863                               lif->kern_pid, &txq);
 864         if (err)
 865                 goto err_qcq_alloc;
 866
 867         txq->q.features = features;
 868
 869         ionic_link_qcq_interrupts(lif->adminqcq, txq);
 870         ionic_debugfs_add_qcq(lif, txq);
 871
 872         lif->hwstamp_txq = txq;
 873
 874         if (netif_running(lif->netdev)) {
 875                 err = ionic_lif_txq_init(lif, txq);
 876                 if (err)
 877                         goto err_qcq_init;
 878
 879                 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
 880                         err = ionic_qcq_enable(txq);
 881                         if (err)
 882                                 goto err_qcq_enable;
 883                 }
 884         }
 885
 886 out:
 887         mutex_unlock(&lif->queue_lock);
 888
 889         return 0;
 890
 891 err_qcq_enable:
 892         ionic_lif_qcq_deinit(lif, txq);
 893 err_qcq_init:
 894         lif->hwstamp_txq = NULL;
 895         ionic_debugfs_del_qcq(txq);
 896         ionic_qcq_free(lif, txq);
 897         devm_kfree(lif->ionic->dev, txq);
 898 err_qcq_alloc:
 899         mutex_unlock(&lif->queue_lock);
 900         return err;
 901 }
 902
 903 int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
 904 {
 905         unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
 906         unsigned int rxq_i, flags;
 907         struct ionic_qcq *rxq;
 908         u64 features;
 909         int err;
 910
 911         mutex_lock(&lif->queue_lock);
 912
 913         if (lif->hwstamp_rxq)
 914                 goto out;
 915
 916         features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
 917
 918         num_desc = IONIC_MIN_TXRX_DESC;
 919         desc_sz = sizeof(struct ionic_rxq_desc);
 920         comp_sz = 2 * sizeof(struct ionic_rxq_comp);
 921         sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
 922
 923         rxq_i = lif->ionic->nrxqs_per_lif;
 924         flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
 925
 926         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
 927                               num_desc, desc_sz, comp_sz, sg_desc_sz,
 928                               lif->kern_pid, &rxq);
 929         if (err)
 930                 goto err_qcq_alloc;
 931
 932         rxq->q.features = features;
 933
 934         ionic_link_qcq_interrupts(lif->adminqcq, rxq);
 935         ionic_debugfs_add_qcq(lif, rxq);
 936
 937         lif->hwstamp_rxq = rxq;
 938
 939         if (netif_running(lif->netdev)) {
 940                 err = ionic_lif_rxq_init(lif, rxq);
 941                 if (err)
 942                         goto err_qcq_init;
 943
 944                 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
 945                         ionic_rx_fill(&rxq->q);
 946                         err = ionic_qcq_enable(rxq);
 947                         if (err)
 948                                 goto err_qcq_enable;
 949                 }
 950         }
 951
 952 out:
 953         mutex_unlock(&lif->queue_lock);
 954
 955         return 0;
 956
 957 err_qcq_enable:
 958         ionic_lif_qcq_deinit(lif, rxq);
 959 err_qcq_init:
 960         lif->hwstamp_rxq = NULL;
 961         ionic_debugfs_del_qcq(rxq);
 962         ionic_qcq_free(lif, rxq);
 963         devm_kfree(lif->ionic->dev, rxq);
 964 err_qcq_alloc:
 965         mutex_unlock(&lif->queue_lock);
 966         return err;
 967 }
 968
 969 int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
 970 {
 971         struct ionic_queue_params qparam;
 972
 973         ionic_init_queue_params(lif, &qparam);
 974
 975         if (rx_all)
 976                 qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
 977         else
 978                 qparam.rxq_features = 0;
 979
 980         /* if we're not running, just set the values and return */
 981         if (!netif_running(lif->netdev)) {
 982                 lif->rxq_features = qparam.rxq_features;
 983                 return 0;
 984         }
 985
 986         return ionic_reconfigure_queues(lif, &qparam);
 987 }
 988
 989 int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
 990 {
 991         struct ionic_admin_ctx ctx = {
 992                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 993                 .cmd.lif_setattr = {
 994                         .opcode = IONIC_CMD_LIF_SETATTR,
 995                         .index = cpu_to_le16(lif->index),
 996                         .attr = IONIC_LIF_ATTR_TXSTAMP,
 997                         .txstamp_mode = cpu_to_le16(txstamp_mode),
 998                 },
 999         };
1000
1001         return ionic_adminq_post_wait(lif, &ctx);
1002 }
1003
1004 static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
1005 {
1006         struct ionic_admin_ctx ctx = {
1007                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1008                 .cmd.rx_filter_del = {
1009                         .opcode = IONIC_CMD_RX_FILTER_DEL,
1010                         .lif_index = cpu_to_le16(lif->index),
1011                 },
1012         };
1013         struct ionic_rx_filter *f;
1014         u32 filter_id;
1015         int err;
1016
1017         spin_lock_bh(&lif->rx_filters.lock);
1018
1019         f = ionic_rx_filter_rxsteer(lif);
1020         if (!f) {
1021                 spin_unlock_bh(&lif->rx_filters.lock);
1022                 return;
1023         }
1024
1025         filter_id = f->filter_id;
1026         ionic_rx_filter_free(lif, f);
1027
1028         spin_unlock_bh(&lif->rx_filters.lock);
1029
1030         netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
1031
1032         ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
1033
1034         err = ionic_adminq_post_wait(lif, &ctx);
1035         if (err && err != -EEXIST)
1036                 netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
1037 }
1038
1039 static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1040 {
1041         struct ionic_admin_ctx ctx = {
1042                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1043                 .cmd.rx_filter_add = {
1044                         .opcode = IONIC_CMD_RX_FILTER_ADD,
1045                         .lif_index = cpu_to_le16(lif->index),
1046                         .match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
1047                         .pkt_class = cpu_to_le64(pkt_class),
1048                 },
1049         };
1050         u8 qtype;
1051         u32 qid;
1052         int err;
1053
1054         if (!lif->hwstamp_rxq)
1055                 return -EINVAL;
1056
1057         qtype = lif->hwstamp_rxq->q.type;
1058         ctx.cmd.rx_filter_add.qtype = qtype;
1059
1060         qid = lif->hwstamp_rxq->q.index;
1061         ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
1062
1063         netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
1064         err = ionic_adminq_post_wait(lif, &ctx);
1065         if (err && err != -EEXIST)
1066                 return err;
1067
1068         return ionic_rx_filter_save(lif, 0, qid, 0, &ctx);
1069 }
1070
1071 int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1072 {
1073         ionic_lif_del_hwstamp_rxfilt(lif);
1074
1075         if (!pkt_class)
1076                 return 0;
1077
1078         return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
1079 }
1080
1081 static bool ionic_notifyq_service(struct ionic_cq *cq,
1082                                   struct ionic_cq_info *cq_info)
1083 {
1084         union ionic_notifyq_comp *comp = cq_info->cq_desc;
1085         struct ionic_deferred_work *work;
1086         struct net_device *netdev;
1087         struct ionic_queue *q;
1088         struct ionic_lif *lif;
1089         u64 eid;
1090
1091         q = cq->bound_q;
1092         lif = q->info[0].cb_arg;
1093         netdev = lif->netdev;
1094         eid = le64_to_cpu(comp->event.eid);
1095
1096         /* Have we run out of new completions to process? */
1097         if ((s64)(eid - lif->last_eid) <= 0)
1098                 return false;
1099
1100         lif->last_eid = eid;
1101
1102         dev_dbg(lif->ionic->dev, "notifyq event:\n");
1103         dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET, 16, 1,
1104                          comp, sizeof(*comp), true);
1105
1106         switch (le16_to_cpu(comp->event.ecode)) {
1107         case IONIC_EVENT_LINK_CHANGE:
1108                 ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1109                 break;
1110         case IONIC_EVENT_RESET:
1111                 work = kzalloc(sizeof(*work), GFP_ATOMIC);
1112                 if (!work) {
1113                         netdev_err(lif->netdev, "Reset event dropped\n");
1114                 } else {
1115                         work->type = IONIC_DW_TYPE_LIF_RESET;
1116                         ionic_lif_deferred_enqueue(&lif->deferred, work);
1117                 }
1118                 break;
1119         default:
1120                 netdev_warn(netdev, "Notifyq event ecode=%d eid=%lld\n",
1121                             comp->event.ecode, eid);
1122                 break;
1123         }
1124
1125         return true;
1126 }
1127
1128 static bool ionic_adminq_service(struct ionic_cq *cq,
1129                                  struct ionic_cq_info *cq_info)
1130 {
1131         struct ionic_admin_comp *comp = cq_info->cq_desc;
1132
1133         if (!color_match(comp->color, cq->done_color))
1134                 return false;
1135
1136         ionic_q_service(cq->bound_q, cq_info, le16_to_cpu(comp->comp_index));
1137
1138         return true;
1139 }
1140
1141 static int ionic_adminq_napi(struct napi_struct *napi, int budget)
1142 {
1143         struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
1144         struct ionic_lif *lif = napi_to_cq(napi)->lif;
1145         struct ionic_dev *idev = &lif->ionic->idev;
1146         unsigned long irqflags;
1147         unsigned int flags = 0;
1148         int rx_work = 0;
1149         int tx_work = 0;
1150         int n_work = 0;
1151         int a_work = 0;
1152         int work_done;
1153         int credits;
1154
1155         if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
1156                 n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
1157                                           ionic_notifyq_service, NULL, NULL);
1158
1159         spin_lock_irqsave(&lif->adminq_lock, irqflags);
1160         if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
1161                 a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
1162                                           ionic_adminq_service, NULL, NULL);
1163         spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
1164
1165         if (lif->hwstamp_rxq)
1166                 rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
1167                                            ionic_rx_service, NULL, NULL);
1168
1169         if (lif->hwstamp_txq)
1170                 tx_work = ionic_cq_service(&lif->hwstamp_txq->cq, budget,
1171                                            ionic_tx_service, NULL, NULL);
1172
1173         work_done = max(max(n_work, a_work), max(rx_work, tx_work));
1174         if (work_done < budget && napi_complete_done(napi, work_done)) {
1175                 flags |= IONIC_INTR_CRED_UNMASK;
1176                 intr->rearm_count++;
1177         }
1178
1179         if (work_done || flags) {
1180                 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1181                 credits = n_work + a_work + rx_work + tx_work;
1182                 ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
1183         }
1184
1185         return work_done;
1186 }
1187
1188 void ionic_get_stats64(struct net_device *netdev,
1189                        struct rtnl_link_stats64 *ns)
1190 {
1191         struct ionic_lif *lif = netdev_priv(netdev);
1192         struct ionic_lif_stats *ls;
1193
1194         memset(ns, 0, sizeof(*ns));
1195         ls = &lif->info->stats;
1196
1197         ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
1198                          le64_to_cpu(ls->rx_mcast_packets) +
1199                          le64_to_cpu(ls->rx_bcast_packets);
1200
1201         ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
1202                          le64_to_cpu(ls->tx_mcast_packets) +
1203                          le64_to_cpu(ls->tx_bcast_packets);
1204
1205         ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
1206                        le64_to_cpu(ls->rx_mcast_bytes) +
1207                        le64_to_cpu(ls->rx_bcast_bytes);
1208
1209         ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
1210                        le64_to_cpu(ls->tx_mcast_bytes) +
1211                        le64_to_cpu(ls->tx_bcast_bytes);
1212
1213         ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
1214                          le64_to_cpu(ls->rx_mcast_drop_packets) +
1215                          le64_to_cpu(ls->rx_bcast_drop_packets);
1216
1217         ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
1218                          le64_to_cpu(ls->tx_mcast_drop_packets) +
1219                          le64_to_cpu(ls->tx_bcast_drop_packets);
1220
1221         ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
1222
1223         ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
1224
1225         ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
1226                                le64_to_cpu(ls->rx_queue_disabled) +
1227                                le64_to_cpu(ls->rx_desc_fetch_error) +
1228                                le64_to_cpu(ls->rx_desc_data_error);
1229
1230         ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
1231                                 le64_to_cpu(ls->tx_queue_disabled) +
1232                                 le64_to_cpu(ls->tx_desc_fetch_error) +
1233                                 le64_to_cpu(ls->tx_desc_data_error);
1234
1235         ns->rx_errors = ns->rx_over_errors +
1236                         ns->rx_missed_errors;
1237
1238         ns->tx_errors = ns->tx_aborted_errors;
1239 }
1240
1241 static int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr)
1242 {
1243         struct ionic_admin_ctx ctx = {
1244                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1245                 .cmd.rx_filter_add = {
1246                         .opcode = IONIC_CMD_RX_FILTER_ADD,
1247                         .lif_index = cpu_to_le16(lif->index),
1248                         .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_MAC),
1249                 },
1250         };
1251         struct ionic_rx_filter *f;
1252         int err;
1253
1254         /* don't bother if we already have it */
1255         spin_lock_bh(&lif->rx_filters.lock);
1256         f = ionic_rx_filter_by_addr(lif, addr);
1257         spin_unlock_bh(&lif->rx_filters.lock);
1258         if (f)
1259                 return 0;
1260
1261         netdev_dbg(lif->netdev, "rx_filter add ADDR %pM\n", addr);
1262
1263         memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, ETH_ALEN);
1264         err = ionic_adminq_post_wait(lif, &ctx);
1265         if (err && err != -EEXIST)
1266                 return err;
1267
1268         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx);
1269 }
1270
1271 static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr)
1272 {
1273         struct ionic_admin_ctx ctx = {
1274                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1275                 .cmd.rx_filter_del = {
1276                         .opcode = IONIC_CMD_RX_FILTER_DEL,
1277                         .lif_index = cpu_to_le16(lif->index),
1278                 },
1279         };
1280         struct ionic_rx_filter *f;
1281         int err;
1282
1283         spin_lock_bh(&lif->rx_filters.lock);
1284         f = ionic_rx_filter_by_addr(lif, addr);
1285         if (!f) {
1286                 spin_unlock_bh(&lif->rx_filters.lock);
1287                 return -ENOENT;
1288         }
1289
1290         netdev_dbg(lif->netdev, "rx_filter del ADDR %pM (id %d)\n",
1291                    addr, f->filter_id);
1292
1293         ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1294         ionic_rx_filter_free(lif, f);
1295         spin_unlock_bh(&lif->rx_filters.lock);
1296
1297         err = ionic_adminq_post_wait(lif, &ctx);
1298         if (err && err != -EEXIST)
1299                 return err;
1300
1301         return 0;
1302 }
1303
1304 static int ionic_lif_addr(struct ionic_lif *lif, const u8 *addr, bool add,
1305                           bool can_sleep)
1306 {
1307         struct ionic_deferred_work *work;
1308         unsigned int nmfilters;
1309         unsigned int nufilters;
1310
1311         if (add) {
1312                 /* Do we have space for this filter?  We test the counters
1313                  * here before checking the need for deferral so that we
1314                  * can return an overflow error to the stack.
1315                  */
1316                 nmfilters = le32_to_cpu(lif->identity->eth.max_mcast_filters);
1317                 nufilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1318
1319                 if ((is_multicast_ether_addr(addr) && lif->nmcast < nmfilters))
1320                         lif->nmcast++;
1321                 else if (!is_multicast_ether_addr(addr) &&
1322                          lif->nucast < nufilters)
1323                         lif->nucast++;
1324                 else
1325                         return -ENOSPC;
1326         } else {
1327                 if (is_multicast_ether_addr(addr) && lif->nmcast)
1328                         lif->nmcast--;
1329                 else if (!is_multicast_ether_addr(addr) && lif->nucast)
1330                         lif->nucast--;
1331         }
1332
1333         if (!can_sleep) {
1334                 work = kzalloc(sizeof(*work), GFP_ATOMIC);
1335                 if (!work)
1336                         return -ENOMEM;
1337                 work->type = add ? IONIC_DW_TYPE_RX_ADDR_ADD :
1338                                    IONIC_DW_TYPE_RX_ADDR_DEL;
1339                 memcpy(work->addr, addr, ETH_ALEN);
1340                 netdev_dbg(lif->netdev, "deferred: rx_filter %s %pM\n",
1341                            add ? "add" : "del", addr);
1342                 ionic_lif_deferred_enqueue(&lif->deferred, work);
1343         } else {
1344                 netdev_dbg(lif->netdev, "rx_filter %s %pM\n",
1345                            add ? "add" : "del", addr);
1346                 if (add)
1347                         return ionic_lif_addr_add(lif, addr);
1348                 else
1349                         return ionic_lif_addr_del(lif, addr);
1350         }
1351
1352         return 0;
1353 }
1354
1355 static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
1356 {
1357         return ionic_lif_addr(netdev_priv(netdev), addr, ADD_ADDR, CAN_SLEEP);
1358 }
1359
1360 static int ionic_ndo_addr_add(struct net_device *netdev, const u8 *addr)
1361 {
1362         return ionic_lif_addr(netdev_priv(netdev), addr, ADD_ADDR, CAN_NOT_SLEEP);
1363 }
1364
1365 static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
1366 {
1367         return ionic_lif_addr(netdev_priv(netdev), addr, DEL_ADDR, CAN_SLEEP);
1368 }
1369
1370 static int ionic_ndo_addr_del(struct net_device *netdev, const u8 *addr)
1371 {
1372         return ionic_lif_addr(netdev_priv(netdev), addr, DEL_ADDR, CAN_NOT_SLEEP);
1373 }
1374
1375 static void ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode)
1376 {
1377         struct ionic_admin_ctx ctx = {
1378                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1379                 .cmd.rx_mode_set = {
1380                         .opcode = IONIC_CMD_RX_MODE_SET,
1381                         .lif_index = cpu_to_le16(lif->index),
1382                         .rx_mode = cpu_to_le16(rx_mode),
1383                 },
1384         };
1385         char buf[128];
1386         int err;
1387         int i;
1388 #define REMAIN(__x) (sizeof(buf) - (__x))
1389
1390         i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
1391                       lif->rx_mode, rx_mode);
1392         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
1393                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
1394         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
1395                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
1396         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
1397                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
1398         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
1399                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
1400         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
1401                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
1402         netdev_dbg(lif->netdev, "lif%d %s\n", lif->index, buf);
1403
1404         err = ionic_adminq_post_wait(lif, &ctx);
1405         if (err)
1406                 netdev_warn(lif->netdev, "set rx_mode 0x%04x failed: %d\n",
1407                             rx_mode, err);
1408         else
1409                 lif->rx_mode = rx_mode;
1410 }
1411
1412 static void ionic_set_rx_mode(struct net_device *netdev, bool can_sleep)
1413 {
1414         struct ionic_lif *lif = netdev_priv(netdev);
1415         struct ionic_deferred_work *work;
1416         unsigned int nfilters;
1417         unsigned int rx_mode;
1418
1419         rx_mode = IONIC_RX_MODE_F_UNICAST;
1420         rx_mode |= (netdev->flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
1421         rx_mode |= (netdev->flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
1422         rx_mode |= (netdev->flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
1423         rx_mode |= (netdev->flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
1424
1425         /* sync unicast addresses
1426          * next check to see if we're in an overflow state
1427          *    if so, we track that we overflowed and enable NIC PROMISC
1428          *    else if the overflow is set and not needed
1429          *       we remove our overflow flag and check the netdev flags
1430          *       to see if we can disable NIC PROMISC
1431          */
1432         if (can_sleep)
1433                 __dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
1434         else
1435                 __dev_uc_sync(netdev, ionic_ndo_addr_add, ionic_ndo_addr_del);
1436         nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1437         if (netdev_uc_count(netdev) + 1 > nfilters) {
1438                 rx_mode |= IONIC_RX_MODE_F_PROMISC;
1439                 lif->uc_overflow = true;
1440         } else if (lif->uc_overflow) {
1441                 lif->uc_overflow = false;
1442                 if (!(netdev->flags & IFF_PROMISC))
1443                         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
1444         }
1445
1446         /* same for multicast */
1447         if (can_sleep)
1448                 __dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
1449         else
1450                 __dev_mc_sync(netdev, ionic_ndo_addr_add, ionic_ndo_addr_del);
1451         nfilters = le32_to_cpu(lif->identity->eth.max_mcast_filters);
1452         if (netdev_mc_count(netdev) > nfilters) {
1453                 rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
1454                 lif->mc_overflow = true;
1455         } else if (lif->mc_overflow) {
1456                 lif->mc_overflow = false;
1457                 if (!(netdev->flags & IFF_ALLMULTI))
1458                         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
1459         }
1460
1461         if (lif->rx_mode != rx_mode) {
1462                 if (!can_sleep) {
1463                         work = kzalloc(sizeof(*work), GFP_ATOMIC);
1464                         if (!work) {
1465                                 netdev_err(lif->netdev, "rxmode change dropped\n");
1466                                 return;
1467                         }
1468                         work->type = IONIC_DW_TYPE_RX_MODE;
1469                         work->rx_mode = rx_mode;
1470                         netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1471                         ionic_lif_deferred_enqueue(&lif->deferred, work);
1472                 } else {
1473                         ionic_lif_rx_mode(lif, rx_mode);
1474                 }
1475         }
1476 }
1477
1478 static void ionic_ndo_set_rx_mode(struct net_device *netdev)
1479 {
1480         ionic_set_rx_mode(netdev, CAN_NOT_SLEEP);
1481 }
1482
1483 static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
1484 {
1485         u64 wanted = 0;
1486
1487         if (features & NETIF_F_HW_VLAN_CTAG_TX)
1488                 wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
1489         if (features & NETIF_F_HW_VLAN_CTAG_RX)
1490                 wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
1491         if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1492                 wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
1493         if (features & NETIF_F_RXHASH)
1494                 wanted |= IONIC_ETH_HW_RX_HASH;
1495         if (features & NETIF_F_RXCSUM)
1496                 wanted |= IONIC_ETH_HW_RX_CSUM;
1497         if (features & NETIF_F_SG)
1498                 wanted |= IONIC_ETH_HW_TX_SG;
1499         if (features & NETIF_F_HW_CSUM)
1500                 wanted |= IONIC_ETH_HW_TX_CSUM;
1501         if (features & NETIF_F_TSO)
1502                 wanted |= IONIC_ETH_HW_TSO;
1503         if (features & NETIF_F_TSO6)
1504                 wanted |= IONIC_ETH_HW_TSO_IPV6;
1505         if (features & NETIF_F_TSO_ECN)
1506                 wanted |= IONIC_ETH_HW_TSO_ECN;
1507         if (features & NETIF_F_GSO_GRE)
1508                 wanted |= IONIC_ETH_HW_TSO_GRE;
1509         if (features & NETIF_F_GSO_GRE_CSUM)
1510                 wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
1511         if (features & NETIF_F_GSO_IPXIP4)
1512                 wanted |= IONIC_ETH_HW_TSO_IPXIP4;
1513         if (features & NETIF_F_GSO_IPXIP6)
1514                 wanted |= IONIC_ETH_HW_TSO_IPXIP6;
1515         if (features & NETIF_F_GSO_UDP_TUNNEL)
1516                 wanted |= IONIC_ETH_HW_TSO_UDP;
1517         if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
1518                 wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
1519
1520         return cpu_to_le64(wanted);
1521 }
1522
1523 static int ionic_set_nic_features(struct ionic_lif *lif,
1524                                   netdev_features_t features)
1525 {
1526         struct device *dev = lif->ionic->dev;
1527         struct ionic_admin_ctx ctx = {
1528                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1529                 .cmd.lif_setattr = {
1530                         .opcode = IONIC_CMD_LIF_SETATTR,
1531                         .index = cpu_to_le16(lif->index),
1532                         .attr = IONIC_LIF_ATTR_FEATURES,
1533                 },
1534         };
1535         u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
1536                          IONIC_ETH_HW_VLAN_RX_STRIP |
1537                          IONIC_ETH_HW_VLAN_RX_FILTER;
1538         u64 old_hw_features;
1539         int err;
1540
1541         ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
1542
1543         if (lif->phc)
1544                 ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
1545
1546         err = ionic_adminq_post_wait(lif, &ctx);
1547         if (err)
1548                 return err;
1549
1550         old_hw_features = lif->hw_features;
1551         lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
1552                                        ctx.comp.lif_setattr.features);
1553
1554         if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
1555                 ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1556
1557         if ((vlan_flags & features) &&
1558             !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
1559                 dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1560
1561         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1562                 dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
1563         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1564                 dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
1565         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1566                 dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
1567         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1568                 dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
1569         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1570                 dev_dbg(dev, "feature ETH_HW_TX_SG\n");
1571         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1572                 dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
1573         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1574                 dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
1575         if (lif->hw_features & IONIC_ETH_HW_TSO)
1576                 dev_dbg(dev, "feature ETH_HW_TSO\n");
1577         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1578                 dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
1579         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1580                 dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
1581         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1582                 dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
1583         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1584                 dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
1585         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1586                 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
1587         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1588                 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
1589         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1590                 dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
1591         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1592                 dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
1593         if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
1594                 dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
1595
1596         return 0;
1597 }
1598
1599 static int ionic_init_nic_features(struct ionic_lif *lif)
1600 {
1601         struct net_device *netdev = lif->netdev;
1602         netdev_features_t features;
1603         int err;
1604
1605         /* set up what we expect to support by default */
1606         features = NETIF_F_HW_VLAN_CTAG_TX |
1607                    NETIF_F_HW_VLAN_CTAG_RX |
1608                    NETIF_F_HW_VLAN_CTAG_FILTER |
1609                    NETIF_F_RXHASH |
1610                    NETIF_F_SG |
1611                    NETIF_F_HW_CSUM |
1612                    NETIF_F_RXCSUM |
1613                    NETIF_F_TSO |
1614                    NETIF_F_TSO6 |
1615                    NETIF_F_TSO_ECN;
1616
1617         err = ionic_set_nic_features(lif, features);
1618         if (err)
1619                 return err;
1620
1621         /* tell the netdev what we actually can support */
1622         netdev->features |= NETIF_F_HIGHDMA;
1623
1624         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1625                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
1626         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1627                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1628         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1629                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1630         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1631                 netdev->hw_features |= NETIF_F_RXHASH;
1632         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1633                 netdev->hw_features |= NETIF_F_SG;
1634
1635         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1636                 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
1637         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1638                 netdev->hw_enc_features |= NETIF_F_RXCSUM;
1639         if (lif->hw_features & IONIC_ETH_HW_TSO)
1640                 netdev->hw_enc_features |= NETIF_F_TSO;
1641         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1642                 netdev->hw_enc_features |= NETIF_F_TSO6;
1643         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1644                 netdev->hw_enc_features |= NETIF_F_TSO_ECN;
1645         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1646                 netdev->hw_enc_features |= NETIF_F_GSO_GRE;
1647         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1648                 netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
1649         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1650                 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
1651         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1652                 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
1653         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1654                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
1655         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1656                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1657
1658         netdev->hw_features |= netdev->hw_enc_features;
1659         netdev->features |= netdev->hw_features;
1660         netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
1661
1662         netdev->priv_flags |= IFF_UNICAST_FLT |
1663                               IFF_LIVE_ADDR_CHANGE;
1664
1665         return 0;
1666 }
1667
1668 static int ionic_set_features(struct net_device *netdev,
1669                               netdev_features_t features)
1670 {
1671         struct ionic_lif *lif = netdev_priv(netdev);
1672         int err;
1673
1674         netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1675                    __func__, (u64)lif->netdev->features, (u64)features);
1676
1677         err = ionic_set_nic_features(lif, features);
1678
1679         return err;
1680 }
1681
1682 static int ionic_set_mac_address(struct net_device *netdev, void *sa)
1683 {
1684         struct sockaddr *addr = sa;
1685         u8 *mac;
1686         int err;
1687
1688         mac = (u8 *)addr->sa_data;
1689         if (ether_addr_equal(netdev->dev_addr, mac))
1690                 return 0;
1691
1692         err = eth_prepare_mac_addr_change(netdev, addr);
1693         if (err)
1694                 return err;
1695
1696         if (!is_zero_ether_addr(netdev->dev_addr)) {
1697                 netdev_info(netdev, "deleting mac addr %pM\n",
1698                             netdev->dev_addr);
1699                 ionic_addr_del(netdev, netdev->dev_addr);
1700         }
1701
1702         eth_commit_mac_addr_change(netdev, addr);
1703         netdev_info(netdev, "updating mac addr %pM\n", mac);
1704
1705         return ionic_addr_add(netdev, mac);
1706 }
1707
1708 static void ionic_stop_queues_reconfig(struct ionic_lif *lif)
1709 {
1710         /* Stop and clean the queues before reconfiguration */
1711         mutex_lock(&lif->queue_lock);
1712         netif_device_detach(lif->netdev);
1713         ionic_stop_queues(lif);
1714         ionic_txrx_deinit(lif);
1715 }
1716
1717 static int ionic_start_queues_reconfig(struct ionic_lif *lif)
1718 {
1719         int err;
1720
1721         /* Re-init the queues after reconfiguration */
1722
1723         /* The only way txrx_init can fail here is if communication
1724          * with FW is suddenly broken.  There's not much we can do
1725          * at this point - error messages have already been printed,
1726          * so we can continue on and the user can eventually do a
1727          * DOWN and UP to try to reset and clear the issue.
1728          */
1729         err = ionic_txrx_init(lif);
1730         mutex_unlock(&lif->queue_lock);
1731         ionic_link_status_check_request(lif, CAN_SLEEP);
1732         netif_device_attach(lif->netdev);
1733
1734         return err;
1735 }
1736
1737 static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
1738 {
1739         struct ionic_lif *lif = netdev_priv(netdev);
1740         struct ionic_admin_ctx ctx = {
1741                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1742                 .cmd.lif_setattr = {
1743                         .opcode = IONIC_CMD_LIF_SETATTR,
1744                         .index = cpu_to_le16(lif->index),
1745                         .attr = IONIC_LIF_ATTR_MTU,
1746                         .mtu = cpu_to_le32(new_mtu),
1747                 },
1748         };
1749         int err;
1750
1751         err = ionic_adminq_post_wait(lif, &ctx);
1752         if (err)
1753                 return err;
1754
1755         /* if we're not running, nothing more to do */
1756         if (!netif_running(netdev)) {
1757                 netdev->mtu = new_mtu;
1758                 return 0;
1759         }
1760
1761         ionic_stop_queues_reconfig(lif);
1762         netdev->mtu = new_mtu;
1763         return ionic_start_queues_reconfig(lif);
1764 }
1765
1766 static void ionic_tx_timeout_work(struct work_struct *ws)
1767 {
1768         struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
1769
1770         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
1771                 return;
1772
1773         /* if we were stopped before this scheduled job was launched,
1774          * don't bother the queues as they are already stopped.
1775          */
1776         if (!netif_running(lif->netdev))
1777                 return;
1778
1779         ionic_stop_queues_reconfig(lif);
1780         ionic_start_queues_reconfig(lif);
1781 }
1782
1783 static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1784 {
1785         struct ionic_lif *lif = netdev_priv(netdev);
1786
1787         netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
1788         schedule_work(&lif->tx_timeout_work);
1789 }
1790
1791 static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1792                                  u16 vid)
1793 {
1794         struct ionic_lif *lif = netdev_priv(netdev);
1795         struct ionic_admin_ctx ctx = {
1796                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1797                 .cmd.rx_filter_add = {
1798                         .opcode = IONIC_CMD_RX_FILTER_ADD,
1799                         .lif_index = cpu_to_le16(lif->index),
1800                         .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_VLAN),
1801                         .vlan.vlan = cpu_to_le16(vid),
1802                 },
1803         };
1804         int err;
1805
1806         netdev_dbg(netdev, "rx_filter add VLAN %d\n", vid);
1807         err = ionic_adminq_post_wait(lif, &ctx);
1808         if (err)
1809                 return err;
1810
1811         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx);
1812 }
1813
1814 static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1815                                   u16 vid)
1816 {
1817         struct ionic_lif *lif = netdev_priv(netdev);
1818         struct ionic_admin_ctx ctx = {
1819                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1820                 .cmd.rx_filter_del = {
1821                         .opcode = IONIC_CMD_RX_FILTER_DEL,
1822                         .lif_index = cpu_to_le16(lif->index),
1823                 },
1824         };
1825         struct ionic_rx_filter *f;
1826
1827         spin_lock_bh(&lif->rx_filters.lock);
1828
1829         f = ionic_rx_filter_by_vlan(lif, vid);
1830         if (!f) {
1831                 spin_unlock_bh(&lif->rx_filters.lock);
1832                 return -ENOENT;
1833         }
1834
1835         netdev_dbg(netdev, "rx_filter del VLAN %d (id %d)\n",
1836                    vid, f->filter_id);
1837
1838         ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1839         ionic_rx_filter_free(lif, f);
1840         spin_unlock_bh(&lif->rx_filters.lock);
1841
1842         return ionic_adminq_post_wait(lif, &ctx);
1843 }
1844
1845 int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
1846                          const u8 *key, const u32 *indir)
1847 {
1848         struct ionic_admin_ctx ctx = {
1849                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1850                 .cmd.lif_setattr = {
1851                         .opcode = IONIC_CMD_LIF_SETATTR,
1852                         .attr = IONIC_LIF_ATTR_RSS,
1853                         .rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
1854                 },
1855         };
1856         unsigned int i, tbl_sz;
1857
1858         if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
1859                 lif->rss_types = types;
1860                 ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
1861         }
1862
1863         if (key)
1864                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
1865
1866         if (indir) {
1867                 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1868                 for (i = 0; i < tbl_sz; i++)
1869                         lif->rss_ind_tbl[i] = indir[i];
1870         }
1871
1872         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
1873                IONIC_RSS_HASH_KEY_SIZE);
1874
1875         return ionic_adminq_post_wait(lif, &ctx);
1876 }
1877
1878 static int ionic_lif_rss_init(struct ionic_lif *lif)
1879 {
1880         unsigned int tbl_sz;
1881         unsigned int i;
1882
1883         lif->rss_types = IONIC_RSS_TYPE_IPV4     |
1884                          IONIC_RSS_TYPE_IPV4_TCP |
1885                          IONIC_RSS_TYPE_IPV4_UDP |
1886                          IONIC_RSS_TYPE_IPV6     |
1887                          IONIC_RSS_TYPE_IPV6_TCP |
1888                          IONIC_RSS_TYPE_IPV6_UDP;
1889
1890         /* Fill indirection table with 'default' values */
1891         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1892         for (i = 0; i < tbl_sz; i++)
1893                 lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
1894
1895         return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1896 }
1897
1898 static void ionic_lif_rss_deinit(struct ionic_lif *lif)
1899 {
1900         int tbl_sz;
1901
1902         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1903         memset(lif->rss_ind_tbl, 0, tbl_sz);
1904         memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
1905
1906         ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1907 }
1908
1909 static void ionic_lif_quiesce(struct ionic_lif *lif)
1910 {
1911         struct ionic_admin_ctx ctx = {
1912                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1913                 .cmd.lif_setattr = {
1914                         .opcode = IONIC_CMD_LIF_SETATTR,
1915                         .index = cpu_to_le16(lif->index),
1916                         .attr = IONIC_LIF_ATTR_STATE,
1917                         .state = IONIC_LIF_QUIESCE,
1918                 },
1919         };
1920         int err;
1921
1922         err = ionic_adminq_post_wait(lif, &ctx);
1923         if (err)
1924                 netdev_err(lif->netdev, "lif quiesce failed %d\n", err);
1925 }
1926
1927 static void ionic_txrx_disable(struct ionic_lif *lif)
1928 {
1929         unsigned int i;
1930         int err = 0;
1931
1932         if (lif->txqcqs) {
1933                 for (i = 0; i < lif->nxqs; i++)
1934                         err = ionic_qcq_disable(lif->txqcqs[i], (err != -ETIMEDOUT));
1935         }
1936
1937         if (lif->hwstamp_txq)
1938                 err = ionic_qcq_disable(lif->hwstamp_txq, (err != -ETIMEDOUT));
1939
1940         if (lif->rxqcqs) {
1941                 for (i = 0; i < lif->nxqs; i++)
1942                         err = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
1943         }
1944
1945         if (lif->hwstamp_rxq)
1946                 err = ionic_qcq_disable(lif->hwstamp_rxq, (err != -ETIMEDOUT));
1947
1948         ionic_lif_quiesce(lif);
1949 }
1950
1951 static void ionic_txrx_deinit(struct ionic_lif *lif)
1952 {
1953         unsigned int i;
1954
1955         if (lif->txqcqs) {
1956                 for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
1957                         ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1958                         ionic_tx_flush(&lif->txqcqs[i]->cq);
1959                         ionic_tx_empty(&lif->txqcqs[i]->q);
1960                 }
1961         }
1962
1963         if (lif->rxqcqs) {
1964                 for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
1965                         ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
1966                         ionic_rx_empty(&lif->rxqcqs[i]->q);
1967                 }
1968         }
1969         lif->rx_mode = 0;
1970
1971         if (lif->hwstamp_txq) {
1972                 ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
1973                 ionic_tx_flush(&lif->hwstamp_txq->cq);
1974                 ionic_tx_empty(&lif->hwstamp_txq->q);
1975         }
1976
1977         if (lif->hwstamp_rxq) {
1978                 ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
1979                 ionic_rx_empty(&lif->hwstamp_rxq->q);
1980         }
1981 }
1982
1983 static void ionic_txrx_free(struct ionic_lif *lif)
1984 {
1985         unsigned int i;
1986
1987         if (lif->txqcqs) {
1988                 for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
1989                         ionic_qcq_free(lif, lif->txqcqs[i]);
1990                         devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
1991                         lif->txqcqs[i] = NULL;
1992                 }
1993         }
1994
1995         if (lif->rxqcqs) {
1996                 for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
1997                         ionic_qcq_free(lif, lif->rxqcqs[i]);
1998                         devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
1999                         lif->rxqcqs[i] = NULL;
2000                 }
2001         }
2002
2003         if (lif->hwstamp_txq) {
2004                 ionic_qcq_free(lif, lif->hwstamp_txq);
2005                 devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
2006                 lif->hwstamp_txq = NULL;
2007         }
2008
2009         if (lif->hwstamp_rxq) {
2010                 ionic_qcq_free(lif, lif->hwstamp_rxq);
2011                 devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
2012                 lif->hwstamp_rxq = NULL;
2013         }
2014 }
2015
2016 static int ionic_txrx_alloc(struct ionic_lif *lif)
2017 {
2018         unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2019         unsigned int flags, i;
2020         int err = 0;
2021
2022         num_desc = lif->ntxq_descs;
2023         desc_sz = sizeof(struct ionic_txq_desc);
2024         comp_sz = sizeof(struct ionic_txq_comp);
2025
2026         if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2027             lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2028                                           sizeof(struct ionic_txq_sg_desc_v1))
2029                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2030         else
2031                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2032
2033         flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2034         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2035                 flags |= IONIC_QCQ_F_INTR;
2036         for (i = 0; i < lif->nxqs; i++) {
2037                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2038                                       num_desc, desc_sz, comp_sz, sg_desc_sz,
2039                                       lif->kern_pid, &lif->txqcqs[i]);
2040                 if (err)
2041                         goto err_out;
2042
2043                 if (flags & IONIC_QCQ_F_INTR) {
2044                         ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2045                                              lif->txqcqs[i]->intr.index,
2046                                              lif->tx_coalesce_hw);
2047                         if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2048                                 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2049                 }
2050
2051                 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2052         }
2053
2054         flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
2055
2056         num_desc = lif->nrxq_descs;
2057         desc_sz = sizeof(struct ionic_rxq_desc);
2058         comp_sz = sizeof(struct ionic_rxq_comp);
2059         sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2060
2061         if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2062                 comp_sz *= 2;
2063
2064         for (i = 0; i < lif->nxqs; i++) {
2065                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2066                                       num_desc, desc_sz, comp_sz, sg_desc_sz,
2067                                       lif->kern_pid, &lif->rxqcqs[i]);
2068                 if (err)
2069                         goto err_out;
2070
2071                 lif->rxqcqs[i]->q.features = lif->rxq_features;
2072
2073                 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2074                                      lif->rxqcqs[i]->intr.index,
2075                                      lif->rx_coalesce_hw);
2076                 if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
2077                         lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
2078
2079                 if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2080                         ionic_link_qcq_interrupts(lif->rxqcqs[i],
2081                                                   lif->txqcqs[i]);
2082
2083                 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2084         }
2085
2086         return 0;
2087
2088 err_out:
2089         ionic_txrx_free(lif);
2090
2091         return err;
2092 }
2093
2094 static int ionic_txrx_init(struct ionic_lif *lif)
2095 {
2096         unsigned int i;
2097         int err;
2098
2099         for (i = 0; i < lif->nxqs; i++) {
2100                 err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
2101                 if (err)
2102                         goto err_out;
2103
2104                 err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
2105                 if (err) {
2106                         ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2107                         goto err_out;
2108                 }
2109         }
2110
2111         if (lif->netdev->features & NETIF_F_RXHASH)
2112                 ionic_lif_rss_init(lif);
2113
2114         ionic_set_rx_mode(lif->netdev, CAN_SLEEP);
2115
2116         return 0;
2117
2118 err_out:
2119         while (i--) {
2120                 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2121                 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2122         }
2123
2124         return err;
2125 }
2126
2127 static int ionic_txrx_enable(struct ionic_lif *lif)
2128 {
2129         int derr = 0;
2130         int i, err;
2131
2132         for (i = 0; i < lif->nxqs; i++) {
2133                 if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
2134                         dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
2135                         err = -ENXIO;
2136                         goto err_out;
2137                 }
2138
2139                 ionic_rx_fill(&lif->rxqcqs[i]->q);
2140                 err = ionic_qcq_enable(lif->rxqcqs[i]);
2141                 if (err)
2142                         goto err_out;
2143
2144                 err = ionic_qcq_enable(lif->txqcqs[i]);
2145                 if (err) {
2146                         derr = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
2147                         goto err_out;
2148                 }
2149         }
2150
2151         if (lif->hwstamp_rxq) {
2152                 ionic_rx_fill(&lif->hwstamp_rxq->q);
2153                 err = ionic_qcq_enable(lif->hwstamp_rxq);
2154                 if (err)
2155                         goto err_out_hwstamp_rx;
2156         }
2157
2158         if (lif->hwstamp_txq) {
2159                 err = ionic_qcq_enable(lif->hwstamp_txq);
2160                 if (err)
2161                         goto err_out_hwstamp_tx;
2162         }
2163
2164         return 0;
2165
2166 err_out_hwstamp_tx:
2167         if (lif->hwstamp_rxq)
2168                 derr = ionic_qcq_disable(lif->hwstamp_rxq, (derr != -ETIMEDOUT));
2169 err_out_hwstamp_rx:
2170         i = lif->nxqs;
2171 err_out:
2172         while (i--) {
2173                 derr = ionic_qcq_disable(lif->txqcqs[i], (derr != -ETIMEDOUT));
2174                 derr = ionic_qcq_disable(lif->rxqcqs[i], (derr != -ETIMEDOUT));
2175         }
2176
2177         return err;
2178 }
2179
2180 static int ionic_start_queues(struct ionic_lif *lif)
2181 {
2182         int err;
2183
2184         if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
2185                 return -EIO;
2186
2187         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2188                 return -EBUSY;
2189
2190         if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
2191                 return 0;
2192
2193         err = ionic_txrx_enable(lif);
2194         if (err) {
2195                 clear_bit(IONIC_LIF_F_UP, lif->state);
2196                 return err;
2197         }
2198         netif_tx_wake_all_queues(lif->netdev);
2199
2200         return 0;
2201 }
2202
2203 static int ionic_open(struct net_device *netdev)
2204 {
2205         struct ionic_lif *lif = netdev_priv(netdev);
2206         int err;
2207
2208         /* If recovering from a broken state, clear the bit and we'll try again */
2209         if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
2210                 netdev_info(netdev, "clearing broken state\n");
2211
2212         err = ionic_txrx_alloc(lif);
2213         if (err)
2214                 return err;
2215
2216         err = ionic_txrx_init(lif);
2217         if (err)
2218                 goto err_txrx_free;
2219
2220         err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
2221         if (err)
2222                 goto err_txrx_deinit;
2223
2224         err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
2225         if (err)
2226                 goto err_txrx_deinit;
2227
2228         /* don't start the queues until we have link */
2229         if (netif_carrier_ok(netdev)) {
2230                 err = ionic_start_queues(lif);
2231                 if (err)
2232                         goto err_txrx_deinit;
2233         }
2234
2235         return 0;
2236
2237 err_txrx_deinit:
2238         ionic_txrx_deinit(lif);
2239 err_txrx_free:
2240         ionic_txrx_free(lif);
2241         return err;
2242 }
2243
2244 static void ionic_stop_queues(struct ionic_lif *lif)
2245 {
2246         if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
2247                 return;
2248
2249         netif_tx_disable(lif->netdev);
2250         ionic_txrx_disable(lif);
2251 }
2252
2253 static int ionic_stop(struct net_device *netdev)
2254 {
2255         struct ionic_lif *lif = netdev_priv(netdev);
2256
2257         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2258                 return 0;
2259
2260         ionic_stop_queues(lif);
2261         ionic_txrx_deinit(lif);
2262         ionic_txrx_free(lif);
2263
2264         return 0;
2265 }
2266
2267 static int ionic_do_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2268 {
2269         struct ionic_lif *lif = netdev_priv(netdev);
2270
2271         switch (cmd) {
2272         case SIOCSHWTSTAMP:
2273                 return ionic_lif_hwstamp_set(lif, ifr);
2274         case SIOCGHWTSTAMP:
2275                 return ionic_lif_hwstamp_get(lif, ifr);
2276         default:
2277                 return -EOPNOTSUPP;
2278         }
2279 }
2280
2281 static int ionic_get_vf_config(struct net_device *netdev,
2282                                int vf, struct ifla_vf_info *ivf)
2283 {
2284         struct ionic_lif *lif = netdev_priv(netdev);
2285         struct ionic *ionic = lif->ionic;
2286         int ret = 0;
2287
2288         if (!netif_device_present(netdev))
2289                 return -EBUSY;
2290
2291         down_read(&ionic->vf_op_lock);
2292
2293         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2294                 ret = -EINVAL;
2295         } else {
2296                 ivf->vf           = vf;
2297                 ivf->vlan         = le16_to_cpu(ionic->vfs[vf].vlanid);
2298                 ivf->qos          = 0;
2299                 ivf->spoofchk     = ionic->vfs[vf].spoofchk;
2300                 ivf->linkstate    = ionic->vfs[vf].linkstate;
2301                 ivf->max_tx_rate  = le32_to_cpu(ionic->vfs[vf].maxrate);
2302                 ivf->trusted      = ionic->vfs[vf].trusted;
2303                 ether_addr_copy(ivf->mac, ionic->vfs[vf].macaddr);
2304         }
2305
2306         up_read(&ionic->vf_op_lock);
2307         return ret;
2308 }
2309
2310 static int ionic_get_vf_stats(struct net_device *netdev, int vf,
2311                               struct ifla_vf_stats *vf_stats)
2312 {
2313         struct ionic_lif *lif = netdev_priv(netdev);
2314         struct ionic *ionic = lif->ionic;
2315         struct ionic_lif_stats *vs;
2316         int ret = 0;
2317
2318         if (!netif_device_present(netdev))
2319                 return -EBUSY;
2320
2321         down_read(&ionic->vf_op_lock);
2322
2323         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2324                 ret = -EINVAL;
2325         } else {
2326                 memset(vf_stats, 0, sizeof(*vf_stats));
2327                 vs = &ionic->vfs[vf].stats;
2328
2329                 vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
2330                 vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
2331                 vf_stats->rx_bytes   = le64_to_cpu(vs->rx_ucast_bytes);
2332                 vf_stats->tx_bytes   = le64_to_cpu(vs->tx_ucast_bytes);
2333                 vf_stats->broadcast  = le64_to_cpu(vs->rx_bcast_packets);
2334                 vf_stats->multicast  = le64_to_cpu(vs->rx_mcast_packets);
2335                 vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
2336                                        le64_to_cpu(vs->rx_mcast_drop_packets) +
2337                                        le64_to_cpu(vs->rx_bcast_drop_packets);
2338                 vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
2339                                        le64_to_cpu(vs->tx_mcast_drop_packets) +
2340                                        le64_to_cpu(vs->tx_bcast_drop_packets);
2341         }
2342
2343         up_read(&ionic->vf_op_lock);
2344         return ret;
2345 }
2346
2347 static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2348 {
2349         struct ionic_lif *lif = netdev_priv(netdev);
2350         struct ionic *ionic = lif->ionic;
2351         int ret;
2352
2353         if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
2354                 return -EINVAL;
2355
2356         if (!netif_device_present(netdev))
2357                 return -EBUSY;
2358
2359         down_write(&ionic->vf_op_lock);
2360
2361         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2362                 ret = -EINVAL;
2363         } else {
2364                 ret = ionic_set_vf_config(ionic, vf, IONIC_VF_ATTR_MAC, mac);
2365                 if (!ret)
2366                         ether_addr_copy(ionic->vfs[vf].macaddr, mac);
2367         }
2368
2369         up_write(&ionic->vf_op_lock);
2370         return ret;
2371 }
2372
2373 static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2374                              u8 qos, __be16 proto)
2375 {
2376         struct ionic_lif *lif = netdev_priv(netdev);
2377         struct ionic *ionic = lif->ionic;
2378         int ret;
2379
2380         /* until someday when we support qos */
2381         if (qos)
2382                 return -EINVAL;
2383
2384         if (vlan > 4095)
2385                 return -EINVAL;
2386
2387         if (proto != htons(ETH_P_8021Q))
2388                 return -EPROTONOSUPPORT;
2389
2390         if (!netif_device_present(netdev))
2391                 return -EBUSY;
2392
2393         down_write(&ionic->vf_op_lock);
2394
2395         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2396                 ret = -EINVAL;
2397         } else {
2398                 ret = ionic_set_vf_config(ionic, vf,
2399                                           IONIC_VF_ATTR_VLAN, (u8 *)&vlan);
2400                 if (!ret)
2401                         ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
2402         }
2403
2404         up_write(&ionic->vf_op_lock);
2405         return ret;
2406 }
2407
2408 static int ionic_set_vf_rate(struct net_device *netdev, int vf,
2409                              int tx_min, int tx_max)
2410 {
2411         struct ionic_lif *lif = netdev_priv(netdev);
2412         struct ionic *ionic = lif->ionic;
2413         int ret;
2414
2415         /* setting the min just seems silly */
2416         if (tx_min)
2417                 return -EINVAL;
2418
2419         if (!netif_device_present(netdev))
2420                 return -EBUSY;
2421
2422         down_write(&ionic->vf_op_lock);
2423
2424         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2425                 ret = -EINVAL;
2426         } else {
2427                 ret = ionic_set_vf_config(ionic, vf,
2428                                           IONIC_VF_ATTR_RATE, (u8 *)&tx_max);
2429                 if (!ret)
2430                         lif->ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
2431         }
2432
2433         up_write(&ionic->vf_op_lock);
2434         return ret;
2435 }
2436
2437 static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
2438 {
2439         struct ionic_lif *lif = netdev_priv(netdev);
2440         struct ionic *ionic = lif->ionic;
2441         u8 data = set;  /* convert to u8 for config */
2442         int ret;
2443
2444         if (!netif_device_present(netdev))
2445                 return -EBUSY;
2446
2447         down_write(&ionic->vf_op_lock);
2448
2449         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2450                 ret = -EINVAL;
2451         } else {
2452                 ret = ionic_set_vf_config(ionic, vf,
2453                                           IONIC_VF_ATTR_SPOOFCHK, &data);
2454                 if (!ret)
2455                         ionic->vfs[vf].spoofchk = data;
2456         }
2457
2458         up_write(&ionic->vf_op_lock);
2459         return ret;
2460 }
2461
2462 static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
2463 {
2464         struct ionic_lif *lif = netdev_priv(netdev);
2465         struct ionic *ionic = lif->ionic;
2466         u8 data = set;  /* convert to u8 for config */
2467         int ret;
2468
2469         if (!netif_device_present(netdev))
2470                 return -EBUSY;
2471
2472         down_write(&ionic->vf_op_lock);
2473
2474         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2475                 ret = -EINVAL;
2476         } else {
2477                 ret = ionic_set_vf_config(ionic, vf,
2478                                           IONIC_VF_ATTR_TRUST, &data);
2479                 if (!ret)
2480                         ionic->vfs[vf].trusted = data;
2481         }
2482
2483         up_write(&ionic->vf_op_lock);
2484         return ret;
2485 }
2486
2487 static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
2488 {
2489         struct ionic_lif *lif = netdev_priv(netdev);
2490         struct ionic *ionic = lif->ionic;
2491         u8 data;
2492         int ret;
2493
2494         switch (set) {
2495         case IFLA_VF_LINK_STATE_ENABLE:
2496                 data = IONIC_VF_LINK_STATUS_UP;
2497                 break;
2498         case IFLA_VF_LINK_STATE_DISABLE:
2499                 data = IONIC_VF_LINK_STATUS_DOWN;
2500                 break;
2501         case IFLA_VF_LINK_STATE_AUTO:
2502                 data = IONIC_VF_LINK_STATUS_AUTO;
2503                 break;
2504         default:
2505                 return -EINVAL;
2506         }
2507
2508         if (!netif_device_present(netdev))
2509                 return -EBUSY;
2510
2511         down_write(&ionic->vf_op_lock);
2512
2513         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2514                 ret = -EINVAL;
2515         } else {
2516                 ret = ionic_set_vf_config(ionic, vf,
2517                                           IONIC_VF_ATTR_LINKSTATE, &data);
2518                 if (!ret)
2519                         ionic->vfs[vf].linkstate = set;
2520         }
2521
2522         up_write(&ionic->vf_op_lock);
2523         return ret;
2524 }
2525
2526 static const struct net_device_ops ionic_netdev_ops = {
2527         .ndo_open               = ionic_open,
2528         .ndo_stop               = ionic_stop,
2529         .ndo_do_ioctl           = ionic_do_ioctl,
2530         .ndo_start_xmit         = ionic_start_xmit,
2531         .ndo_get_stats64        = ionic_get_stats64,
2532         .ndo_set_rx_mode        = ionic_ndo_set_rx_mode,
2533         .ndo_set_features       = ionic_set_features,
2534         .ndo_set_mac_address    = ionic_set_mac_address,
2535         .ndo_validate_addr      = eth_validate_addr,
2536         .ndo_tx_timeout         = ionic_tx_timeout,
2537         .ndo_change_mtu         = ionic_change_mtu,
2538         .ndo_vlan_rx_add_vid    = ionic_vlan_rx_add_vid,
2539         .ndo_vlan_rx_kill_vid   = ionic_vlan_rx_kill_vid,
2540         .ndo_set_vf_vlan        = ionic_set_vf_vlan,
2541         .ndo_set_vf_trust       = ionic_set_vf_trust,
2542         .ndo_set_vf_mac         = ionic_set_vf_mac,
2543         .ndo_set_vf_rate        = ionic_set_vf_rate,
2544         .ndo_set_vf_spoofchk    = ionic_set_vf_spoofchk,
2545         .ndo_get_vf_config      = ionic_get_vf_config,
2546         .ndo_set_vf_link_state  = ionic_set_vf_link_state,
2547         .ndo_get_vf_stats       = ionic_get_vf_stats,
2548 };
2549
2550 static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2551 {
2552         /* only swapping the queues, not the napi, flags, or other stuff */
2553         swap(a->q.features,   b->q.features);
2554         swap(a->q.num_descs,  b->q.num_descs);
2555         swap(a->q.desc_size,  b->q.desc_size);
2556         swap(a->q.base,       b->q.base);
2557         swap(a->q.base_pa,    b->q.base_pa);
2558         swap(a->q.info,       b->q.info);
2559         swap(a->q_base,       b->q_base);
2560         swap(a->q_base_pa,    b->q_base_pa);
2561         swap(a->q_size,       b->q_size);
2562
2563         swap(a->q.sg_desc_size, b->q.sg_desc_size);
2564         swap(a->q.sg_base,    b->q.sg_base);
2565         swap(a->q.sg_base_pa, b->q.sg_base_pa);
2566         swap(a->sg_base,      b->sg_base);
2567         swap(a->sg_base_pa,   b->sg_base_pa);
2568         swap(a->sg_size,      b->sg_size);
2569
2570         swap(a->cq.num_descs, b->cq.num_descs);
2571         swap(a->cq.desc_size, b->cq.desc_size);
2572         swap(a->cq.base,      b->cq.base);
2573         swap(a->cq.base_pa,   b->cq.base_pa);
2574         swap(a->cq.info,      b->cq.info);
2575         swap(a->cq_base,      b->cq_base);
2576         swap(a->cq_base_pa,   b->cq_base_pa);
2577         swap(a->cq_size,      b->cq_size);
2578
2579         ionic_debugfs_del_qcq(a);
2580         ionic_debugfs_add_qcq(a->q.lif, a);
2581 }
2582
2583 int ionic_reconfigure_queues(struct ionic_lif *lif,
2584                              struct ionic_queue_params *qparam)
2585 {
2586         unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2587         struct ionic_qcq **tx_qcqs = NULL;
2588         struct ionic_qcq **rx_qcqs = NULL;
2589         unsigned int flags, i;
2590         int err = -ENOMEM;
2591
2592         /* allocate temporary qcq arrays to hold new queue structs */
2593         if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
2594                 tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
2595                                        sizeof(struct ionic_qcq *), GFP_KERNEL);
2596                 if (!tx_qcqs)
2597                         goto err_out;
2598         }
2599         if (qparam->nxqs != lif->nxqs ||
2600             qparam->nrxq_descs != lif->nrxq_descs ||
2601             qparam->rxq_features != lif->rxq_features) {
2602                 rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
2603                                        sizeof(struct ionic_qcq *), GFP_KERNEL);
2604                 if (!rx_qcqs)
2605                         goto err_out;
2606         }
2607
2608         /* allocate new desc_info and rings, but leave the interrupt setup
2609          * until later so as to not mess with the still-running queues
2610          */
2611         if (tx_qcqs) {
2612                 num_desc = qparam->ntxq_descs;
2613                 desc_sz = sizeof(struct ionic_txq_desc);
2614                 comp_sz = sizeof(struct ionic_txq_comp);
2615
2616                 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2617                     lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2618                     sizeof(struct ionic_txq_sg_desc_v1))
2619                         sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2620                 else
2621                         sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2622
2623                 for (i = 0; i < qparam->nxqs; i++) {
2624                         flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2625                         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2626                                               num_desc, desc_sz, comp_sz, sg_desc_sz,
2627                                               lif->kern_pid, &tx_qcqs[i]);
2628                         if (err)
2629                                 goto err_out;
2630                 }
2631         }
2632
2633         if (rx_qcqs) {
2634                 num_desc = qparam->nrxq_descs;
2635                 desc_sz = sizeof(struct ionic_rxq_desc);
2636                 comp_sz = sizeof(struct ionic_rxq_comp);
2637                 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2638
2639                 if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2640                         comp_sz *= 2;
2641
2642                 for (i = 0; i < qparam->nxqs; i++) {
2643                         flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2644                         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2645                                               num_desc, desc_sz, comp_sz, sg_desc_sz,
2646                                               lif->kern_pid, &rx_qcqs[i]);
2647                         if (err)
2648                                 goto err_out;
2649
2650                         rx_qcqs[i]->q.features = qparam->rxq_features;
2651                 }
2652         }
2653
2654         /* stop and clean the queues */
2655         ionic_stop_queues_reconfig(lif);
2656
2657         if (qparam->nxqs != lif->nxqs) {
2658                 err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
2659                 if (err)
2660                         goto err_out_reinit_unlock;
2661                 err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
2662                 if (err) {
2663                         netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
2664                         goto err_out_reinit_unlock;
2665                 }
2666         }
2667
2668         /* swap new desc_info and rings, keeping existing interrupt config */
2669         if (tx_qcqs) {
2670                 lif->ntxq_descs = qparam->ntxq_descs;
2671                 for (i = 0; i < qparam->nxqs; i++)
2672                         ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
2673         }
2674
2675         if (rx_qcqs) {
2676                 lif->nrxq_descs = qparam->nrxq_descs;
2677                 for (i = 0; i < qparam->nxqs; i++)
2678                         ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
2679         }
2680
2681         /* if we need to change the interrupt layout, this is the time */
2682         if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
2683             qparam->nxqs != lif->nxqs) {
2684                 if (qparam->intr_split) {
2685                         set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2686                 } else {
2687                         clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2688                         lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2689                         lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2690                 }
2691
2692                 /* clear existing interrupt assignments */
2693                 for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
2694                         ionic_qcq_intr_free(lif, lif->txqcqs[i]);
2695                         ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
2696                 }
2697
2698                 /* re-assign the interrupts */
2699                 for (i = 0; i < qparam->nxqs; i++) {
2700                         lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2701                         err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
2702                         ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2703                                              lif->rxqcqs[i]->intr.index,
2704                                              lif->rx_coalesce_hw);
2705
2706                         if (qparam->intr_split) {
2707                                 lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2708                                 err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
2709                                 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2710                                                      lif->txqcqs[i]->intr.index,
2711                                                      lif->tx_coalesce_hw);
2712                                 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2713                                         lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2714                         } else {
2715                                 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2716                                 ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
2717                         }
2718                 }
2719         }
2720
2721         /* now we can rework the debugfs mappings */
2722         if (tx_qcqs) {
2723                 for (i = 0; i < qparam->nxqs; i++) {
2724                         ionic_debugfs_del_qcq(lif->txqcqs[i]);
2725                         ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2726                 }
2727         }
2728
2729         if (rx_qcqs) {
2730                 for (i = 0; i < qparam->nxqs; i++) {
2731                         ionic_debugfs_del_qcq(lif->rxqcqs[i]);
2732                         ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2733                 }
2734         }
2735
2736         swap(lif->nxqs, qparam->nxqs);
2737         swap(lif->rxq_features, qparam->rxq_features);
2738
2739 err_out_reinit_unlock:
2740         /* re-init the queues, but don't lose an error code */
2741         if (err)
2742                 ionic_start_queues_reconfig(lif);
2743         else
2744                 err = ionic_start_queues_reconfig(lif);
2745
2746 err_out:
2747         /* free old allocs without cleaning intr */
2748         for (i = 0; i < qparam->nxqs; i++) {
2749                 if (tx_qcqs && tx_qcqs[i]) {
2750                         tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2751                         ionic_qcq_free(lif, tx_qcqs[i]);
2752                         devm_kfree(lif->ionic->dev, tx_qcqs[i]);
2753                         tx_qcqs[i] = NULL;
2754                 }
2755                 if (rx_qcqs && rx_qcqs[i]) {
2756                         rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2757                         ionic_qcq_free(lif, rx_qcqs[i]);
2758                         devm_kfree(lif->ionic->dev, rx_qcqs[i]);
2759                         rx_qcqs[i] = NULL;
2760                 }
2761         }
2762
2763         /* free q array */
2764         if (rx_qcqs) {
2765                 devm_kfree(lif->ionic->dev, rx_qcqs);
2766                 rx_qcqs = NULL;
2767         }
2768         if (tx_qcqs) {
2769                 devm_kfree(lif->ionic->dev, tx_qcqs);
2770                 tx_qcqs = NULL;
2771         }
2772
2773         /* clean the unused dma and info allocations when new set is smaller
2774          * than the full array, but leave the qcq shells in place
2775          */
2776         for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
2777                 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2778                 ionic_qcq_free(lif, lif->txqcqs[i]);
2779
2780                 lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2781                 ionic_qcq_free(lif, lif->rxqcqs[i]);
2782         }
2783
2784         return err;
2785 }
2786
2787 int ionic_lif_alloc(struct ionic *ionic)
2788 {
2789         struct device *dev = ionic->dev;
2790         union ionic_lif_identity *lid;
2791         struct net_device *netdev;
2792         struct ionic_lif *lif;
2793         int tbl_sz;
2794         int err;
2795
2796         lid = kzalloc(sizeof(*lid), GFP_KERNEL);
2797         if (!lid)
2798                 return -ENOMEM;
2799
2800         netdev = alloc_etherdev_mqs(sizeof(*lif),
2801                                     ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
2802         if (!netdev) {
2803                 dev_err(dev, "Cannot allocate netdev, aborting\n");
2804                 err = -ENOMEM;
2805                 goto err_out_free_lid;
2806         }
2807
2808         SET_NETDEV_DEV(netdev, dev);
2809
2810         lif = netdev_priv(netdev);
2811         lif->netdev = netdev;
2812         ionic->lif = lif;
2813         netdev->netdev_ops = &ionic_netdev_ops;
2814         ionic_ethtool_set_ops(netdev);
2815
2816         netdev->watchdog_timeo = 2 * HZ;
2817         netif_carrier_off(netdev);
2818
2819         lif->identity = lid;
2820         lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
2821         err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
2822         if (err) {
2823                 dev_err(ionic->dev, "Cannot identify type %d: %d\n",
2824                         lif->lif_type, err);
2825                 goto err_out_free_netdev;
2826         }
2827         lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
2828                                      le32_to_cpu(lif->identity->eth.min_frame_size));
2829         lif->netdev->max_mtu =
2830                 le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
2831
2832         lif->neqs = ionic->neqs_per_lif;
2833         lif->nxqs = ionic->ntxqs_per_lif;
2834
2835         lif->ionic = ionic;
2836         lif->index = 0;
2837         lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
2838         lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
2839
2840         /* Convert the default coalesce value to actual hw resolution */
2841         lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
2842         lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
2843                                                     lif->rx_coalesce_usecs);
2844         lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2845         lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2846         set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
2847         set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
2848
2849         snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
2850
2851         spin_lock_init(&lif->adminq_lock);
2852
2853         spin_lock_init(&lif->deferred.lock);
2854         INIT_LIST_HEAD(&lif->deferred.list);
2855         INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
2856
2857         /* allocate lif info */
2858         lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
2859         lif->info = dma_alloc_coherent(dev, lif->info_sz,
2860                                        &lif->info_pa, GFP_KERNEL);
2861         if (!lif->info) {
2862                 dev_err(dev, "Failed to allocate lif info, aborting\n");
2863                 err = -ENOMEM;
2864                 goto err_out_free_netdev;
2865         }
2866
2867         ionic_debugfs_add_lif(lif);
2868
2869         /* allocate control queues and txrx queue arrays */
2870         ionic_lif_queue_identify(lif);
2871         err = ionic_qcqs_alloc(lif);
2872         if (err)
2873                 goto err_out_free_lif_info;
2874
2875         /* allocate rss indirection table */
2876         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
2877         lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
2878         lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
2879                                               &lif->rss_ind_tbl_pa,
2880                                               GFP_KERNEL);
2881
2882         if (!lif->rss_ind_tbl) {
2883                 err = -ENOMEM;
2884                 dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
2885                 goto err_out_free_qcqs;
2886         }
2887         netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
2888
2889         ionic_lif_alloc_phc(lif);
2890
2891         return 0;
2892
2893 err_out_free_qcqs:
2894         ionic_qcqs_free(lif);
2895 err_out_free_lif_info:
2896         dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
2897         lif->info = NULL;
2898         lif->info_pa = 0;
2899 err_out_free_netdev:
2900         free_netdev(lif->netdev);
2901         lif = NULL;
2902 err_out_free_lid:
2903         kfree(lid);
2904
2905         return err;
2906 }
2907
2908 static void ionic_lif_reset(struct ionic_lif *lif)
2909 {
2910         struct ionic_dev *idev = &lif->ionic->idev;
2911
2912         mutex_lock(&lif->ionic->dev_cmd_lock);
2913         ionic_dev_cmd_lif_reset(idev, lif->index);
2914         ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
2915         mutex_unlock(&lif->ionic->dev_cmd_lock);
2916 }
2917
2918 static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
2919 {
2920         struct ionic *ionic = lif->ionic;
2921
2922         if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
2923                 return;
2924
2925         dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
2926
2927         netif_device_detach(lif->netdev);
2928
2929         if (test_bit(IONIC_LIF_F_UP, lif->state)) {
2930                 dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
2931                 mutex_lock(&lif->queue_lock);
2932                 ionic_stop_queues(lif);
2933                 mutex_unlock(&lif->queue_lock);
2934         }
2935
2936         if (netif_running(lif->netdev)) {
2937                 ionic_txrx_deinit(lif);
2938                 ionic_txrx_free(lif);
2939         }
2940         ionic_lif_deinit(lif);
2941         ionic_reset(ionic);
2942         ionic_qcqs_free(lif);
2943
2944         dev_info(ionic->dev, "FW Down: LIFs stopped\n");
2945 }
2946
2947 static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
2948 {
2949         struct ionic *ionic = lif->ionic;
2950         int err;
2951
2952         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2953                 return;
2954
2955         dev_info(ionic->dev, "FW Up: restarting LIFs\n");
2956
2957         ionic_init_devinfo(ionic);
2958         err = ionic_identify(ionic);
2959         if (err)
2960                 goto err_out;
2961         err = ionic_port_identify(ionic);
2962         if (err)
2963                 goto err_out;
2964         err = ionic_port_init(ionic);
2965         if (err)
2966                 goto err_out;
2967         err = ionic_qcqs_alloc(lif);
2968         if (err)
2969                 goto err_out;
2970
2971         err = ionic_lif_init(lif);
2972         if (err)
2973                 goto err_qcqs_free;
2974
2975         if (lif->registered)
2976                 ionic_lif_set_netdev_info(lif);
2977
2978         ionic_rx_filter_replay(lif);
2979
2980         if (netif_running(lif->netdev)) {
2981                 err = ionic_txrx_alloc(lif);
2982                 if (err)
2983                         goto err_lifs_deinit;
2984
2985                 err = ionic_txrx_init(lif);
2986                 if (err)
2987                         goto err_txrx_free;
2988         }
2989
2990         clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
2991         ionic_link_status_check_request(lif, CAN_SLEEP);
2992         netif_device_attach(lif->netdev);
2993         dev_info(ionic->dev, "FW Up: LIFs restarted\n");
2994
2995         /* restore the hardware timestamping queues */
2996         ionic_lif_hwstamp_replay(lif);
2997
2998         return;
2999
3000 err_txrx_free:
3001         ionic_txrx_free(lif);
3002 err_lifs_deinit:
3003         ionic_lif_deinit(lif);
3004 err_qcqs_free:
3005         ionic_qcqs_free(lif);
3006 err_out:
3007         dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
3008 }
3009
3010 void ionic_lif_free(struct ionic_lif *lif)
3011 {
3012         struct device *dev = lif->ionic->dev;
3013
3014         ionic_lif_free_phc(lif);
3015
3016         /* free rss indirection table */
3017         dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
3018                           lif->rss_ind_tbl_pa);
3019         lif->rss_ind_tbl = NULL;
3020         lif->rss_ind_tbl_pa = 0;
3021
3022         /* free queues */
3023         ionic_qcqs_free(lif);
3024         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3025                 ionic_lif_reset(lif);
3026
3027         /* free lif info */
3028         kfree(lif->identity);
3029         dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3030         lif->info = NULL;
3031         lif->info_pa = 0;
3032
3033         /* unmap doorbell page */
3034         ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3035         lif->kern_dbpage = NULL;
3036         kfree(lif->dbid_inuse);
3037         lif->dbid_inuse = NULL;
3038
3039         /* free netdev & lif */
3040         ionic_debugfs_del_lif(lif);
3041         free_netdev(lif->netdev);
3042 }
3043
3044 void ionic_lif_deinit(struct ionic_lif *lif)
3045 {
3046         if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
3047                 return;
3048
3049         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3050                 cancel_work_sync(&lif->deferred.work);
3051                 cancel_work_sync(&lif->tx_timeout_work);
3052                 ionic_rx_filters_deinit(lif);
3053                 if (lif->netdev->features & NETIF_F_RXHASH)
3054                         ionic_lif_rss_deinit(lif);
3055         }
3056
3057         napi_disable(&lif->adminqcq->napi);
3058         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3059         ionic_lif_qcq_deinit(lif, lif->adminqcq);
3060
3061         mutex_destroy(&lif->queue_lock);
3062         ionic_lif_reset(lif);
3063 }
3064
3065 static int ionic_lif_adminq_init(struct ionic_lif *lif)
3066 {
3067         struct device *dev = lif->ionic->dev;
3068         struct ionic_q_init_comp comp;
3069         struct ionic_dev *idev;
3070         struct ionic_qcq *qcq;
3071         struct ionic_queue *q;
3072         int err;
3073
3074         idev = &lif->ionic->idev;
3075         qcq = lif->adminqcq;
3076         q = &qcq->q;
3077
3078         mutex_lock(&lif->ionic->dev_cmd_lock);
3079         ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
3080         err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3081         ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3082         mutex_unlock(&lif->ionic->dev_cmd_lock);
3083         if (err) {
3084                 netdev_err(lif->netdev, "adminq init failed %d\n", err);
3085                 return err;
3086         }
3087
3088         q->hw_type = comp.hw_type;
3089         q->hw_index = le32_to_cpu(comp.hw_index);
3090         q->dbval = IONIC_DBELL_QID(q->hw_index);
3091
3092         dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
3093         dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
3094
3095         netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi,
3096                        NAPI_POLL_WEIGHT);
3097
3098         napi_enable(&qcq->napi);
3099
3100         if (qcq->flags & IONIC_QCQ_F_INTR)
3101                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
3102                                 IONIC_INTR_MASK_CLEAR);
3103
3104         qcq->flags |= IONIC_QCQ_F_INITED;
3105
3106         return 0;
3107 }
3108
3109 static int ionic_lif_notifyq_init(struct ionic_lif *lif)
3110 {
3111         struct ionic_qcq *qcq = lif->notifyqcq;
3112         struct device *dev = lif->ionic->dev;
3113         struct ionic_queue *q = &qcq->q;
3114         int err;
3115
3116         struct ionic_admin_ctx ctx = {
3117                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3118                 .cmd.q_init = {
3119                         .opcode = IONIC_CMD_Q_INIT,
3120                         .lif_index = cpu_to_le16(lif->index),
3121                         .type = q->type,
3122                         .ver = lif->qtype_info[q->type].version,
3123                         .index = cpu_to_le32(q->index),
3124                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
3125                                              IONIC_QINIT_F_ENA),
3126                         .intr_index = cpu_to_le16(lif->adminqcq->intr.index),
3127                         .pid = cpu_to_le16(q->pid),
3128                         .ring_size = ilog2(q->num_descs),
3129                         .ring_base = cpu_to_le64(q->base_pa),
3130                 }
3131         };
3132
3133         dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
3134         dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
3135         dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
3136         dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
3137
3138         err = ionic_adminq_post_wait(lif, &ctx);
3139         if (err)
3140                 return err;
3141
3142         lif->last_eid = 0;
3143         q->hw_type = ctx.comp.q_init.hw_type;
3144         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
3145         q->dbval = IONIC_DBELL_QID(q->hw_index);
3146
3147         dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
3148         dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
3149
3150         /* preset the callback info */
3151         q->info[0].cb_arg = lif;
3152
3153         qcq->flags |= IONIC_QCQ_F_INITED;
3154
3155         return 0;
3156 }
3157
3158 static int ionic_station_set(struct ionic_lif *lif)
3159 {
3160         struct net_device *netdev = lif->netdev;
3161         struct ionic_admin_ctx ctx = {
3162                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3163                 .cmd.lif_getattr = {
3164                         .opcode = IONIC_CMD_LIF_GETATTR,
3165                         .index = cpu_to_le16(lif->index),
3166                         .attr = IONIC_LIF_ATTR_MAC,
3167                 },
3168         };
3169         struct sockaddr addr;
3170         int err;
3171
3172         err = ionic_adminq_post_wait(lif, &ctx);
3173         if (err)
3174                 return err;
3175         netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
3176                    ctx.comp.lif_getattr.mac);
3177         if (is_zero_ether_addr(ctx.comp.lif_getattr.mac))
3178                 return 0;
3179
3180         if (!is_zero_ether_addr(netdev->dev_addr)) {
3181                 /* If the netdev mac is non-zero and doesn't match the default
3182                  * device address, it was set by something earlier and we're
3183                  * likely here again after a fw-upgrade reset.  We need to be
3184                  * sure the netdev mac is in our filter list.
3185                  */
3186                 if (!ether_addr_equal(ctx.comp.lif_getattr.mac,
3187                                       netdev->dev_addr))
3188                         ionic_lif_addr(lif, netdev->dev_addr, ADD_ADDR, CAN_SLEEP);
3189         } else {
3190                 /* Update the netdev mac with the device's mac */
3191                 memcpy(addr.sa_data, ctx.comp.lif_getattr.mac, netdev->addr_len);
3192                 addr.sa_family = AF_INET;
3193                 err = eth_prepare_mac_addr_change(netdev, &addr);
3194                 if (err) {
3195                         netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
3196                                     addr.sa_data, err);
3197                         return 0;
3198                 }
3199
3200                 eth_commit_mac_addr_change(netdev, &addr);
3201         }
3202
3203         netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
3204                    netdev->dev_addr);
3205         ionic_lif_addr(lif, netdev->dev_addr, ADD_ADDR, CAN_SLEEP);
3206
3207         return 0;
3208 }
3209
3210 int ionic_lif_init(struct ionic_lif *lif)
3211 {
3212         struct ionic_dev *idev = &lif->ionic->idev;
3213         struct device *dev = lif->ionic->dev;
3214         struct ionic_lif_init_comp comp;
3215         int dbpage_num;
3216         int err;
3217
3218         mutex_lock(&lif->ionic->dev_cmd_lock);
3219         ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
3220         err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3221         ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3222         mutex_unlock(&lif->ionic->dev_cmd_lock);
3223         if (err)
3224                 return err;
3225
3226         lif->hw_index = le16_to_cpu(comp.hw_index);
3227         mutex_init(&lif->queue_lock);
3228
3229         /* now that we have the hw_index we can figure out our doorbell page */
3230         lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
3231         if (!lif->dbid_count) {
3232                 dev_err(dev, "No doorbell pages, aborting\n");
3233                 return -EINVAL;
3234         }
3235
3236         lif->dbid_inuse = bitmap_alloc(lif->dbid_count, GFP_KERNEL);
3237         if (!lif->dbid_inuse) {
3238                 dev_err(dev, "Failed alloc doorbell id bitmap, aborting\n");
3239                 return -ENOMEM;
3240         }
3241
3242         /* first doorbell id reserved for kernel (dbid aka pid == zero) */
3243         set_bit(0, lif->dbid_inuse);
3244         lif->kern_pid = 0;
3245
3246         dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
3247         lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
3248         if (!lif->kern_dbpage) {
3249                 dev_err(dev, "Cannot map dbpage, aborting\n");
3250                 err = -ENOMEM;
3251                 goto err_out_free_dbid;
3252         }
3253
3254         err = ionic_lif_adminq_init(lif);
3255         if (err)
3256                 goto err_out_adminq_deinit;
3257
3258         if (lif->ionic->nnqs_per_lif) {
3259                 err = ionic_lif_notifyq_init(lif);
3260                 if (err)
3261                         goto err_out_notifyq_deinit;
3262         }
3263
3264         err = ionic_init_nic_features(lif);
3265         if (err)
3266                 goto err_out_notifyq_deinit;
3267
3268         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3269                 err = ionic_rx_filters_init(lif);
3270                 if (err)
3271                         goto err_out_notifyq_deinit;
3272         }
3273
3274         err = ionic_station_set(lif);
3275         if (err)
3276                 goto err_out_notifyq_deinit;
3277
3278         lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
3279
3280         set_bit(IONIC_LIF_F_INITED, lif->state);
3281
3282         INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
3283
3284         return 0;
3285
3286 err_out_notifyq_deinit:
3287         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3288 err_out_adminq_deinit:
3289         ionic_lif_qcq_deinit(lif, lif->adminqcq);
3290         ionic_lif_reset(lif);
3291         ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3292         lif->kern_dbpage = NULL;
3293 err_out_free_dbid:
3294         kfree(lif->dbid_inuse);
3295         lif->dbid_inuse = NULL;
3296
3297         return err;
3298 }
3299
3300 static void ionic_lif_notify_work(struct work_struct *ws)
3301 {
3302 }
3303
3304 static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
3305 {
3306         struct ionic_admin_ctx ctx = {
3307                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3308                 .cmd.lif_setattr = {
3309                         .opcode = IONIC_CMD_LIF_SETATTR,
3310                         .index = cpu_to_le16(lif->index),
3311                         .attr = IONIC_LIF_ATTR_NAME,
3312                 },
3313         };
3314
3315         strlcpy(ctx.cmd.lif_setattr.name, lif->netdev->name,
3316                 sizeof(ctx.cmd.lif_setattr.name));
3317
3318         ionic_adminq_post_wait(lif, &ctx);
3319 }
3320
3321 static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
3322 {
3323         if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
3324                 return NULL;
3325
3326         return netdev_priv(netdev);
3327 }
3328
3329 static int ionic_lif_notify(struct notifier_block *nb,
3330                             unsigned long event, void *info)
3331 {
3332         struct net_device *ndev = netdev_notifier_info_to_dev(info);
3333         struct ionic *ionic = container_of(nb, struct ionic, nb);
3334         struct ionic_lif *lif = ionic_netdev_lif(ndev);
3335
3336         if (!lif || lif->ionic != ionic)
3337                 return NOTIFY_DONE;
3338
3339         switch (event) {
3340         case NETDEV_CHANGENAME:
3341                 ionic_lif_set_netdev_info(lif);
3342                 break;
3343         }
3344
3345         return NOTIFY_DONE;
3346 }
3347
3348 int ionic_lif_register(struct ionic_lif *lif)
3349 {
3350         int err;
3351
3352         ionic_lif_register_phc(lif);
3353
3354         INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
3355
3356         lif->ionic->nb.notifier_call = ionic_lif_notify;
3357
3358         err = register_netdevice_notifier(&lif->ionic->nb);
3359         if (err)
3360                 lif->ionic->nb.notifier_call = NULL;
3361
3362         /* only register LIF0 for now */
3363         err = register_netdev(lif->netdev);
3364         if (err) {
3365                 dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
3366                 ionic_lif_unregister_phc(lif);
3367                 return err;
3368         }
3369
3370         ionic_link_status_check_request(lif, CAN_SLEEP);
3371         lif->registered = true;
3372         ionic_lif_set_netdev_info(lif);
3373
3374         return 0;
3375 }
3376
3377 void ionic_lif_unregister(struct ionic_lif *lif)
3378 {
3379         if (lif->ionic->nb.notifier_call) {
3380                 unregister_netdevice_notifier(&lif->ionic->nb);
3381                 cancel_work_sync(&lif->ionic->nb_work);
3382                 lif->ionic->nb.notifier_call = NULL;
3383         }
3384
3385         if (lif->netdev->reg_state == NETREG_REGISTERED)
3386                 unregister_netdev(lif->netdev);
3387
3388         ionic_lif_unregister_phc(lif);
3389
3390         lif->registered = false;
3391 }
3392
3393 static void ionic_lif_queue_identify(struct ionic_lif *lif)
3394 {
3395         union ionic_q_identity __iomem *q_ident;
3396         struct ionic *ionic = lif->ionic;
3397         struct ionic_dev *idev;
3398         int qtype;
3399         int err;
3400
3401         idev = &lif->ionic->idev;
3402         q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
3403
3404         for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
3405                 struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
3406
3407                 /* filter out the ones we know about */
3408                 switch (qtype) {
3409                 case IONIC_QTYPE_ADMINQ:
3410                 case IONIC_QTYPE_NOTIFYQ:
3411                 case IONIC_QTYPE_RXQ:
3412                 case IONIC_QTYPE_TXQ:
3413                         break;
3414                 default:
3415                         continue;
3416                 }
3417
3418                 memset(qti, 0, sizeof(*qti));
3419
3420                 mutex_lock(&ionic->dev_cmd_lock);
3421                 ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
3422                                              ionic_qtype_versions[qtype]);
3423                 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3424                 if (!err) {
3425                         qti->version   = readb(&q_ident->version);
3426                         qti->supported = readb(&q_ident->supported);
3427                         qti->features  = readq(&q_ident->features);
3428                         qti->desc_sz   = readw(&q_ident->desc_sz);
3429                         qti->comp_sz   = readw(&q_ident->comp_sz);
3430                         qti->sg_desc_sz   = readw(&q_ident->sg_desc_sz);
3431                         qti->max_sg_elems = readw(&q_ident->max_sg_elems);
3432                         qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
3433                 }
3434                 mutex_unlock(&ionic->dev_cmd_lock);
3435
3436                 if (err == -EINVAL) {
3437                         dev_err(ionic->dev, "qtype %d not supported\n", qtype);
3438                         continue;
3439                 } else if (err == -EIO) {
3440                         dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
3441                         return;
3442                 } else if (err) {
3443                         dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
3444                                 qtype, err);
3445                         return;
3446                 }
3447
3448                 dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
3449                         qtype, qti->version);
3450                 dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
3451                         qtype, qti->supported);
3452                 dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
3453                         qtype, qti->features);
3454                 dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
3455                         qtype, qti->desc_sz);
3456                 dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
3457                         qtype, qti->comp_sz);
3458                 dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
3459                         qtype, qti->sg_desc_sz);
3460                 dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
3461                         qtype, qti->max_sg_elems);
3462                 dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
3463                         qtype, qti->sg_desc_stride);
3464         }
3465 }
3466
3467 int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
3468                        union ionic_lif_identity *lid)
3469 {
3470         struct ionic_dev *idev = &ionic->idev;
3471         size_t sz;
3472         int err;
3473
3474         sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
3475
3476         mutex_lock(&ionic->dev_cmd_lock);
3477         ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
3478         err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3479         memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
3480         mutex_unlock(&ionic->dev_cmd_lock);
3481         if (err)
3482                 return (err);
3483
3484         dev_dbg(ionic->dev, "capabilities 0x%llx\n",
3485                 le64_to_cpu(lid->capabilities));
3486
3487         dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
3488                 le32_to_cpu(lid->eth.max_ucast_filters));
3489         dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
3490                 le32_to_cpu(lid->eth.max_mcast_filters));
3491         dev_dbg(ionic->dev, "eth.features 0x%llx\n",
3492                 le64_to_cpu(lid->eth.config.features));
3493         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
3494                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
3495         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
3496                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
3497         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
3498                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
3499         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
3500                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
3501         dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
3502         dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
3503         dev_dbg(ionic->dev, "eth.config.mtu %d\n",
3504                 le32_to_cpu(lid->eth.config.mtu));
3505
3506         return 0;
3507 }
3508
3509 int ionic_lif_size(struct ionic *ionic)
3510 {
3511         struct ionic_identity *ident = &ionic->ident;
3512         unsigned int nintrs, dev_nintrs;
3513         union ionic_lif_config *lc;
3514         unsigned int ntxqs_per_lif;
3515         unsigned int nrxqs_per_lif;
3516         unsigned int neqs_per_lif;
3517         unsigned int nnqs_per_lif;
3518         unsigned int nxqs, neqs;
3519         unsigned int min_intrs;
3520         int err;
3521
3522         lc = &ident->lif.eth.config;
3523         dev_nintrs = le32_to_cpu(ident->dev.nintrs);
3524         neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
3525         nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
3526         ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
3527         nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
3528
3529         /* reserve last queue id for hardware timestamping */
3530         if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
3531                 if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
3532                         lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
3533                 } else {
3534                         ntxqs_per_lif -= 1;
3535                         nrxqs_per_lif -= 1;
3536                 }
3537         }
3538
3539         nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
3540         nxqs = min(nxqs, num_online_cpus());
3541         neqs = min(neqs_per_lif, num_online_cpus());
3542
3543 try_again:
3544         /* interrupt usage:
3545          *    1 for master lif adminq/notifyq
3546          *    1 for each CPU for master lif TxRx queue pairs
3547          *    whatever's left is for RDMA queues
3548          */
3549         nintrs = 1 + nxqs + neqs;
3550         min_intrs = 2;  /* adminq + 1 TxRx queue pair */
3551
3552         if (nintrs > dev_nintrs)
3553                 goto try_fewer;
3554
3555         err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
3556         if (err < 0 && err != -ENOSPC) {
3557                 dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
3558                 return err;
3559         }
3560         if (err == -ENOSPC)
3561                 goto try_fewer;
3562
3563         if (err != nintrs) {
3564                 ionic_bus_free_irq_vectors(ionic);
3565                 goto try_fewer;
3566         }
3567
3568         ionic->nnqs_per_lif = nnqs_per_lif;
3569         ionic->neqs_per_lif = neqs;
3570         ionic->ntxqs_per_lif = nxqs;
3571         ionic->nrxqs_per_lif = nxqs;
3572         ionic->nintrs = nintrs;
3573
3574         ionic_debugfs_add_sizes(ionic);
3575
3576         return 0;
3577
3578 try_fewer:
3579         if (nnqs_per_lif > 1) {
3580                 nnqs_per_lif >>= 1;
3581                 goto try_again;
3582         }
3583         if (neqs > 1) {
3584                 neqs >>= 1;
3585                 goto try_again;
3586         }
3587         if (nxqs > 1) {
3588                 nxqs >>= 1;
3589                 goto try_again;
3590         }
3591         dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
3592         return -ENOSPC;
3593 }