drivers/net/ethernet/pensando/ionic/ionic_lif.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4 #include <linux/ethtool.h>
   5 #include <linux/printk.h>
   6 #include <linux/dynamic_debug.h>
   7 #include <linux/netdevice.h>
   8 #include <linux/etherdevice.h>
   9 #include <linux/if_vlan.h>
  10 #include <linux/rtnetlink.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/pci.h>
  13 #include <linux/cpumask.h>
  14 #include <linux/crash_dump.h>
  15
  16 #include "ionic.h"
  17 #include "ionic_bus.h"
  18 #include "ionic_lif.h"
  19 #include "ionic_txrx.h"
  20 #include "ionic_ethtool.h"
  21 #include "ionic_debugfs.h"
  22
  23 /* queuetype support level */
  24 static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
  25         [IONIC_QTYPE_ADMINQ]  = 0,   /* 0 = Base version with CQ support */
  26         [IONIC_QTYPE_NOTIFYQ] = 0,   /* 0 = Base version */
  27         [IONIC_QTYPE_RXQ]     = 0,   /* 0 = Base version with CQ+SG support */
  28         [IONIC_QTYPE_TXQ]     = 1,   /* 0 = Base version with CQ+SG support
  29                                       * 1 =   ... with Tx SG version 1
  30                                       */
  31 };
  32
  33 static void ionic_link_status_check(struct ionic_lif *lif);
  34 static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
  35 static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
  36 static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
  37
  38 static void ionic_txrx_deinit(struct ionic_lif *lif);
  39 static int ionic_txrx_init(struct ionic_lif *lif);
  40 static int ionic_start_queues(struct ionic_lif *lif);
  41 static void ionic_stop_queues(struct ionic_lif *lif);
  42 static void ionic_lif_queue_identify(struct ionic_lif *lif);
  43
  44 static void ionic_dim_work(struct work_struct *work)
  45 {
  46         struct dim *dim = container_of(work, struct dim, work);
  47         struct dim_cq_moder cur_moder;
  48         struct ionic_qcq *qcq;
  49         u32 new_coal;
  50
  51         cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
  52         qcq = container_of(dim, struct ionic_qcq, dim);
  53         new_coal = ionic_coal_usec_to_hw(qcq->q.lif->ionic, cur_moder.usec);
  54         new_coal = new_coal ? new_coal : 1;
  55
  56         if (qcq->intr.dim_coal_hw != new_coal) {
  57                 unsigned int qi = qcq->cq.bound_q->index;
  58                 struct ionic_lif *lif = qcq->q.lif;
  59
  60                 qcq->intr.dim_coal_hw = new_coal;
  61
  62                 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
  63                                      lif->rxqcqs[qi]->intr.index,
  64                                      qcq->intr.dim_coal_hw);
  65         }
  66
  67         dim->state = DIM_START_MEASURE;
  68 }
  69
  70 static void ionic_lif_deferred_work(struct work_struct *work)
  71 {
  72         struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
  73         struct ionic_deferred *def = &lif->deferred;
  74         struct ionic_deferred_work *w = NULL;
  75
  76         do {
  77                 spin_lock_bh(&def->lock);
  78                 if (!list_empty(&def->list)) {
  79                         w = list_first_entry(&def->list,
  80                                              struct ionic_deferred_work, list);
  81                         list_del(&w->list);
  82                 }
  83                 spin_unlock_bh(&def->lock);
  84
  85                 if (!w)
  86                         break;
  87
  88                 switch (w->type) {
  89                 case IONIC_DW_TYPE_RX_MODE:
  90                         ionic_lif_rx_mode(lif);
  91                         break;
  92                 case IONIC_DW_TYPE_LINK_STATUS:
  93                         ionic_link_status_check(lif);
  94                         break;
  95                 case IONIC_DW_TYPE_LIF_RESET:
  96                         if (w->fw_status) {
  97                                 ionic_lif_handle_fw_up(lif);
  98                         } else {
  99                                 ionic_lif_handle_fw_down(lif);
 100
 101                                 /* Fire off another watchdog to see
 102                                  * if the FW is already back rather than
 103                                  * waiting another whole cycle
 104                                  */
 105                                 mod_timer(&lif->ionic->watchdog_timer, jiffies + 1);
 106                         }
 107                         break;
 108                 default:
 109                         break;
 110                 }
 111                 kfree(w);
 112                 w = NULL;
 113         } while (true);
 114 }
 115
 116 void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
 117                                 struct ionic_deferred_work *work)
 118 {
 119         spin_lock_bh(&def->lock);
 120         list_add_tail(&work->list, &def->list);
 121         spin_unlock_bh(&def->lock);
 122         schedule_work(&def->work);
 123 }
 124
 125 static void ionic_link_status_check(struct ionic_lif *lif)
 126 {
 127         struct net_device *netdev = lif->netdev;
 128         u16 link_status;
 129         bool link_up;
 130
 131         if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 132                 return;
 133
 134         /* Don't put carrier back up if we're in a broken state */
 135         if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
 136                 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 137                 return;
 138         }
 139
 140         link_status = le16_to_cpu(lif->info->status.link_status);
 141         link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
 142
 143         if (link_up) {
 144                 int err = 0;
 145
 146                 if (netdev->flags & IFF_UP && netif_running(netdev)) {
 147                         mutex_lock(&lif->queue_lock);
 148                         err = ionic_start_queues(lif);
 149                         if (err && err != -EBUSY) {
 150                                 netdev_err(lif->netdev,
 151                                            "Failed to start queues: %d\n", err);
 152                                 set_bit(IONIC_LIF_F_BROKEN, lif->state);
 153                                 netif_carrier_off(lif->netdev);
 154                         }
 155                         mutex_unlock(&lif->queue_lock);
 156                 }
 157
 158                 if (!err && !netif_carrier_ok(netdev)) {
 159                         ionic_port_identify(lif->ionic);
 160                         netdev_info(netdev, "Link up - %d Gbps\n",
 161                                     le32_to_cpu(lif->info->status.link_speed) / 1000);
 162                         netif_carrier_on(netdev);
 163                 }
 164         } else {
 165                 if (netif_carrier_ok(netdev)) {
 166                         netdev_info(netdev, "Link down\n");
 167                         netif_carrier_off(netdev);
 168                 }
 169
 170                 if (netdev->flags & IFF_UP && netif_running(netdev)) {
 171                         mutex_lock(&lif->queue_lock);
 172                         ionic_stop_queues(lif);
 173                         mutex_unlock(&lif->queue_lock);
 174                 }
 175         }
 176
 177         clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 178 }
 179
 180 void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
 181 {
 182         struct ionic_deferred_work *work;
 183
 184         /* we only need one request outstanding at a time */
 185         if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 186                 return;
 187
 188         if (!can_sleep) {
 189                 work = kzalloc(sizeof(*work), GFP_ATOMIC);
 190                 if (!work) {
 191                         clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 192                         return;
 193                 }
 194
 195                 work->type = IONIC_DW_TYPE_LINK_STATUS;
 196                 ionic_lif_deferred_enqueue(&lif->deferred, work);
 197         } else {
 198                 ionic_link_status_check(lif);
 199         }
 200 }
 201
 202 static irqreturn_t ionic_isr(int irq, void *data)
 203 {
 204         struct napi_struct *napi = data;
 205
 206         napi_schedule_irqoff(napi);
 207
 208         return IRQ_HANDLED;
 209 }
 210
 211 static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
 212 {
 213         struct ionic_intr_info *intr = &qcq->intr;
 214         struct device *dev = lif->ionic->dev;
 215         struct ionic_queue *q = &qcq->q;
 216         const char *name;
 217
 218         if (lif->registered)
 219                 name = lif->netdev->name;
 220         else
 221                 name = dev_name(dev);
 222
 223         snprintf(intr->name, sizeof(intr->name),
 224                  "%s-%s-%s", IONIC_DRV_NAME, name, q->name);
 225
 226         return devm_request_irq(dev, intr->vector, ionic_isr,
 227                                 0, intr->name, &qcq->napi);
 228 }
 229
 230 static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 231 {
 232         struct ionic *ionic = lif->ionic;
 233         int index;
 234
 235         index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
 236         if (index == ionic->nintrs) {
 237                 netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
 238                             __func__, index, ionic->nintrs);
 239                 return -ENOSPC;
 240         }
 241
 242         set_bit(index, ionic->intrs);
 243         ionic_intr_init(&ionic->idev, intr, index);
 244
 245         return 0;
 246 }
 247
 248 static void ionic_intr_free(struct ionic *ionic, int index)
 249 {
 250         if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
 251                 clear_bit(index, ionic->intrs);
 252 }
 253
 254 static int ionic_qcq_enable(struct ionic_qcq *qcq)
 255 {
 256         struct ionic_queue *q = &qcq->q;
 257         struct ionic_lif *lif = q->lif;
 258         struct ionic_dev *idev;
 259         struct device *dev;
 260
 261         struct ionic_admin_ctx ctx = {
 262                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 263                 .cmd.q_control = {
 264                         .opcode = IONIC_CMD_Q_CONTROL,
 265                         .lif_index = cpu_to_le16(lif->index),
 266                         .type = q->type,
 267                         .index = cpu_to_le32(q->index),
 268                         .oper = IONIC_Q_ENABLE,
 269                 },
 270         };
 271
 272         idev = &lif->ionic->idev;
 273         dev = lif->ionic->dev;
 274
 275         dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
 276                 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
 277
 278         if (qcq->flags & IONIC_QCQ_F_INTR) {
 279                 irq_set_affinity_hint(qcq->intr.vector,
 280                                       &qcq->intr.affinity_mask);
 281                 napi_enable(&qcq->napi);
 282                 ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
 283                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 284                                 IONIC_INTR_MASK_CLEAR);
 285         }
 286
 287         return ionic_adminq_post_wait(lif, &ctx);
 288 }
 289
 290 static int ionic_qcq_disable(struct ionic_qcq *qcq, bool send_to_hw)
 291 {
 292         struct ionic_queue *q;
 293         struct ionic_lif *lif;
 294         int err = 0;
 295
 296         struct ionic_admin_ctx ctx = {
 297                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 298                 .cmd.q_control = {
 299                         .opcode = IONIC_CMD_Q_CONTROL,
 300                         .oper = IONIC_Q_DISABLE,
 301                 },
 302         };
 303
 304         if (!qcq)
 305                 return -ENXIO;
 306
 307         q = &qcq->q;
 308         lif = q->lif;
 309
 310         if (qcq->flags & IONIC_QCQ_F_INTR) {
 311                 struct ionic_dev *idev = &lif->ionic->idev;
 312
 313                 cancel_work_sync(&qcq->dim.work);
 314                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 315                                 IONIC_INTR_MASK_SET);
 316                 synchronize_irq(qcq->intr.vector);
 317                 irq_set_affinity_hint(qcq->intr.vector, NULL);
 318                 napi_disable(&qcq->napi);
 319         }
 320
 321         if (send_to_hw) {
 322                 ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
 323                 ctx.cmd.q_control.type = q->type;
 324                 ctx.cmd.q_control.index = cpu_to_le32(q->index);
 325                 dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
 326                         ctx.cmd.q_control.index, ctx.cmd.q_control.type);
 327
 328                 err = ionic_adminq_post_wait(lif, &ctx);
 329         }
 330
 331         return err;
 332 }
 333
 334 static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
 335 {
 336         struct ionic_dev *idev = &lif->ionic->idev;
 337
 338         if (!qcq)
 339                 return;
 340
 341         if (!(qcq->flags & IONIC_QCQ_F_INITED))
 342                 return;
 343
 344         if (qcq->flags & IONIC_QCQ_F_INTR) {
 345                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 346                                 IONIC_INTR_MASK_SET);
 347                 netif_napi_del(&qcq->napi);
 348         }
 349
 350         qcq->flags &= ~IONIC_QCQ_F_INITED;
 351 }
 352
 353 static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
 354 {
 355         if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
 356                 return;
 357
 358         irq_set_affinity_hint(qcq->intr.vector, NULL);
 359         devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
 360         qcq->intr.vector = 0;
 361         ionic_intr_free(lif->ionic, qcq->intr.index);
 362         qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 363 }
 364
 365 static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
 366 {
 367         struct device *dev = lif->ionic->dev;
 368
 369         if (!qcq)
 370                 return;
 371
 372         ionic_debugfs_del_qcq(qcq);
 373
 374         if (qcq->q_base) {
 375                 dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
 376                 qcq->q_base = NULL;
 377                 qcq->q_base_pa = 0;
 378         }
 379
 380         if (qcq->cq_base) {
 381                 dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
 382                 qcq->cq_base = NULL;
 383                 qcq->cq_base_pa = 0;
 384         }
 385
 386         if (qcq->sg_base) {
 387                 dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
 388                 qcq->sg_base = NULL;
 389                 qcq->sg_base_pa = 0;
 390         }
 391
 392         ionic_qcq_intr_free(lif, qcq);
 393
 394         if (qcq->cq.info) {
 395                 devm_kfree(dev, qcq->cq.info);
 396                 qcq->cq.info = NULL;
 397         }
 398         if (qcq->q.info) {
 399                 devm_kfree(dev, qcq->q.info);
 400                 qcq->q.info = NULL;
 401         }
 402 }
 403
 404 static void ionic_qcqs_free(struct ionic_lif *lif)
 405 {
 406         struct device *dev = lif->ionic->dev;
 407         struct ionic_qcq *adminqcq;
 408         unsigned long irqflags;
 409
 410         if (lif->notifyqcq) {
 411                 ionic_qcq_free(lif, lif->notifyqcq);
 412                 devm_kfree(dev, lif->notifyqcq);
 413                 lif->notifyqcq = NULL;
 414         }
 415
 416         if (lif->adminqcq) {
 417                 spin_lock_irqsave(&lif->adminq_lock, irqflags);
 418                 adminqcq = READ_ONCE(lif->adminqcq);
 419                 lif->adminqcq = NULL;
 420                 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
 421                 if (adminqcq) {
 422                         ionic_qcq_free(lif, adminqcq);
 423                         devm_kfree(dev, adminqcq);
 424                 }
 425         }
 426
 427         if (lif->rxqcqs) {
 428                 devm_kfree(dev, lif->rxqstats);
 429                 lif->rxqstats = NULL;
 430                 devm_kfree(dev, lif->rxqcqs);
 431                 lif->rxqcqs = NULL;
 432         }
 433
 434         if (lif->txqcqs) {
 435                 devm_kfree(dev, lif->txqstats);
 436                 lif->txqstats = NULL;
 437                 devm_kfree(dev, lif->txqcqs);
 438                 lif->txqcqs = NULL;
 439         }
 440 }
 441
 442 static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
 443                                       struct ionic_qcq *n_qcq)
 444 {
 445         if (WARN_ON(n_qcq->flags & IONIC_QCQ_F_INTR)) {
 446                 ionic_intr_free(n_qcq->cq.lif->ionic, n_qcq->intr.index);
 447                 n_qcq->flags &= ~IONIC_QCQ_F_INTR;
 448         }
 449
 450         n_qcq->intr.vector = src_qcq->intr.vector;
 451         n_qcq->intr.index = src_qcq->intr.index;
 452 }
 453
 454 static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
 455 {
 456         int err;
 457
 458         if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
 459                 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 460                 return 0;
 461         }
 462
 463         err = ionic_intr_alloc(lif, &qcq->intr);
 464         if (err) {
 465                 netdev_warn(lif->netdev, "no intr for %s: %d\n",
 466                             qcq->q.name, err);
 467                 goto err_out;
 468         }
 469
 470         err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
 471         if (err < 0) {
 472                 netdev_warn(lif->netdev, "no vector for %s: %d\n",
 473                             qcq->q.name, err);
 474                 goto err_out_free_intr;
 475         }
 476         qcq->intr.vector = err;
 477         ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
 478                                IONIC_INTR_MASK_SET);
 479
 480         err = ionic_request_irq(lif, qcq);
 481         if (err) {
 482                 netdev_warn(lif->netdev, "irq request failed %d\n", err);
 483                 goto err_out_free_intr;
 484         }
 485
 486         /* try to get the irq on the local numa node first */
 487         qcq->intr.cpu = cpumask_local_spread(qcq->intr.index,
 488                                              dev_to_node(lif->ionic->dev));
 489         if (qcq->intr.cpu != -1)
 490                 cpumask_set_cpu(qcq->intr.cpu, &qcq->intr.affinity_mask);
 491
 492         netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
 493         return 0;
 494
 495 err_out_free_intr:
 496         ionic_intr_free(lif->ionic, qcq->intr.index);
 497 err_out:
 498         return err;
 499 }
 500
 501 static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
 502                            unsigned int index,
 503                            const char *name, unsigned int flags,
 504                            unsigned int num_descs, unsigned int desc_size,
 505                            unsigned int cq_desc_size,
 506                            unsigned int sg_desc_size,
 507                            unsigned int pid, struct ionic_qcq **qcq)
 508 {
 509         struct ionic_dev *idev = &lif->ionic->idev;
 510         struct device *dev = lif->ionic->dev;
 511         void *q_base, *cq_base, *sg_base;
 512         dma_addr_t cq_base_pa = 0;
 513         dma_addr_t sg_base_pa = 0;
 514         dma_addr_t q_base_pa = 0;
 515         struct ionic_qcq *new;
 516         int err;
 517
 518         *qcq = NULL;
 519
 520         new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
 521         if (!new) {
 522                 netdev_err(lif->netdev, "Cannot allocate queue structure\n");
 523                 err = -ENOMEM;
 524                 goto err_out;
 525         }
 526
 527         new->q.dev = dev;
 528         new->flags = flags;
 529
 530         new->q.info = devm_kcalloc(dev, num_descs, sizeof(*new->q.info),
 531                                    GFP_KERNEL);
 532         if (!new->q.info) {
 533                 netdev_err(lif->netdev, "Cannot allocate queue info\n");
 534                 err = -ENOMEM;
 535                 goto err_out_free_qcq;
 536         }
 537
 538         new->q.type = type;
 539         new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
 540
 541         err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
 542                            desc_size, sg_desc_size, pid);
 543         if (err) {
 544                 netdev_err(lif->netdev, "Cannot initialize queue\n");
 545                 goto err_out_free_q_info;
 546         }
 547
 548         err = ionic_alloc_qcq_interrupt(lif, new);
 549         if (err)
 550                 goto err_out;
 551
 552         new->cq.info = devm_kcalloc(dev, num_descs, sizeof(*new->cq.info),
 553                                     GFP_KERNEL);
 554         if (!new->cq.info) {
 555                 netdev_err(lif->netdev, "Cannot allocate completion queue info\n");
 556                 err = -ENOMEM;
 557                 goto err_out_free_irq;
 558         }
 559
 560         err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
 561         if (err) {
 562                 netdev_err(lif->netdev, "Cannot initialize completion queue\n");
 563                 goto err_out_free_cq_info;
 564         }
 565
 566         if (flags & IONIC_QCQ_F_NOTIFYQ) {
 567                 int q_size, cq_size;
 568
 569                 /* q & cq need to be contiguous in case of notifyq */
 570                 q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
 571                 cq_size = ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
 572
 573                 new->q_size = PAGE_SIZE + q_size + cq_size;
 574                 new->q_base = dma_alloc_coherent(dev, new->q_size,
 575                                                  &new->q_base_pa, GFP_KERNEL);
 576                 if (!new->q_base) {
 577                         netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
 578                         err = -ENOMEM;
 579                         goto err_out_free_cq_info;
 580                 }
 581                 q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
 582                 q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
 583                 ionic_q_map(&new->q, q_base, q_base_pa);
 584
 585                 cq_base = PTR_ALIGN(q_base + q_size, PAGE_SIZE);
 586                 cq_base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
 587                 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 588                 ionic_cq_bind(&new->cq, &new->q);
 589         } else {
 590                 new->q_size = PAGE_SIZE + (num_descs * desc_size);
 591                 new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
 592                                                  GFP_KERNEL);
 593                 if (!new->q_base) {
 594                         netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
 595                         err = -ENOMEM;
 596                         goto err_out_free_cq_info;
 597                 }
 598                 q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
 599                 q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
 600                 ionic_q_map(&new->q, q_base, q_base_pa);
 601
 602                 new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
 603                 new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
 604                                                   GFP_KERNEL);
 605                 if (!new->cq_base) {
 606                         netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
 607                         err = -ENOMEM;
 608                         goto err_out_free_q;
 609                 }
 610                 cq_base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
 611                 cq_base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
 612                 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 613                 ionic_cq_bind(&new->cq, &new->q);
 614         }
 615
 616         if (flags & IONIC_QCQ_F_SG) {
 617                 new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
 618                 new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
 619                                                   GFP_KERNEL);
 620                 if (!new->sg_base) {
 621                         netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
 622                         err = -ENOMEM;
 623                         goto err_out_free_cq;
 624                 }
 625                 sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
 626                 sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
 627                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 628         }
 629
 630         INIT_WORK(&new->dim.work, ionic_dim_work);
 631         new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 632
 633         *qcq = new;
 634
 635         return 0;
 636
 637 err_out_free_cq:
 638         dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
 639 err_out_free_q:
 640         dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
 641 err_out_free_cq_info:
 642         devm_kfree(dev, new->cq.info);
 643 err_out_free_irq:
 644         if (flags & IONIC_QCQ_F_INTR) {
 645                 devm_free_irq(dev, new->intr.vector, &new->napi);
 646                 ionic_intr_free(lif->ionic, new->intr.index);
 647         }
 648 err_out_free_q_info:
 649         devm_kfree(dev, new->q.info);
 650 err_out_free_qcq:
 651         devm_kfree(dev, new);
 652 err_out:
 653         dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
 654         return err;
 655 }
 656
 657 static int ionic_qcqs_alloc(struct ionic_lif *lif)
 658 {
 659         struct device *dev = lif->ionic->dev;
 660         unsigned int flags;
 661         int err;
 662
 663         flags = IONIC_QCQ_F_INTR;
 664         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 665                               IONIC_ADMINQ_LENGTH,
 666                               sizeof(struct ionic_admin_cmd),
 667                               sizeof(struct ionic_admin_comp),
 668                               0, lif->kern_pid, &lif->adminqcq);
 669         if (err)
 670                 return err;
 671         ionic_debugfs_add_qcq(lif, lif->adminqcq);
 672
 673         if (lif->ionic->nnqs_per_lif) {
 674                 flags = IONIC_QCQ_F_NOTIFYQ;
 675                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
 676                                       flags, IONIC_NOTIFYQ_LENGTH,
 677                                       sizeof(struct ionic_notifyq_cmd),
 678                                       sizeof(union ionic_notifyq_comp),
 679                                       0, lif->kern_pid, &lif->notifyqcq);
 680                 if (err)
 681                         goto err_out;
 682                 ionic_debugfs_add_qcq(lif, lif->notifyqcq);
 683
 684                 /* Let the notifyq ride on the adminq interrupt */
 685                 ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
 686         }
 687
 688         err = -ENOMEM;
 689         lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
 690                                    sizeof(*lif->txqcqs), GFP_KERNEL);
 691         if (!lif->txqcqs)
 692                 goto err_out;
 693         lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
 694                                    sizeof(*lif->rxqcqs), GFP_KERNEL);
 695         if (!lif->rxqcqs)
 696                 goto err_out;
 697
 698         lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
 699                                      sizeof(*lif->txqstats), GFP_KERNEL);
 700         if (!lif->txqstats)
 701                 goto err_out;
 702         lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
 703                                      sizeof(*lif->rxqstats), GFP_KERNEL);
 704         if (!lif->rxqstats)
 705                 goto err_out;
 706
 707         return 0;
 708
 709 err_out:
 710         ionic_qcqs_free(lif);
 711         return err;
 712 }
 713
 714 static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
 715 {
 716         qcq->q.tail_idx = 0;
 717         qcq->q.head_idx = 0;
 718         qcq->cq.tail_idx = 0;
 719         qcq->cq.done_color = 1;
 720         memset(qcq->q_base, 0, qcq->q_size);
 721         memset(qcq->cq_base, 0, qcq->cq_size);
 722         memset(qcq->sg_base, 0, qcq->sg_size);
 723 }
 724
 725 static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 726 {
 727         struct device *dev = lif->ionic->dev;
 728         struct ionic_queue *q = &qcq->q;
 729         struct ionic_cq *cq = &qcq->cq;
 730         struct ionic_admin_ctx ctx = {
 731                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 732                 .cmd.q_init = {
 733                         .opcode = IONIC_CMD_Q_INIT,
 734                         .lif_index = cpu_to_le16(lif->index),
 735                         .type = q->type,
 736                         .ver = lif->qtype_info[q->type].version,
 737                         .index = cpu_to_le32(q->index),
 738                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 739                                              IONIC_QINIT_F_SG),
 740                         .pid = cpu_to_le16(q->pid),
 741                         .ring_size = ilog2(q->num_descs),
 742                         .ring_base = cpu_to_le64(q->base_pa),
 743                         .cq_ring_base = cpu_to_le64(cq->base_pa),
 744                         .sg_ring_base = cpu_to_le64(q->sg_base_pa),
 745                         .features = cpu_to_le64(q->features),
 746                 },
 747         };
 748         unsigned int intr_index;
 749         int err;
 750
 751         intr_index = qcq->intr.index;
 752
 753         ctx.cmd.q_init.intr_index = cpu_to_le16(intr_index);
 754
 755         dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
 756         dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
 757         dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
 758         dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
 759         dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
 760         dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
 761         dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
 762
 763         ionic_qcq_sanitize(qcq);
 764
 765         err = ionic_adminq_post_wait(lif, &ctx);
 766         if (err)
 767                 return err;
 768
 769         q->hw_type = ctx.comp.q_init.hw_type;
 770         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
 771         q->dbval = IONIC_DBELL_QID(q->hw_index);
 772
 773         dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
 774         dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
 775
 776         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
 777                 netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi,
 778                                NAPI_POLL_WEIGHT);
 779
 780         qcq->flags |= IONIC_QCQ_F_INITED;
 781
 782         return 0;
 783 }
 784
 785 static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 786 {
 787         struct device *dev = lif->ionic->dev;
 788         struct ionic_queue *q = &qcq->q;
 789         struct ionic_cq *cq = &qcq->cq;
 790         struct ionic_admin_ctx ctx = {
 791                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 792                 .cmd.q_init = {
 793                         .opcode = IONIC_CMD_Q_INIT,
 794                         .lif_index = cpu_to_le16(lif->index),
 795                         .type = q->type,
 796                         .ver = lif->qtype_info[q->type].version,
 797                         .index = cpu_to_le32(q->index),
 798                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 799                                              IONIC_QINIT_F_SG),
 800                         .intr_index = cpu_to_le16(cq->bound_intr->index),
 801                         .pid = cpu_to_le16(q->pid),
 802                         .ring_size = ilog2(q->num_descs),
 803                         .ring_base = cpu_to_le64(q->base_pa),
 804                         .cq_ring_base = cpu_to_le64(cq->base_pa),
 805                         .sg_ring_base = cpu_to_le64(q->sg_base_pa),
 806                         .features = cpu_to_le64(q->features),
 807                 },
 808         };
 809         int err;
 810
 811         dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
 812         dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
 813         dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
 814         dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
 815         dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
 816         dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
 817         dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
 818
 819         ionic_qcq_sanitize(qcq);
 820
 821         err = ionic_adminq_post_wait(lif, &ctx);
 822         if (err)
 823                 return err;
 824
 825         q->hw_type = ctx.comp.q_init.hw_type;
 826         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
 827         q->dbval = IONIC_DBELL_QID(q->hw_index);
 828
 829         dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
 830         dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
 831
 832         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
 833                 netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi,
 834                                NAPI_POLL_WEIGHT);
 835         else
 836                 netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi,
 837                                NAPI_POLL_WEIGHT);
 838
 839         qcq->flags |= IONIC_QCQ_F_INITED;
 840
 841         return 0;
 842 }
 843
 844 int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
 845 {
 846         unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
 847         unsigned int txq_i, flags;
 848         struct ionic_qcq *txq;
 849         u64 features;
 850         int err;
 851
 852         if (lif->hwstamp_txq)
 853                 return 0;
 854
 855         features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
 856
 857         num_desc = IONIC_MIN_TXRX_DESC;
 858         desc_sz = sizeof(struct ionic_txq_desc);
 859         comp_sz = 2 * sizeof(struct ionic_txq_comp);
 860
 861         if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
 862             lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
 863                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
 864         else
 865                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
 866
 867         txq_i = lif->ionic->ntxqs_per_lif;
 868         flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
 869
 870         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
 871                               num_desc, desc_sz, comp_sz, sg_desc_sz,
 872                               lif->kern_pid, &txq);
 873         if (err)
 874                 goto err_qcq_alloc;
 875
 876         txq->q.features = features;
 877
 878         ionic_link_qcq_interrupts(lif->adminqcq, txq);
 879         ionic_debugfs_add_qcq(lif, txq);
 880
 881         lif->hwstamp_txq = txq;
 882
 883         if (netif_running(lif->netdev)) {
 884                 err = ionic_lif_txq_init(lif, txq);
 885                 if (err)
 886                         goto err_qcq_init;
 887
 888                 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
 889                         err = ionic_qcq_enable(txq);
 890                         if (err)
 891                                 goto err_qcq_enable;
 892                 }
 893         }
 894
 895         return 0;
 896
 897 err_qcq_enable:
 898         ionic_lif_qcq_deinit(lif, txq);
 899 err_qcq_init:
 900         lif->hwstamp_txq = NULL;
 901         ionic_debugfs_del_qcq(txq);
 902         ionic_qcq_free(lif, txq);
 903         devm_kfree(lif->ionic->dev, txq);
 904 err_qcq_alloc:
 905         return err;
 906 }
 907
 908 int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
 909 {
 910         unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
 911         unsigned int rxq_i, flags;
 912         struct ionic_qcq *rxq;
 913         u64 features;
 914         int err;
 915
 916         if (lif->hwstamp_rxq)
 917                 return 0;
 918
 919         features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
 920
 921         num_desc = IONIC_MIN_TXRX_DESC;
 922         desc_sz = sizeof(struct ionic_rxq_desc);
 923         comp_sz = 2 * sizeof(struct ionic_rxq_comp);
 924         sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
 925
 926         rxq_i = lif->ionic->nrxqs_per_lif;
 927         flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
 928
 929         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
 930                               num_desc, desc_sz, comp_sz, sg_desc_sz,
 931                               lif->kern_pid, &rxq);
 932         if (err)
 933                 goto err_qcq_alloc;
 934
 935         rxq->q.features = features;
 936
 937         ionic_link_qcq_interrupts(lif->adminqcq, rxq);
 938         ionic_debugfs_add_qcq(lif, rxq);
 939
 940         lif->hwstamp_rxq = rxq;
 941
 942         if (netif_running(lif->netdev)) {
 943                 err = ionic_lif_rxq_init(lif, rxq);
 944                 if (err)
 945                         goto err_qcq_init;
 946
 947                 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
 948                         ionic_rx_fill(&rxq->q);
 949                         err = ionic_qcq_enable(rxq);
 950                         if (err)
 951                                 goto err_qcq_enable;
 952                 }
 953         }
 954
 955         return 0;
 956
 957 err_qcq_enable:
 958         ionic_lif_qcq_deinit(lif, rxq);
 959 err_qcq_init:
 960         lif->hwstamp_rxq = NULL;
 961         ionic_debugfs_del_qcq(rxq);
 962         ionic_qcq_free(lif, rxq);
 963         devm_kfree(lif->ionic->dev, rxq);
 964 err_qcq_alloc:
 965         return err;
 966 }
 967
 968 int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
 969 {
 970         struct ionic_queue_params qparam;
 971
 972         ionic_init_queue_params(lif, &qparam);
 973
 974         if (rx_all)
 975                 qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
 976         else
 977                 qparam.rxq_features = 0;
 978
 979         /* if we're not running, just set the values and return */
 980         if (!netif_running(lif->netdev)) {
 981                 lif->rxq_features = qparam.rxq_features;
 982                 return 0;
 983         }
 984
 985         return ionic_reconfigure_queues(lif, &qparam);
 986 }
 987
 988 int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
 989 {
 990         struct ionic_admin_ctx ctx = {
 991                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 992                 .cmd.lif_setattr = {
 993                         .opcode = IONIC_CMD_LIF_SETATTR,
 994                         .index = cpu_to_le16(lif->index),
 995                         .attr = IONIC_LIF_ATTR_TXSTAMP,
 996                         .txstamp_mode = cpu_to_le16(txstamp_mode),
 997                 },
 998         };
 999
1000         return ionic_adminq_post_wait(lif, &ctx);
1001 }
1002
1003 static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
1004 {
1005         struct ionic_admin_ctx ctx = {
1006                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1007                 .cmd.rx_filter_del = {
1008                         .opcode = IONIC_CMD_RX_FILTER_DEL,
1009                         .lif_index = cpu_to_le16(lif->index),
1010                 },
1011         };
1012         struct ionic_rx_filter *f;
1013         u32 filter_id;
1014         int err;
1015
1016         spin_lock_bh(&lif->rx_filters.lock);
1017
1018         f = ionic_rx_filter_rxsteer(lif);
1019         if (!f) {
1020                 spin_unlock_bh(&lif->rx_filters.lock);
1021                 return;
1022         }
1023
1024         filter_id = f->filter_id;
1025         ionic_rx_filter_free(lif, f);
1026
1027         spin_unlock_bh(&lif->rx_filters.lock);
1028
1029         netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
1030
1031         ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
1032
1033         err = ionic_adminq_post_wait(lif, &ctx);
1034         if (err && err != -EEXIST)
1035                 netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
1036 }
1037
1038 static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1039 {
1040         struct ionic_admin_ctx ctx = {
1041                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1042                 .cmd.rx_filter_add = {
1043                         .opcode = IONIC_CMD_RX_FILTER_ADD,
1044                         .lif_index = cpu_to_le16(lif->index),
1045                         .match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
1046                         .pkt_class = cpu_to_le64(pkt_class),
1047                 },
1048         };
1049         u8 qtype;
1050         u32 qid;
1051         int err;
1052
1053         if (!lif->hwstamp_rxq)
1054                 return -EINVAL;
1055
1056         qtype = lif->hwstamp_rxq->q.type;
1057         ctx.cmd.rx_filter_add.qtype = qtype;
1058
1059         qid = lif->hwstamp_rxq->q.index;
1060         ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
1061
1062         netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
1063         err = ionic_adminq_post_wait(lif, &ctx);
1064         if (err && err != -EEXIST)
1065                 return err;
1066
1067         spin_lock_bh(&lif->rx_filters.lock);
1068         err = ionic_rx_filter_save(lif, 0, qid, 0, &ctx, IONIC_FILTER_STATE_SYNCED);
1069         spin_unlock_bh(&lif->rx_filters.lock);
1070
1071         return err;
1072 }
1073
1074 int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1075 {
1076         ionic_lif_del_hwstamp_rxfilt(lif);
1077
1078         if (!pkt_class)
1079                 return 0;
1080
1081         return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
1082 }
1083
1084 static bool ionic_notifyq_service(struct ionic_cq *cq,
1085                                   struct ionic_cq_info *cq_info)
1086 {
1087         union ionic_notifyq_comp *comp = cq_info->cq_desc;
1088         struct ionic_deferred_work *work;
1089         struct net_device *netdev;
1090         struct ionic_queue *q;
1091         struct ionic_lif *lif;
1092         u64 eid;
1093
1094         q = cq->bound_q;
1095         lif = q->info[0].cb_arg;
1096         netdev = lif->netdev;
1097         eid = le64_to_cpu(comp->event.eid);
1098
1099         /* Have we run out of new completions to process? */
1100         if ((s64)(eid - lif->last_eid) <= 0)
1101                 return false;
1102
1103         lif->last_eid = eid;
1104
1105         dev_dbg(lif->ionic->dev, "notifyq event:\n");
1106         dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET, 16, 1,
1107                          comp, sizeof(*comp), true);
1108
1109         switch (le16_to_cpu(comp->event.ecode)) {
1110         case IONIC_EVENT_LINK_CHANGE:
1111                 ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1112                 break;
1113         case IONIC_EVENT_RESET:
1114                 work = kzalloc(sizeof(*work), GFP_ATOMIC);
1115                 if (!work) {
1116                         netdev_err(lif->netdev, "Reset event dropped\n");
1117                 } else {
1118                         work->type = IONIC_DW_TYPE_LIF_RESET;
1119                         ionic_lif_deferred_enqueue(&lif->deferred, work);
1120                 }
1121                 break;
1122         default:
1123                 netdev_warn(netdev, "Notifyq event ecode=%d eid=%lld\n",
1124                             comp->event.ecode, eid);
1125                 break;
1126         }
1127
1128         return true;
1129 }
1130
1131 static bool ionic_adminq_service(struct ionic_cq *cq,
1132                                  struct ionic_cq_info *cq_info)
1133 {
1134         struct ionic_admin_comp *comp = cq_info->cq_desc;
1135
1136         if (!color_match(comp->color, cq->done_color))
1137                 return false;
1138
1139         ionic_q_service(cq->bound_q, cq_info, le16_to_cpu(comp->comp_index));
1140
1141         return true;
1142 }
1143
1144 static int ionic_adminq_napi(struct napi_struct *napi, int budget)
1145 {
1146         struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
1147         struct ionic_lif *lif = napi_to_cq(napi)->lif;
1148         struct ionic_dev *idev = &lif->ionic->idev;
1149         unsigned long irqflags;
1150         unsigned int flags = 0;
1151         int rx_work = 0;
1152         int tx_work = 0;
1153         int n_work = 0;
1154         int a_work = 0;
1155         int work_done;
1156         int credits;
1157
1158         if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
1159                 n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
1160                                           ionic_notifyq_service, NULL, NULL);
1161
1162         spin_lock_irqsave(&lif->adminq_lock, irqflags);
1163         if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
1164                 a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
1165                                           ionic_adminq_service, NULL, NULL);
1166         spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
1167
1168         if (lif->hwstamp_rxq)
1169                 rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
1170                                            ionic_rx_service, NULL, NULL);
1171
1172         if (lif->hwstamp_txq)
1173                 tx_work = ionic_cq_service(&lif->hwstamp_txq->cq, budget,
1174                                            ionic_tx_service, NULL, NULL);
1175
1176         work_done = max(max(n_work, a_work), max(rx_work, tx_work));
1177         if (work_done < budget && napi_complete_done(napi, work_done)) {
1178                 flags |= IONIC_INTR_CRED_UNMASK;
1179                 intr->rearm_count++;
1180         }
1181
1182         if (work_done || flags) {
1183                 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1184                 credits = n_work + a_work + rx_work + tx_work;
1185                 ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
1186         }
1187
1188         return work_done;
1189 }
1190
1191 void ionic_get_stats64(struct net_device *netdev,
1192                        struct rtnl_link_stats64 *ns)
1193 {
1194         struct ionic_lif *lif = netdev_priv(netdev);
1195         struct ionic_lif_stats *ls;
1196
1197         memset(ns, 0, sizeof(*ns));
1198         ls = &lif->info->stats;
1199
1200         ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
1201                          le64_to_cpu(ls->rx_mcast_packets) +
1202                          le64_to_cpu(ls->rx_bcast_packets);
1203
1204         ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
1205                          le64_to_cpu(ls->tx_mcast_packets) +
1206                          le64_to_cpu(ls->tx_bcast_packets);
1207
1208         ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
1209                        le64_to_cpu(ls->rx_mcast_bytes) +
1210                        le64_to_cpu(ls->rx_bcast_bytes);
1211
1212         ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
1213                        le64_to_cpu(ls->tx_mcast_bytes) +
1214                        le64_to_cpu(ls->tx_bcast_bytes);
1215
1216         ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
1217                          le64_to_cpu(ls->rx_mcast_drop_packets) +
1218                          le64_to_cpu(ls->rx_bcast_drop_packets);
1219
1220         ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
1221                          le64_to_cpu(ls->tx_mcast_drop_packets) +
1222                          le64_to_cpu(ls->tx_bcast_drop_packets);
1223
1224         ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
1225
1226         ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
1227
1228         ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
1229                                le64_to_cpu(ls->rx_queue_disabled) +
1230                                le64_to_cpu(ls->rx_desc_fetch_error) +
1231                                le64_to_cpu(ls->rx_desc_data_error);
1232
1233         ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
1234                                 le64_to_cpu(ls->tx_queue_disabled) +
1235                                 le64_to_cpu(ls->tx_desc_fetch_error) +
1236                                 le64_to_cpu(ls->tx_desc_data_error);
1237
1238         ns->rx_errors = ns->rx_over_errors +
1239                         ns->rx_missed_errors;
1240
1241         ns->tx_errors = ns->tx_aborted_errors;
1242 }
1243
1244 int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr)
1245 {
1246         struct ionic_admin_ctx ctx = {
1247                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1248                 .cmd.rx_filter_add = {
1249                         .opcode = IONIC_CMD_RX_FILTER_ADD,
1250                         .lif_index = cpu_to_le16(lif->index),
1251                         .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_MAC),
1252                 },
1253         };
1254         int nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1255         bool mc = is_multicast_ether_addr(addr);
1256         struct ionic_rx_filter *f;
1257         int err = 0;
1258
1259         memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, ETH_ALEN);
1260
1261         spin_lock_bh(&lif->rx_filters.lock);
1262         f = ionic_rx_filter_by_addr(lif, addr);
1263         if (f) {
1264                 /* don't bother if we already have it and it is sync'd */
1265                 if (f->state == IONIC_FILTER_STATE_SYNCED) {
1266                         spin_unlock_bh(&lif->rx_filters.lock);
1267                         return 0;
1268                 }
1269
1270                 /* mark preemptively as sync'd to block any parallel attempts */
1271                 f->state = IONIC_FILTER_STATE_SYNCED;
1272         } else {
1273                 /* save as SYNCED to catch any DEL requests while processing */
1274                 err = ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx,
1275                                            IONIC_FILTER_STATE_SYNCED);
1276         }
1277         spin_unlock_bh(&lif->rx_filters.lock);
1278         if (err)
1279                 return err;
1280
1281         netdev_dbg(lif->netdev, "rx_filter add ADDR %pM\n", addr);
1282
1283         /* Don't bother with the write to FW if we know there's no room,
1284          * we can try again on the next sync attempt.
1285          */
1286         if ((lif->nucast + lif->nmcast) >= nfilters)
1287                 err = -ENOSPC;
1288         else
1289                 err = ionic_adminq_post_wait(lif, &ctx);
1290
1291         spin_lock_bh(&lif->rx_filters.lock);
1292         if (err && err != -EEXIST) {
1293                 /* set the state back to NEW so we can try again later */
1294                 f = ionic_rx_filter_by_addr(lif, addr);
1295                 if (f && f->state == IONIC_FILTER_STATE_SYNCED)
1296                         f->state = IONIC_FILTER_STATE_NEW;
1297
1298                 spin_unlock_bh(&lif->rx_filters.lock);
1299
1300                 if (err == -ENOSPC)
1301                         return 0;
1302                 else
1303                         return err;
1304         }
1305
1306         if (mc)
1307                 lif->nmcast++;
1308         else
1309                 lif->nucast++;
1310
1311         f = ionic_rx_filter_by_addr(lif, addr);
1312         if (f && f->state == IONIC_FILTER_STATE_OLD) {
1313                 /* Someone requested a delete while we were adding
1314                  * so update the filter info with the results from the add
1315                  * and the data will be there for the delete on the next
1316                  * sync cycle.
1317                  */
1318                 err = ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx,
1319                                            IONIC_FILTER_STATE_OLD);
1320         } else {
1321                 err = ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx,
1322                                            IONIC_FILTER_STATE_SYNCED);
1323         }
1324
1325         spin_unlock_bh(&lif->rx_filters.lock);
1326
1327         return err;
1328 }
1329
1330 int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr)
1331 {
1332         struct ionic_admin_ctx ctx = {
1333                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1334                 .cmd.rx_filter_del = {
1335                         .opcode = IONIC_CMD_RX_FILTER_DEL,
1336                         .lif_index = cpu_to_le16(lif->index),
1337                 },
1338         };
1339         struct ionic_rx_filter *f;
1340         int state;
1341         int err;
1342
1343         spin_lock_bh(&lif->rx_filters.lock);
1344         f = ionic_rx_filter_by_addr(lif, addr);
1345         if (!f) {
1346                 spin_unlock_bh(&lif->rx_filters.lock);
1347                 return -ENOENT;
1348         }
1349
1350         netdev_dbg(lif->netdev, "rx_filter del ADDR %pM (id %d)\n",
1351                    addr, f->filter_id);
1352
1353         state = f->state;
1354         ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1355         ionic_rx_filter_free(lif, f);
1356
1357         if (is_multicast_ether_addr(addr) && lif->nmcast)
1358                 lif->nmcast--;
1359         else if (!is_multicast_ether_addr(addr) && lif->nucast)
1360                 lif->nucast--;
1361
1362         spin_unlock_bh(&lif->rx_filters.lock);
1363
1364         if (state != IONIC_FILTER_STATE_NEW) {
1365                 err = ionic_adminq_post_wait(lif, &ctx);
1366                 if (err && err != -EEXIST)
1367                         return err;
1368         }
1369
1370         return 0;
1371 }
1372
1373 static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
1374 {
1375         return ionic_lif_list_addr(netdev_priv(netdev), addr, ADD_ADDR);
1376 }
1377
1378 static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
1379 {
1380         return ionic_lif_list_addr(netdev_priv(netdev), addr, DEL_ADDR);
1381 }
1382
1383 void ionic_lif_rx_mode(struct ionic_lif *lif)
1384 {
1385         struct net_device *netdev = lif->netdev;
1386         unsigned int nfilters;
1387         unsigned int nd_flags;
1388         char buf[128];
1389         u16 rx_mode;
1390         int i;
1391 #define REMAIN(__x) (sizeof(buf) - (__x))
1392
1393         mutex_lock(&lif->config_lock);
1394
1395         /* grab the flags once for local use */
1396         nd_flags = netdev->flags;
1397
1398         rx_mode = IONIC_RX_MODE_F_UNICAST;
1399         rx_mode |= (nd_flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
1400         rx_mode |= (nd_flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
1401         rx_mode |= (nd_flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
1402         rx_mode |= (nd_flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
1403
1404         /* sync the mac filters */
1405         ionic_rx_filter_sync(lif);
1406
1407         /* check for overflow state
1408          *    if so, we track that we overflowed and enable NIC PROMISC
1409          *    else if the overflow is set and not needed
1410          *       we remove our overflow flag and check the netdev flags
1411          *       to see if we can disable NIC PROMISC
1412          */
1413         nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1414         if ((lif->nucast + lif->nmcast) >= nfilters) {
1415                 rx_mode |= IONIC_RX_MODE_F_PROMISC;
1416                 rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
1417                 lif->uc_overflow = true;
1418                 lif->mc_overflow = true;
1419         } else if (lif->uc_overflow) {
1420                 lif->uc_overflow = false;
1421                 lif->mc_overflow = false;
1422                 if (!(nd_flags & IFF_PROMISC))
1423                         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
1424                 if (!(nd_flags & IFF_ALLMULTI))
1425                         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
1426         }
1427
1428         i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
1429                       lif->rx_mode, rx_mode);
1430         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
1431                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
1432         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
1433                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
1434         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
1435                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
1436         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
1437                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
1438         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
1439                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
1440         if (rx_mode & IONIC_RX_MODE_F_RDMA_SNIFFER)
1441                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_RDMA_SNIFFER");
1442         netdev_dbg(netdev, "lif%d %s\n", lif->index, buf);
1443
1444         if (lif->rx_mode != rx_mode) {
1445                 struct ionic_admin_ctx ctx = {
1446                         .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1447                         .cmd.rx_mode_set = {
1448                                 .opcode = IONIC_CMD_RX_MODE_SET,
1449                                 .lif_index = cpu_to_le16(lif->index),
1450                         },
1451                 };
1452                 int err;
1453
1454                 ctx.cmd.rx_mode_set.rx_mode = cpu_to_le16(rx_mode);
1455                 err = ionic_adminq_post_wait(lif, &ctx);
1456                 if (err)
1457                         netdev_warn(netdev, "set rx_mode 0x%04x failed: %d\n",
1458                                     rx_mode, err);
1459                 else
1460                         lif->rx_mode = rx_mode;
1461         }
1462
1463         mutex_unlock(&lif->config_lock);
1464 }
1465
1466 static void ionic_ndo_set_rx_mode(struct net_device *netdev)
1467 {
1468         struct ionic_lif *lif = netdev_priv(netdev);
1469         struct ionic_deferred_work *work;
1470
1471         /* Sync the kernel filter list with the driver filter list */
1472         __dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
1473         __dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
1474
1475         /* Shove off the rest of the rxmode work to the work task
1476          * which will include syncing the filters to the firmware.
1477          */
1478         work = kzalloc(sizeof(*work), GFP_ATOMIC);
1479         if (!work) {
1480                 netdev_err(lif->netdev, "rxmode change dropped\n");
1481                 return;
1482         }
1483         work->type = IONIC_DW_TYPE_RX_MODE;
1484         netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1485         ionic_lif_deferred_enqueue(&lif->deferred, work);
1486 }
1487
1488 static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
1489 {
1490         u64 wanted = 0;
1491
1492         if (features & NETIF_F_HW_VLAN_CTAG_TX)
1493                 wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
1494         if (features & NETIF_F_HW_VLAN_CTAG_RX)
1495                 wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
1496         if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1497                 wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
1498         if (features & NETIF_F_RXHASH)
1499                 wanted |= IONIC_ETH_HW_RX_HASH;
1500         if (features & NETIF_F_RXCSUM)
1501                 wanted |= IONIC_ETH_HW_RX_CSUM;
1502         if (features & NETIF_F_SG)
1503                 wanted |= IONIC_ETH_HW_TX_SG;
1504         if (features & NETIF_F_HW_CSUM)
1505                 wanted |= IONIC_ETH_HW_TX_CSUM;
1506         if (features & NETIF_F_TSO)
1507                 wanted |= IONIC_ETH_HW_TSO;
1508         if (features & NETIF_F_TSO6)
1509                 wanted |= IONIC_ETH_HW_TSO_IPV6;
1510         if (features & NETIF_F_TSO_ECN)
1511                 wanted |= IONIC_ETH_HW_TSO_ECN;
1512         if (features & NETIF_F_GSO_GRE)
1513                 wanted |= IONIC_ETH_HW_TSO_GRE;
1514         if (features & NETIF_F_GSO_GRE_CSUM)
1515                 wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
1516         if (features & NETIF_F_GSO_IPXIP4)
1517                 wanted |= IONIC_ETH_HW_TSO_IPXIP4;
1518         if (features & NETIF_F_GSO_IPXIP6)
1519                 wanted |= IONIC_ETH_HW_TSO_IPXIP6;
1520         if (features & NETIF_F_GSO_UDP_TUNNEL)
1521                 wanted |= IONIC_ETH_HW_TSO_UDP;
1522         if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
1523                 wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
1524
1525         return cpu_to_le64(wanted);
1526 }
1527
1528 static int ionic_set_nic_features(struct ionic_lif *lif,
1529                                   netdev_features_t features)
1530 {
1531         struct device *dev = lif->ionic->dev;
1532         struct ionic_admin_ctx ctx = {
1533                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1534                 .cmd.lif_setattr = {
1535                         .opcode = IONIC_CMD_LIF_SETATTR,
1536                         .index = cpu_to_le16(lif->index),
1537                         .attr = IONIC_LIF_ATTR_FEATURES,
1538                 },
1539         };
1540         u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
1541                          IONIC_ETH_HW_VLAN_RX_STRIP |
1542                          IONIC_ETH_HW_VLAN_RX_FILTER;
1543         u64 old_hw_features;
1544         int err;
1545
1546         ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
1547
1548         if (lif->phc)
1549                 ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
1550
1551         err = ionic_adminq_post_wait(lif, &ctx);
1552         if (err)
1553                 return err;
1554
1555         old_hw_features = lif->hw_features;
1556         lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
1557                                        ctx.comp.lif_setattr.features);
1558
1559         if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
1560                 ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1561
1562         if ((vlan_flags & features) &&
1563             !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
1564                 dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1565
1566         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1567                 dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
1568         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1569                 dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
1570         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1571                 dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
1572         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1573                 dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
1574         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1575                 dev_dbg(dev, "feature ETH_HW_TX_SG\n");
1576         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1577                 dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
1578         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1579                 dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
1580         if (lif->hw_features & IONIC_ETH_HW_TSO)
1581                 dev_dbg(dev, "feature ETH_HW_TSO\n");
1582         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1583                 dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
1584         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1585                 dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
1586         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1587                 dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
1588         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1589                 dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
1590         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1591                 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
1592         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1593                 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
1594         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1595                 dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
1596         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1597                 dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
1598         if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
1599                 dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
1600
1601         return 0;
1602 }
1603
1604 static int ionic_init_nic_features(struct ionic_lif *lif)
1605 {
1606         struct net_device *netdev = lif->netdev;
1607         netdev_features_t features;
1608         int err;
1609
1610         /* set up what we expect to support by default */
1611         features = NETIF_F_HW_VLAN_CTAG_TX |
1612                    NETIF_F_HW_VLAN_CTAG_RX |
1613                    NETIF_F_HW_VLAN_CTAG_FILTER |
1614                    NETIF_F_SG |
1615                    NETIF_F_HW_CSUM |
1616                    NETIF_F_RXCSUM |
1617                    NETIF_F_TSO |
1618                    NETIF_F_TSO6 |
1619                    NETIF_F_TSO_ECN;
1620
1621         if (lif->nxqs > 1)
1622                 features |= NETIF_F_RXHASH;
1623
1624         err = ionic_set_nic_features(lif, features);
1625         if (err)
1626                 return err;
1627
1628         /* tell the netdev what we actually can support */
1629         netdev->features |= NETIF_F_HIGHDMA;
1630
1631         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1632                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
1633         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1634                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1635         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1636                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1637         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1638                 netdev->hw_features |= NETIF_F_RXHASH;
1639         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1640                 netdev->hw_features |= NETIF_F_SG;
1641
1642         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1643                 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
1644         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1645                 netdev->hw_enc_features |= NETIF_F_RXCSUM;
1646         if (lif->hw_features & IONIC_ETH_HW_TSO)
1647                 netdev->hw_enc_features |= NETIF_F_TSO;
1648         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1649                 netdev->hw_enc_features |= NETIF_F_TSO6;
1650         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1651                 netdev->hw_enc_features |= NETIF_F_TSO_ECN;
1652         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1653                 netdev->hw_enc_features |= NETIF_F_GSO_GRE;
1654         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1655                 netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
1656         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1657                 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
1658         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1659                 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
1660         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1661                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
1662         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1663                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1664
1665         netdev->hw_features |= netdev->hw_enc_features;
1666         netdev->features |= netdev->hw_features;
1667         netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
1668
1669         netdev->priv_flags |= IFF_UNICAST_FLT |
1670                               IFF_LIVE_ADDR_CHANGE;
1671
1672         return 0;
1673 }
1674
1675 static int ionic_set_features(struct net_device *netdev,
1676                               netdev_features_t features)
1677 {
1678         struct ionic_lif *lif = netdev_priv(netdev);
1679         int err;
1680
1681         netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1682                    __func__, (u64)lif->netdev->features, (u64)features);
1683
1684         err = ionic_set_nic_features(lif, features);
1685
1686         return err;
1687 }
1688
1689 static int ionic_set_mac_address(struct net_device *netdev, void *sa)
1690 {
1691         struct sockaddr *addr = sa;
1692         u8 *mac;
1693         int err;
1694
1695         mac = (u8 *)addr->sa_data;
1696         if (ether_addr_equal(netdev->dev_addr, mac))
1697                 return 0;
1698
1699         err = eth_prepare_mac_addr_change(netdev, addr);
1700         if (err)
1701                 return err;
1702
1703         if (!is_zero_ether_addr(netdev->dev_addr)) {
1704                 netdev_info(netdev, "deleting mac addr %pM\n",
1705                             netdev->dev_addr);
1706                 ionic_lif_addr_del(netdev_priv(netdev), netdev->dev_addr);
1707         }
1708
1709         eth_commit_mac_addr_change(netdev, addr);
1710         netdev_info(netdev, "updating mac addr %pM\n", mac);
1711
1712         return ionic_lif_addr_add(netdev_priv(netdev), mac);
1713 }
1714
1715 static void ionic_stop_queues_reconfig(struct ionic_lif *lif)
1716 {
1717         /* Stop and clean the queues before reconfiguration */
1718         netif_device_detach(lif->netdev);
1719         ionic_stop_queues(lif);
1720         ionic_txrx_deinit(lif);
1721 }
1722
1723 static int ionic_start_queues_reconfig(struct ionic_lif *lif)
1724 {
1725         int err;
1726
1727         /* Re-init the queues after reconfiguration */
1728
1729         /* The only way txrx_init can fail here is if communication
1730          * with FW is suddenly broken.  There's not much we can do
1731          * at this point - error messages have already been printed,
1732          * so we can continue on and the user can eventually do a
1733          * DOWN and UP to try to reset and clear the issue.
1734          */
1735         err = ionic_txrx_init(lif);
1736         ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1737         netif_device_attach(lif->netdev);
1738
1739         return err;
1740 }
1741
1742 static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
1743 {
1744         struct ionic_lif *lif = netdev_priv(netdev);
1745         struct ionic_admin_ctx ctx = {
1746                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1747                 .cmd.lif_setattr = {
1748                         .opcode = IONIC_CMD_LIF_SETATTR,
1749                         .index = cpu_to_le16(lif->index),
1750                         .attr = IONIC_LIF_ATTR_MTU,
1751                         .mtu = cpu_to_le32(new_mtu),
1752                 },
1753         };
1754         int err;
1755
1756         err = ionic_adminq_post_wait(lif, &ctx);
1757         if (err)
1758                 return err;
1759
1760         /* if we're not running, nothing more to do */
1761         if (!netif_running(netdev)) {
1762                 netdev->mtu = new_mtu;
1763                 return 0;
1764         }
1765
1766         mutex_lock(&lif->queue_lock);
1767         ionic_stop_queues_reconfig(lif);
1768         netdev->mtu = new_mtu;
1769         err = ionic_start_queues_reconfig(lif);
1770         mutex_unlock(&lif->queue_lock);
1771
1772         return err;
1773 }
1774
1775 static void ionic_tx_timeout_work(struct work_struct *ws)
1776 {
1777         struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
1778
1779         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
1780                 return;
1781
1782         /* if we were stopped before this scheduled job was launched,
1783          * don't bother the queues as they are already stopped.
1784          */
1785         if (!netif_running(lif->netdev))
1786                 return;
1787
1788         mutex_lock(&lif->queue_lock);
1789         ionic_stop_queues_reconfig(lif);
1790         ionic_start_queues_reconfig(lif);
1791         mutex_unlock(&lif->queue_lock);
1792 }
1793
1794 static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1795 {
1796         struct ionic_lif *lif = netdev_priv(netdev);
1797
1798         netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
1799         schedule_work(&lif->tx_timeout_work);
1800 }
1801
1802 static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1803                                  u16 vid)
1804 {
1805         struct ionic_lif *lif = netdev_priv(netdev);
1806         struct ionic_admin_ctx ctx = {
1807                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1808                 .cmd.rx_filter_add = {
1809                         .opcode = IONIC_CMD_RX_FILTER_ADD,
1810                         .lif_index = cpu_to_le16(lif->index),
1811                         .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_VLAN),
1812                         .vlan.vlan = cpu_to_le16(vid),
1813                 },
1814         };
1815         int err;
1816
1817         netdev_dbg(netdev, "rx_filter add VLAN %d\n", vid);
1818         err = ionic_adminq_post_wait(lif, &ctx);
1819         if (err)
1820                 return err;
1821
1822         spin_lock_bh(&lif->rx_filters.lock);
1823         err = ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx,
1824                                    IONIC_FILTER_STATE_SYNCED);
1825         spin_unlock_bh(&lif->rx_filters.lock);
1826
1827         return err;
1828 }
1829
1830 static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1831                                   u16 vid)
1832 {
1833         struct ionic_lif *lif = netdev_priv(netdev);
1834         struct ionic_admin_ctx ctx = {
1835                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1836                 .cmd.rx_filter_del = {
1837                         .opcode = IONIC_CMD_RX_FILTER_DEL,
1838                         .lif_index = cpu_to_le16(lif->index),
1839                 },
1840         };
1841         struct ionic_rx_filter *f;
1842
1843         spin_lock_bh(&lif->rx_filters.lock);
1844
1845         f = ionic_rx_filter_by_vlan(lif, vid);
1846         if (!f) {
1847                 spin_unlock_bh(&lif->rx_filters.lock);
1848                 return -ENOENT;
1849         }
1850
1851         netdev_dbg(netdev, "rx_filter del VLAN %d (id %d)\n",
1852                    vid, f->filter_id);
1853
1854         ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1855         ionic_rx_filter_free(lif, f);
1856         spin_unlock_bh(&lif->rx_filters.lock);
1857
1858         return ionic_adminq_post_wait(lif, &ctx);
1859 }
1860
1861 int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
1862                          const u8 *key, const u32 *indir)
1863 {
1864         struct ionic_admin_ctx ctx = {
1865                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1866                 .cmd.lif_setattr = {
1867                         .opcode = IONIC_CMD_LIF_SETATTR,
1868                         .attr = IONIC_LIF_ATTR_RSS,
1869                         .rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
1870                 },
1871         };
1872         unsigned int i, tbl_sz;
1873
1874         if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
1875                 lif->rss_types = types;
1876                 ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
1877         }
1878
1879         if (key)
1880                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
1881
1882         if (indir) {
1883                 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1884                 for (i = 0; i < tbl_sz; i++)
1885                         lif->rss_ind_tbl[i] = indir[i];
1886         }
1887
1888         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
1889                IONIC_RSS_HASH_KEY_SIZE);
1890
1891         return ionic_adminq_post_wait(lif, &ctx);
1892 }
1893
1894 static int ionic_lif_rss_init(struct ionic_lif *lif)
1895 {
1896         unsigned int tbl_sz;
1897         unsigned int i;
1898
1899         lif->rss_types = IONIC_RSS_TYPE_IPV4     |
1900                          IONIC_RSS_TYPE_IPV4_TCP |
1901                          IONIC_RSS_TYPE_IPV4_UDP |
1902                          IONIC_RSS_TYPE_IPV6     |
1903                          IONIC_RSS_TYPE_IPV6_TCP |
1904                          IONIC_RSS_TYPE_IPV6_UDP;
1905
1906         /* Fill indirection table with 'default' values */
1907         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1908         for (i = 0; i < tbl_sz; i++)
1909                 lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
1910
1911         return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1912 }
1913
1914 static void ionic_lif_rss_deinit(struct ionic_lif *lif)
1915 {
1916         int tbl_sz;
1917
1918         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1919         memset(lif->rss_ind_tbl, 0, tbl_sz);
1920         memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
1921
1922         ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1923 }
1924
1925 static void ionic_lif_quiesce(struct ionic_lif *lif)
1926 {
1927         struct ionic_admin_ctx ctx = {
1928                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1929                 .cmd.lif_setattr = {
1930                         .opcode = IONIC_CMD_LIF_SETATTR,
1931                         .index = cpu_to_le16(lif->index),
1932                         .attr = IONIC_LIF_ATTR_STATE,
1933                         .state = IONIC_LIF_QUIESCE,
1934                 },
1935         };
1936         int err;
1937
1938         err = ionic_adminq_post_wait(lif, &ctx);
1939         if (err)
1940                 netdev_err(lif->netdev, "lif quiesce failed %d\n", err);
1941 }
1942
1943 static void ionic_txrx_disable(struct ionic_lif *lif)
1944 {
1945         unsigned int i;
1946         int err = 0;
1947
1948         if (lif->txqcqs) {
1949                 for (i = 0; i < lif->nxqs; i++)
1950                         err = ionic_qcq_disable(lif->txqcqs[i], (err != -ETIMEDOUT));
1951         }
1952
1953         if (lif->hwstamp_txq)
1954                 err = ionic_qcq_disable(lif->hwstamp_txq, (err != -ETIMEDOUT));
1955
1956         if (lif->rxqcqs) {
1957                 for (i = 0; i < lif->nxqs; i++)
1958                         err = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
1959         }
1960
1961         if (lif->hwstamp_rxq)
1962                 err = ionic_qcq_disable(lif->hwstamp_rxq, (err != -ETIMEDOUT));
1963
1964         ionic_lif_quiesce(lif);
1965 }
1966
1967 static void ionic_txrx_deinit(struct ionic_lif *lif)
1968 {
1969         unsigned int i;
1970
1971         if (lif->txqcqs) {
1972                 for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
1973                         ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1974                         ionic_tx_flush(&lif->txqcqs[i]->cq);
1975                         ionic_tx_empty(&lif->txqcqs[i]->q);
1976                 }
1977         }
1978
1979         if (lif->rxqcqs) {
1980                 for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
1981                         ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
1982                         ionic_rx_empty(&lif->rxqcqs[i]->q);
1983                 }
1984         }
1985         lif->rx_mode = 0;
1986
1987         if (lif->hwstamp_txq) {
1988                 ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
1989                 ionic_tx_flush(&lif->hwstamp_txq->cq);
1990                 ionic_tx_empty(&lif->hwstamp_txq->q);
1991         }
1992
1993         if (lif->hwstamp_rxq) {
1994                 ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
1995                 ionic_rx_empty(&lif->hwstamp_rxq->q);
1996         }
1997 }
1998
1999 static void ionic_txrx_free(struct ionic_lif *lif)
2000 {
2001         unsigned int i;
2002
2003         if (lif->txqcqs) {
2004                 for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
2005                         ionic_qcq_free(lif, lif->txqcqs[i]);
2006                         devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
2007                         lif->txqcqs[i] = NULL;
2008                 }
2009         }
2010
2011         if (lif->rxqcqs) {
2012                 for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
2013                         ionic_qcq_free(lif, lif->rxqcqs[i]);
2014                         devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
2015                         lif->rxqcqs[i] = NULL;
2016                 }
2017         }
2018
2019         if (lif->hwstamp_txq) {
2020                 ionic_qcq_free(lif, lif->hwstamp_txq);
2021                 devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
2022                 lif->hwstamp_txq = NULL;
2023         }
2024
2025         if (lif->hwstamp_rxq) {
2026                 ionic_qcq_free(lif, lif->hwstamp_rxq);
2027                 devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
2028                 lif->hwstamp_rxq = NULL;
2029         }
2030 }
2031
2032 static int ionic_txrx_alloc(struct ionic_lif *lif)
2033 {
2034         unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2035         unsigned int flags, i;
2036         int err = 0;
2037
2038         num_desc = lif->ntxq_descs;
2039         desc_sz = sizeof(struct ionic_txq_desc);
2040         comp_sz = sizeof(struct ionic_txq_comp);
2041
2042         if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2043             lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2044                                           sizeof(struct ionic_txq_sg_desc_v1))
2045                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2046         else
2047                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2048
2049         flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2050         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2051                 flags |= IONIC_QCQ_F_INTR;
2052         for (i = 0; i < lif->nxqs; i++) {
2053                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2054                                       num_desc, desc_sz, comp_sz, sg_desc_sz,
2055                                       lif->kern_pid, &lif->txqcqs[i]);
2056                 if (err)
2057                         goto err_out;
2058
2059                 if (flags & IONIC_QCQ_F_INTR) {
2060                         ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2061                                              lif->txqcqs[i]->intr.index,
2062                                              lif->tx_coalesce_hw);
2063                         if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2064                                 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2065                 }
2066
2067                 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2068         }
2069
2070         flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
2071
2072         num_desc = lif->nrxq_descs;
2073         desc_sz = sizeof(struct ionic_rxq_desc);
2074         comp_sz = sizeof(struct ionic_rxq_comp);
2075         sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2076
2077         if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2078                 comp_sz *= 2;
2079
2080         for (i = 0; i < lif->nxqs; i++) {
2081                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2082                                       num_desc, desc_sz, comp_sz, sg_desc_sz,
2083                                       lif->kern_pid, &lif->rxqcqs[i]);
2084                 if (err)
2085                         goto err_out;
2086
2087                 lif->rxqcqs[i]->q.features = lif->rxq_features;
2088
2089                 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2090                                      lif->rxqcqs[i]->intr.index,
2091                                      lif->rx_coalesce_hw);
2092                 if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
2093                         lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
2094
2095                 if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2096                         ionic_link_qcq_interrupts(lif->rxqcqs[i],
2097                                                   lif->txqcqs[i]);
2098
2099                 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2100         }
2101
2102         return 0;
2103
2104 err_out:
2105         ionic_txrx_free(lif);
2106
2107         return err;
2108 }
2109
2110 static int ionic_txrx_init(struct ionic_lif *lif)
2111 {
2112         unsigned int i;
2113         int err;
2114
2115         for (i = 0; i < lif->nxqs; i++) {
2116                 err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
2117                 if (err)
2118                         goto err_out;
2119
2120                 err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
2121                 if (err) {
2122                         ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2123                         goto err_out;
2124                 }
2125         }
2126
2127         if (lif->netdev->features & NETIF_F_RXHASH)
2128                 ionic_lif_rss_init(lif);
2129
2130         ionic_lif_rx_mode(lif);
2131
2132         return 0;
2133
2134 err_out:
2135         while (i--) {
2136                 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2137                 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2138         }
2139
2140         return err;
2141 }
2142
2143 static int ionic_txrx_enable(struct ionic_lif *lif)
2144 {
2145         int derr = 0;
2146         int i, err;
2147
2148         for (i = 0; i < lif->nxqs; i++) {
2149                 if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
2150                         dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
2151                         err = -ENXIO;
2152                         goto err_out;
2153                 }
2154
2155                 ionic_rx_fill(&lif->rxqcqs[i]->q);
2156                 err = ionic_qcq_enable(lif->rxqcqs[i]);
2157                 if (err)
2158                         goto err_out;
2159
2160                 err = ionic_qcq_enable(lif->txqcqs[i]);
2161                 if (err) {
2162                         derr = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
2163                         goto err_out;
2164                 }
2165         }
2166
2167         if (lif->hwstamp_rxq) {
2168                 ionic_rx_fill(&lif->hwstamp_rxq->q);
2169                 err = ionic_qcq_enable(lif->hwstamp_rxq);
2170                 if (err)
2171                         goto err_out_hwstamp_rx;
2172         }
2173
2174         if (lif->hwstamp_txq) {
2175                 err = ionic_qcq_enable(lif->hwstamp_txq);
2176                 if (err)
2177                         goto err_out_hwstamp_tx;
2178         }
2179
2180         return 0;
2181
2182 err_out_hwstamp_tx:
2183         if (lif->hwstamp_rxq)
2184                 derr = ionic_qcq_disable(lif->hwstamp_rxq, (derr != -ETIMEDOUT));
2185 err_out_hwstamp_rx:
2186         i = lif->nxqs;
2187 err_out:
2188         while (i--) {
2189                 derr = ionic_qcq_disable(lif->txqcqs[i], (derr != -ETIMEDOUT));
2190                 derr = ionic_qcq_disable(lif->rxqcqs[i], (derr != -ETIMEDOUT));
2191         }
2192
2193         return err;
2194 }
2195
2196 static int ionic_start_queues(struct ionic_lif *lif)
2197 {
2198         int err;
2199
2200         if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
2201                 return -EIO;
2202
2203         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2204                 return -EBUSY;
2205
2206         if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
2207                 return 0;
2208
2209         err = ionic_txrx_enable(lif);
2210         if (err) {
2211                 clear_bit(IONIC_LIF_F_UP, lif->state);
2212                 return err;
2213         }
2214         netif_tx_wake_all_queues(lif->netdev);
2215
2216         return 0;
2217 }
2218
2219 static int ionic_open(struct net_device *netdev)
2220 {
2221         struct ionic_lif *lif = netdev_priv(netdev);
2222         int err;
2223
2224         /* If recovering from a broken state, clear the bit and we'll try again */
2225         if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
2226                 netdev_info(netdev, "clearing broken state\n");
2227
2228         mutex_lock(&lif->queue_lock);
2229
2230         err = ionic_txrx_alloc(lif);
2231         if (err)
2232                 goto err_unlock;
2233
2234         err = ionic_txrx_init(lif);
2235         if (err)
2236                 goto err_txrx_free;
2237
2238         err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
2239         if (err)
2240                 goto err_txrx_deinit;
2241
2242         err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
2243         if (err)
2244                 goto err_txrx_deinit;
2245
2246         /* don't start the queues until we have link */
2247         if (netif_carrier_ok(netdev)) {
2248                 err = ionic_start_queues(lif);
2249                 if (err)
2250                         goto err_txrx_deinit;
2251         }
2252
2253         /* If hardware timestamping is enabled, but the queues were freed by
2254          * ionic_stop, those need to be reallocated and initialized, too.
2255          */
2256         ionic_lif_hwstamp_recreate_queues(lif);
2257
2258         mutex_unlock(&lif->queue_lock);
2259
2260         return 0;
2261
2262 err_txrx_deinit:
2263         ionic_txrx_deinit(lif);
2264 err_txrx_free:
2265         ionic_txrx_free(lif);
2266 err_unlock:
2267         mutex_unlock(&lif->queue_lock);
2268         return err;
2269 }
2270
2271 static void ionic_stop_queues(struct ionic_lif *lif)
2272 {
2273         if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
2274                 return;
2275
2276         netif_tx_disable(lif->netdev);
2277         ionic_txrx_disable(lif);
2278 }
2279
2280 static int ionic_stop(struct net_device *netdev)
2281 {
2282         struct ionic_lif *lif = netdev_priv(netdev);
2283
2284         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2285                 return 0;
2286
2287         mutex_lock(&lif->queue_lock);
2288         ionic_stop_queues(lif);
2289         ionic_txrx_deinit(lif);
2290         ionic_txrx_free(lif);
2291         mutex_unlock(&lif->queue_lock);
2292
2293         return 0;
2294 }
2295
2296 static int ionic_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2297 {
2298         struct ionic_lif *lif = netdev_priv(netdev);
2299
2300         switch (cmd) {
2301         case SIOCSHWTSTAMP:
2302                 return ionic_lif_hwstamp_set(lif, ifr);
2303         case SIOCGHWTSTAMP:
2304                 return ionic_lif_hwstamp_get(lif, ifr);
2305         default:
2306                 return -EOPNOTSUPP;
2307         }
2308 }
2309
2310 static int ionic_get_vf_config(struct net_device *netdev,
2311                                int vf, struct ifla_vf_info *ivf)
2312 {
2313         struct ionic_lif *lif = netdev_priv(netdev);
2314         struct ionic *ionic = lif->ionic;
2315         int ret = 0;
2316
2317         if (!netif_device_present(netdev))
2318                 return -EBUSY;
2319
2320         down_read(&ionic->vf_op_lock);
2321
2322         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2323                 ret = -EINVAL;
2324         } else {
2325                 ivf->vf           = vf;
2326                 ivf->vlan         = le16_to_cpu(ionic->vfs[vf].vlanid);
2327                 ivf->qos          = 0;
2328                 ivf->spoofchk     = ionic->vfs[vf].spoofchk;
2329                 ivf->linkstate    = ionic->vfs[vf].linkstate;
2330                 ivf->max_tx_rate  = le32_to_cpu(ionic->vfs[vf].maxrate);
2331                 ivf->trusted      = ionic->vfs[vf].trusted;
2332                 ether_addr_copy(ivf->mac, ionic->vfs[vf].macaddr);
2333         }
2334
2335         up_read(&ionic->vf_op_lock);
2336         return ret;
2337 }
2338
2339 static int ionic_get_vf_stats(struct net_device *netdev, int vf,
2340                               struct ifla_vf_stats *vf_stats)
2341 {
2342         struct ionic_lif *lif = netdev_priv(netdev);
2343         struct ionic *ionic = lif->ionic;
2344         struct ionic_lif_stats *vs;
2345         int ret = 0;
2346
2347         if (!netif_device_present(netdev))
2348                 return -EBUSY;
2349
2350         down_read(&ionic->vf_op_lock);
2351
2352         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2353                 ret = -EINVAL;
2354         } else {
2355                 memset(vf_stats, 0, sizeof(*vf_stats));
2356                 vs = &ionic->vfs[vf].stats;
2357
2358                 vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
2359                 vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
2360                 vf_stats->rx_bytes   = le64_to_cpu(vs->rx_ucast_bytes);
2361                 vf_stats->tx_bytes   = le64_to_cpu(vs->tx_ucast_bytes);
2362                 vf_stats->broadcast  = le64_to_cpu(vs->rx_bcast_packets);
2363                 vf_stats->multicast  = le64_to_cpu(vs->rx_mcast_packets);
2364                 vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
2365                                        le64_to_cpu(vs->rx_mcast_drop_packets) +
2366                                        le64_to_cpu(vs->rx_bcast_drop_packets);
2367                 vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
2368                                        le64_to_cpu(vs->tx_mcast_drop_packets) +
2369                                        le64_to_cpu(vs->tx_bcast_drop_packets);
2370         }
2371
2372         up_read(&ionic->vf_op_lock);
2373         return ret;
2374 }
2375
2376 static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2377 {
2378         struct ionic_lif *lif = netdev_priv(netdev);
2379         struct ionic *ionic = lif->ionic;
2380         int ret;
2381
2382         if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
2383                 return -EINVAL;
2384
2385         if (!netif_device_present(netdev))
2386                 return -EBUSY;
2387
2388         down_write(&ionic->vf_op_lock);
2389
2390         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2391                 ret = -EINVAL;
2392         } else {
2393                 ret = ionic_set_vf_config(ionic, vf, IONIC_VF_ATTR_MAC, mac);
2394                 if (!ret)
2395                         ether_addr_copy(ionic->vfs[vf].macaddr, mac);
2396         }
2397
2398         up_write(&ionic->vf_op_lock);
2399         return ret;
2400 }
2401
2402 static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2403                              u8 qos, __be16 proto)
2404 {
2405         struct ionic_lif *lif = netdev_priv(netdev);
2406         struct ionic *ionic = lif->ionic;
2407         int ret;
2408
2409         /* until someday when we support qos */
2410         if (qos)
2411                 return -EINVAL;
2412
2413         if (vlan > 4095)
2414                 return -EINVAL;
2415
2416         if (proto != htons(ETH_P_8021Q))
2417                 return -EPROTONOSUPPORT;
2418
2419         if (!netif_device_present(netdev))
2420                 return -EBUSY;
2421
2422         down_write(&ionic->vf_op_lock);
2423
2424         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2425                 ret = -EINVAL;
2426         } else {
2427                 ret = ionic_set_vf_config(ionic, vf,
2428                                           IONIC_VF_ATTR_VLAN, (u8 *)&vlan);
2429                 if (!ret)
2430                         ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
2431         }
2432
2433         up_write(&ionic->vf_op_lock);
2434         return ret;
2435 }
2436
2437 static int ionic_set_vf_rate(struct net_device *netdev, int vf,
2438                              int tx_min, int tx_max)
2439 {
2440         struct ionic_lif *lif = netdev_priv(netdev);
2441         struct ionic *ionic = lif->ionic;
2442         int ret;
2443
2444         /* setting the min just seems silly */
2445         if (tx_min)
2446                 return -EINVAL;
2447
2448         if (!netif_device_present(netdev))
2449                 return -EBUSY;
2450
2451         down_write(&ionic->vf_op_lock);
2452
2453         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2454                 ret = -EINVAL;
2455         } else {
2456                 ret = ionic_set_vf_config(ionic, vf,
2457                                           IONIC_VF_ATTR_RATE, (u8 *)&tx_max);
2458                 if (!ret)
2459                         lif->ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
2460         }
2461
2462         up_write(&ionic->vf_op_lock);
2463         return ret;
2464 }
2465
2466 static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
2467 {
2468         struct ionic_lif *lif = netdev_priv(netdev);
2469         struct ionic *ionic = lif->ionic;
2470         u8 data = set;  /* convert to u8 for config */
2471         int ret;
2472
2473         if (!netif_device_present(netdev))
2474                 return -EBUSY;
2475
2476         down_write(&ionic->vf_op_lock);
2477
2478         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2479                 ret = -EINVAL;
2480         } else {
2481                 ret = ionic_set_vf_config(ionic, vf,
2482                                           IONIC_VF_ATTR_SPOOFCHK, &data);
2483                 if (!ret)
2484                         ionic->vfs[vf].spoofchk = data;
2485         }
2486
2487         up_write(&ionic->vf_op_lock);
2488         return ret;
2489 }
2490
2491 static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
2492 {
2493         struct ionic_lif *lif = netdev_priv(netdev);
2494         struct ionic *ionic = lif->ionic;
2495         u8 data = set;  /* convert to u8 for config */
2496         int ret;
2497
2498         if (!netif_device_present(netdev))
2499                 return -EBUSY;
2500
2501         down_write(&ionic->vf_op_lock);
2502
2503         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2504                 ret = -EINVAL;
2505         } else {
2506                 ret = ionic_set_vf_config(ionic, vf,
2507                                           IONIC_VF_ATTR_TRUST, &data);
2508                 if (!ret)
2509                         ionic->vfs[vf].trusted = data;
2510         }
2511
2512         up_write(&ionic->vf_op_lock);
2513         return ret;
2514 }
2515
2516 static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
2517 {
2518         struct ionic_lif *lif = netdev_priv(netdev);
2519         struct ionic *ionic = lif->ionic;
2520         u8 data;
2521         int ret;
2522
2523         switch (set) {
2524         case IFLA_VF_LINK_STATE_ENABLE:
2525                 data = IONIC_VF_LINK_STATUS_UP;
2526                 break;
2527         case IFLA_VF_LINK_STATE_DISABLE:
2528                 data = IONIC_VF_LINK_STATUS_DOWN;
2529                 break;
2530         case IFLA_VF_LINK_STATE_AUTO:
2531                 data = IONIC_VF_LINK_STATUS_AUTO;
2532                 break;
2533         default:
2534                 return -EINVAL;
2535         }
2536
2537         if (!netif_device_present(netdev))
2538                 return -EBUSY;
2539
2540         down_write(&ionic->vf_op_lock);
2541
2542         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2543                 ret = -EINVAL;
2544         } else {
2545                 ret = ionic_set_vf_config(ionic, vf,
2546                                           IONIC_VF_ATTR_LINKSTATE, &data);
2547                 if (!ret)
2548                         ionic->vfs[vf].linkstate = set;
2549         }
2550
2551         up_write(&ionic->vf_op_lock);
2552         return ret;
2553 }
2554
2555 static const struct net_device_ops ionic_netdev_ops = {
2556         .ndo_open               = ionic_open,
2557         .ndo_stop               = ionic_stop,
2558         .ndo_eth_ioctl          = ionic_eth_ioctl,
2559         .ndo_start_xmit         = ionic_start_xmit,
2560         .ndo_get_stats64        = ionic_get_stats64,
2561         .ndo_set_rx_mode        = ionic_ndo_set_rx_mode,
2562         .ndo_set_features       = ionic_set_features,
2563         .ndo_set_mac_address    = ionic_set_mac_address,
2564         .ndo_validate_addr      = eth_validate_addr,
2565         .ndo_tx_timeout         = ionic_tx_timeout,
2566         .ndo_change_mtu         = ionic_change_mtu,
2567         .ndo_vlan_rx_add_vid    = ionic_vlan_rx_add_vid,
2568         .ndo_vlan_rx_kill_vid   = ionic_vlan_rx_kill_vid,
2569         .ndo_set_vf_vlan        = ionic_set_vf_vlan,
2570         .ndo_set_vf_trust       = ionic_set_vf_trust,
2571         .ndo_set_vf_mac         = ionic_set_vf_mac,
2572         .ndo_set_vf_rate        = ionic_set_vf_rate,
2573         .ndo_set_vf_spoofchk    = ionic_set_vf_spoofchk,
2574         .ndo_get_vf_config      = ionic_get_vf_config,
2575         .ndo_set_vf_link_state  = ionic_set_vf_link_state,
2576         .ndo_get_vf_stats       = ionic_get_vf_stats,
2577 };
2578
2579 static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2580 {
2581         /* only swapping the queues, not the napi, flags, or other stuff */
2582         swap(a->q.features,   b->q.features);
2583         swap(a->q.num_descs,  b->q.num_descs);
2584         swap(a->q.desc_size,  b->q.desc_size);
2585         swap(a->q.base,       b->q.base);
2586         swap(a->q.base_pa,    b->q.base_pa);
2587         swap(a->q.info,       b->q.info);
2588         swap(a->q_base,       b->q_base);
2589         swap(a->q_base_pa,    b->q_base_pa);
2590         swap(a->q_size,       b->q_size);
2591
2592         swap(a->q.sg_desc_size, b->q.sg_desc_size);
2593         swap(a->q.sg_base,    b->q.sg_base);
2594         swap(a->q.sg_base_pa, b->q.sg_base_pa);
2595         swap(a->sg_base,      b->sg_base);
2596         swap(a->sg_base_pa,   b->sg_base_pa);
2597         swap(a->sg_size,      b->sg_size);
2598
2599         swap(a->cq.num_descs, b->cq.num_descs);
2600         swap(a->cq.desc_size, b->cq.desc_size);
2601         swap(a->cq.base,      b->cq.base);
2602         swap(a->cq.base_pa,   b->cq.base_pa);
2603         swap(a->cq.info,      b->cq.info);
2604         swap(a->cq_base,      b->cq_base);
2605         swap(a->cq_base_pa,   b->cq_base_pa);
2606         swap(a->cq_size,      b->cq_size);
2607
2608         ionic_debugfs_del_qcq(a);
2609         ionic_debugfs_add_qcq(a->q.lif, a);
2610 }
2611
2612 int ionic_reconfigure_queues(struct ionic_lif *lif,
2613                              struct ionic_queue_params *qparam)
2614 {
2615         unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2616         struct ionic_qcq **tx_qcqs = NULL;
2617         struct ionic_qcq **rx_qcqs = NULL;
2618         unsigned int flags, i;
2619         int err = 0;
2620
2621         /* allocate temporary qcq arrays to hold new queue structs */
2622         if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
2623                 tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
2624                                        sizeof(struct ionic_qcq *), GFP_KERNEL);
2625                 if (!tx_qcqs) {
2626                         err = -ENOMEM;
2627                         goto err_out;
2628                 }
2629         }
2630         if (qparam->nxqs != lif->nxqs ||
2631             qparam->nrxq_descs != lif->nrxq_descs ||
2632             qparam->rxq_features != lif->rxq_features) {
2633                 rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
2634                                        sizeof(struct ionic_qcq *), GFP_KERNEL);
2635                 if (!rx_qcqs) {
2636                         err = -ENOMEM;
2637                         goto err_out;
2638                 }
2639         }
2640
2641         /* allocate new desc_info and rings, but leave the interrupt setup
2642          * until later so as to not mess with the still-running queues
2643          */
2644         if (tx_qcqs) {
2645                 num_desc = qparam->ntxq_descs;
2646                 desc_sz = sizeof(struct ionic_txq_desc);
2647                 comp_sz = sizeof(struct ionic_txq_comp);
2648
2649                 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2650                     lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2651                     sizeof(struct ionic_txq_sg_desc_v1))
2652                         sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2653                 else
2654                         sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2655
2656                 for (i = 0; i < qparam->nxqs; i++) {
2657                         flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2658                         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2659                                               num_desc, desc_sz, comp_sz, sg_desc_sz,
2660                                               lif->kern_pid, &tx_qcqs[i]);
2661                         if (err)
2662                                 goto err_out;
2663                 }
2664         }
2665
2666         if (rx_qcqs) {
2667                 num_desc = qparam->nrxq_descs;
2668                 desc_sz = sizeof(struct ionic_rxq_desc);
2669                 comp_sz = sizeof(struct ionic_rxq_comp);
2670                 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2671
2672                 if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2673                         comp_sz *= 2;
2674
2675                 for (i = 0; i < qparam->nxqs; i++) {
2676                         flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2677                         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2678                                               num_desc, desc_sz, comp_sz, sg_desc_sz,
2679                                               lif->kern_pid, &rx_qcqs[i]);
2680                         if (err)
2681                                 goto err_out;
2682
2683                         rx_qcqs[i]->q.features = qparam->rxq_features;
2684                 }
2685         }
2686
2687         /* stop and clean the queues */
2688         ionic_stop_queues_reconfig(lif);
2689
2690         if (qparam->nxqs != lif->nxqs) {
2691                 err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
2692                 if (err)
2693                         goto err_out_reinit_unlock;
2694                 err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
2695                 if (err) {
2696                         netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
2697                         goto err_out_reinit_unlock;
2698                 }
2699         }
2700
2701         /* swap new desc_info and rings, keeping existing interrupt config */
2702         if (tx_qcqs) {
2703                 lif->ntxq_descs = qparam->ntxq_descs;
2704                 for (i = 0; i < qparam->nxqs; i++)
2705                         ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
2706         }
2707
2708         if (rx_qcqs) {
2709                 lif->nrxq_descs = qparam->nrxq_descs;
2710                 for (i = 0; i < qparam->nxqs; i++)
2711                         ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
2712         }
2713
2714         /* if we need to change the interrupt layout, this is the time */
2715         if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
2716             qparam->nxqs != lif->nxqs) {
2717                 if (qparam->intr_split) {
2718                         set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2719                 } else {
2720                         clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2721                         lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2722                         lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2723                 }
2724
2725                 /* clear existing interrupt assignments */
2726                 for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
2727                         ionic_qcq_intr_free(lif, lif->txqcqs[i]);
2728                         ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
2729                 }
2730
2731                 /* re-assign the interrupts */
2732                 for (i = 0; i < qparam->nxqs; i++) {
2733                         lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2734                         err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
2735                         ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2736                                              lif->rxqcqs[i]->intr.index,
2737                                              lif->rx_coalesce_hw);
2738
2739                         if (qparam->intr_split) {
2740                                 lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2741                                 err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
2742                                 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2743                                                      lif->txqcqs[i]->intr.index,
2744                                                      lif->tx_coalesce_hw);
2745                                 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2746                                         lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2747                         } else {
2748                                 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2749                                 ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
2750                         }
2751                 }
2752         }
2753
2754         /* now we can rework the debugfs mappings */
2755         if (tx_qcqs) {
2756                 for (i = 0; i < qparam->nxqs; i++) {
2757                         ionic_debugfs_del_qcq(lif->txqcqs[i]);
2758                         ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2759                 }
2760         }
2761
2762         if (rx_qcqs) {
2763                 for (i = 0; i < qparam->nxqs; i++) {
2764                         ionic_debugfs_del_qcq(lif->rxqcqs[i]);
2765                         ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2766                 }
2767         }
2768
2769         swap(lif->nxqs, qparam->nxqs);
2770         swap(lif->rxq_features, qparam->rxq_features);
2771
2772 err_out_reinit_unlock:
2773         /* re-init the queues, but don't lose an error code */
2774         if (err)
2775                 ionic_start_queues_reconfig(lif);
2776         else
2777                 err = ionic_start_queues_reconfig(lif);
2778
2779 err_out:
2780         /* free old allocs without cleaning intr */
2781         for (i = 0; i < qparam->nxqs; i++) {
2782                 if (tx_qcqs && tx_qcqs[i]) {
2783                         tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2784                         ionic_qcq_free(lif, tx_qcqs[i]);
2785                         devm_kfree(lif->ionic->dev, tx_qcqs[i]);
2786                         tx_qcqs[i] = NULL;
2787                 }
2788                 if (rx_qcqs && rx_qcqs[i]) {
2789                         rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2790                         ionic_qcq_free(lif, rx_qcqs[i]);
2791                         devm_kfree(lif->ionic->dev, rx_qcqs[i]);
2792                         rx_qcqs[i] = NULL;
2793                 }
2794         }
2795
2796         /* free q array */
2797         if (rx_qcqs) {
2798                 devm_kfree(lif->ionic->dev, rx_qcqs);
2799                 rx_qcqs = NULL;
2800         }
2801         if (tx_qcqs) {
2802                 devm_kfree(lif->ionic->dev, tx_qcqs);
2803                 tx_qcqs = NULL;
2804         }
2805
2806         /* clean the unused dma and info allocations when new set is smaller
2807          * than the full array, but leave the qcq shells in place
2808          */
2809         for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
2810                 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2811                 ionic_qcq_free(lif, lif->txqcqs[i]);
2812
2813                 lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2814                 ionic_qcq_free(lif, lif->rxqcqs[i]);
2815         }
2816
2817         if (err)
2818                 netdev_info(lif->netdev, "%s: failed %d\n", __func__, err);
2819
2820         return err;
2821 }
2822
2823 int ionic_lif_alloc(struct ionic *ionic)
2824 {
2825         struct device *dev = ionic->dev;
2826         union ionic_lif_identity *lid;
2827         struct net_device *netdev;
2828         struct ionic_lif *lif;
2829         int tbl_sz;
2830         int err;
2831
2832         lid = kzalloc(sizeof(*lid), GFP_KERNEL);
2833         if (!lid)
2834                 return -ENOMEM;
2835
2836         netdev = alloc_etherdev_mqs(sizeof(*lif),
2837                                     ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
2838         if (!netdev) {
2839                 dev_err(dev, "Cannot allocate netdev, aborting\n");
2840                 err = -ENOMEM;
2841                 goto err_out_free_lid;
2842         }
2843
2844         SET_NETDEV_DEV(netdev, dev);
2845
2846         lif = netdev_priv(netdev);
2847         lif->netdev = netdev;
2848         ionic->lif = lif;
2849         netdev->netdev_ops = &ionic_netdev_ops;
2850         ionic_ethtool_set_ops(netdev);
2851
2852         netdev->watchdog_timeo = 2 * HZ;
2853         netif_carrier_off(netdev);
2854
2855         lif->identity = lid;
2856         lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
2857         err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
2858         if (err) {
2859                 dev_err(ionic->dev, "Cannot identify type %d: %d\n",
2860                         lif->lif_type, err);
2861                 goto err_out_free_netdev;
2862         }
2863         lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
2864                                      le32_to_cpu(lif->identity->eth.min_frame_size));
2865         lif->netdev->max_mtu =
2866                 le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
2867
2868         lif->neqs = ionic->neqs_per_lif;
2869         lif->nxqs = ionic->ntxqs_per_lif;
2870
2871         lif->ionic = ionic;
2872         lif->index = 0;
2873
2874         if (is_kdump_kernel()) {
2875                 lif->ntxq_descs = IONIC_MIN_TXRX_DESC;
2876                 lif->nrxq_descs = IONIC_MIN_TXRX_DESC;
2877         } else {
2878                 lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
2879                 lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
2880         }
2881
2882         /* Convert the default coalesce value to actual hw resolution */
2883         lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
2884         lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
2885                                                     lif->rx_coalesce_usecs);
2886         lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2887         lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2888         set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
2889         set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
2890
2891         snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
2892
2893         spin_lock_init(&lif->adminq_lock);
2894
2895         spin_lock_init(&lif->deferred.lock);
2896         INIT_LIST_HEAD(&lif->deferred.list);
2897         INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
2898
2899         /* allocate lif info */
2900         lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
2901         lif->info = dma_alloc_coherent(dev, lif->info_sz,
2902                                        &lif->info_pa, GFP_KERNEL);
2903         if (!lif->info) {
2904                 dev_err(dev, "Failed to allocate lif info, aborting\n");
2905                 err = -ENOMEM;
2906                 goto err_out_free_netdev;
2907         }
2908
2909         ionic_debugfs_add_lif(lif);
2910
2911         /* allocate control queues and txrx queue arrays */
2912         ionic_lif_queue_identify(lif);
2913         err = ionic_qcqs_alloc(lif);
2914         if (err)
2915                 goto err_out_free_lif_info;
2916
2917         /* allocate rss indirection table */
2918         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
2919         lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
2920         lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
2921                                               &lif->rss_ind_tbl_pa,
2922                                               GFP_KERNEL);
2923
2924         if (!lif->rss_ind_tbl) {
2925                 err = -ENOMEM;
2926                 dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
2927                 goto err_out_free_qcqs;
2928         }
2929         netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
2930
2931         ionic_lif_alloc_phc(lif);
2932
2933         return 0;
2934
2935 err_out_free_qcqs:
2936         ionic_qcqs_free(lif);
2937 err_out_free_lif_info:
2938         dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
2939         lif->info = NULL;
2940         lif->info_pa = 0;
2941 err_out_free_netdev:
2942         free_netdev(lif->netdev);
2943         lif = NULL;
2944 err_out_free_lid:
2945         kfree(lid);
2946
2947         return err;
2948 }
2949
2950 static void ionic_lif_reset(struct ionic_lif *lif)
2951 {
2952         struct ionic_dev *idev = &lif->ionic->idev;
2953
2954         mutex_lock(&lif->ionic->dev_cmd_lock);
2955         ionic_dev_cmd_lif_reset(idev, lif->index);
2956         ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
2957         mutex_unlock(&lif->ionic->dev_cmd_lock);
2958 }
2959
2960 static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
2961 {
2962         struct ionic *ionic = lif->ionic;
2963
2964         if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
2965                 return;
2966
2967         dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
2968
2969         netif_device_detach(lif->netdev);
2970
2971         if (test_bit(IONIC_LIF_F_UP, lif->state)) {
2972                 dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
2973                 mutex_lock(&lif->queue_lock);
2974                 ionic_stop_queues(lif);
2975                 mutex_unlock(&lif->queue_lock);
2976         }
2977
2978         if (netif_running(lif->netdev)) {
2979                 ionic_txrx_deinit(lif);
2980                 ionic_txrx_free(lif);
2981         }
2982         ionic_lif_deinit(lif);
2983         ionic_reset(ionic);
2984         ionic_qcqs_free(lif);
2985
2986         dev_info(ionic->dev, "FW Down: LIFs stopped\n");
2987 }
2988
2989 static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
2990 {
2991         struct ionic *ionic = lif->ionic;
2992         int err;
2993
2994         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2995                 return;
2996
2997         dev_info(ionic->dev, "FW Up: restarting LIFs\n");
2998
2999         ionic_init_devinfo(ionic);
3000         err = ionic_identify(ionic);
3001         if (err)
3002                 goto err_out;
3003         err = ionic_port_identify(ionic);
3004         if (err)
3005                 goto err_out;
3006         err = ionic_port_init(ionic);
3007         if (err)
3008                 goto err_out;
3009         err = ionic_qcqs_alloc(lif);
3010         if (err)
3011                 goto err_out;
3012
3013         err = ionic_lif_init(lif);
3014         if (err)
3015                 goto err_qcqs_free;
3016
3017         if (lif->registered)
3018                 ionic_lif_set_netdev_info(lif);
3019
3020         ionic_rx_filter_replay(lif);
3021
3022         if (netif_running(lif->netdev)) {
3023                 err = ionic_txrx_alloc(lif);
3024                 if (err)
3025                         goto err_lifs_deinit;
3026
3027                 err = ionic_txrx_init(lif);
3028                 if (err)
3029                         goto err_txrx_free;
3030         }
3031
3032         clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
3033         ionic_link_status_check_request(lif, CAN_SLEEP);
3034         netif_device_attach(lif->netdev);
3035         dev_info(ionic->dev, "FW Up: LIFs restarted\n");
3036
3037         /* restore the hardware timestamping queues */
3038         ionic_lif_hwstamp_replay(lif);
3039
3040         return;
3041
3042 err_txrx_free:
3043         ionic_txrx_free(lif);
3044 err_lifs_deinit:
3045         ionic_lif_deinit(lif);
3046 err_qcqs_free:
3047         ionic_qcqs_free(lif);
3048 err_out:
3049         dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
3050 }
3051
3052 void ionic_lif_free(struct ionic_lif *lif)
3053 {
3054         struct device *dev = lif->ionic->dev;
3055
3056         ionic_lif_free_phc(lif);
3057
3058         /* free rss indirection table */
3059         dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
3060                           lif->rss_ind_tbl_pa);
3061         lif->rss_ind_tbl = NULL;
3062         lif->rss_ind_tbl_pa = 0;
3063
3064         /* free queues */
3065         ionic_qcqs_free(lif);
3066         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3067                 ionic_lif_reset(lif);
3068
3069         /* free lif info */
3070         kfree(lif->identity);
3071         dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3072         lif->info = NULL;
3073         lif->info_pa = 0;
3074
3075         /* unmap doorbell page */
3076         ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3077         lif->kern_dbpage = NULL;
3078         kfree(lif->dbid_inuse);
3079         lif->dbid_inuse = NULL;
3080
3081         /* free netdev & lif */
3082         ionic_debugfs_del_lif(lif);
3083         free_netdev(lif->netdev);
3084 }
3085
3086 void ionic_lif_deinit(struct ionic_lif *lif)
3087 {
3088         if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
3089                 return;
3090
3091         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3092                 cancel_work_sync(&lif->deferred.work);
3093                 cancel_work_sync(&lif->tx_timeout_work);
3094                 ionic_rx_filters_deinit(lif);
3095                 if (lif->netdev->features & NETIF_F_RXHASH)
3096                         ionic_lif_rss_deinit(lif);
3097         }
3098
3099         napi_disable(&lif->adminqcq->napi);
3100         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3101         ionic_lif_qcq_deinit(lif, lif->adminqcq);
3102
3103         mutex_destroy(&lif->config_lock);
3104         mutex_destroy(&lif->queue_lock);
3105         ionic_lif_reset(lif);
3106 }
3107
3108 static int ionic_lif_adminq_init(struct ionic_lif *lif)
3109 {
3110         struct device *dev = lif->ionic->dev;
3111         struct ionic_q_init_comp comp;
3112         struct ionic_dev *idev;
3113         struct ionic_qcq *qcq;
3114         struct ionic_queue *q;
3115         int err;
3116
3117         idev = &lif->ionic->idev;
3118         qcq = lif->adminqcq;
3119         q = &qcq->q;
3120
3121         mutex_lock(&lif->ionic->dev_cmd_lock);
3122         ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
3123         err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3124         ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3125         mutex_unlock(&lif->ionic->dev_cmd_lock);
3126         if (err) {
3127                 netdev_err(lif->netdev, "adminq init failed %d\n", err);
3128                 return err;
3129         }
3130
3131         q->hw_type = comp.hw_type;
3132         q->hw_index = le32_to_cpu(comp.hw_index);
3133         q->dbval = IONIC_DBELL_QID(q->hw_index);
3134
3135         dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
3136         dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
3137
3138         netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi,
3139                        NAPI_POLL_WEIGHT);
3140
3141         napi_enable(&qcq->napi);
3142
3143         if (qcq->flags & IONIC_QCQ_F_INTR)
3144                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
3145                                 IONIC_INTR_MASK_CLEAR);
3146
3147         qcq->flags |= IONIC_QCQ_F_INITED;
3148
3149         return 0;
3150 }
3151
3152 static int ionic_lif_notifyq_init(struct ionic_lif *lif)
3153 {
3154         struct ionic_qcq *qcq = lif->notifyqcq;
3155         struct device *dev = lif->ionic->dev;
3156         struct ionic_queue *q = &qcq->q;
3157         int err;
3158
3159         struct ionic_admin_ctx ctx = {
3160                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3161                 .cmd.q_init = {
3162                         .opcode = IONIC_CMD_Q_INIT,
3163                         .lif_index = cpu_to_le16(lif->index),
3164                         .type = q->type,
3165                         .ver = lif->qtype_info[q->type].version,
3166                         .index = cpu_to_le32(q->index),
3167                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
3168                                              IONIC_QINIT_F_ENA),
3169                         .intr_index = cpu_to_le16(lif->adminqcq->intr.index),
3170                         .pid = cpu_to_le16(q->pid),
3171                         .ring_size = ilog2(q->num_descs),
3172                         .ring_base = cpu_to_le64(q->base_pa),
3173                 }
3174         };
3175
3176         dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
3177         dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
3178         dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
3179         dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
3180
3181         err = ionic_adminq_post_wait(lif, &ctx);
3182         if (err)
3183                 return err;
3184
3185         lif->last_eid = 0;
3186         q->hw_type = ctx.comp.q_init.hw_type;
3187         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
3188         q->dbval = IONIC_DBELL_QID(q->hw_index);
3189
3190         dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
3191         dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
3192
3193         /* preset the callback info */
3194         q->info[0].cb_arg = lif;
3195
3196         qcq->flags |= IONIC_QCQ_F_INITED;
3197
3198         return 0;
3199 }
3200
3201 static int ionic_station_set(struct ionic_lif *lif)
3202 {
3203         struct net_device *netdev = lif->netdev;
3204         struct ionic_admin_ctx ctx = {
3205                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3206                 .cmd.lif_getattr = {
3207                         .opcode = IONIC_CMD_LIF_GETATTR,
3208                         .index = cpu_to_le16(lif->index),
3209                         .attr = IONIC_LIF_ATTR_MAC,
3210                 },
3211         };
3212         struct sockaddr addr;
3213         int err;
3214
3215         err = ionic_adminq_post_wait(lif, &ctx);
3216         if (err)
3217                 return err;
3218         netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
3219                    ctx.comp.lif_getattr.mac);
3220         if (is_zero_ether_addr(ctx.comp.lif_getattr.mac))
3221                 return 0;
3222
3223         if (!is_zero_ether_addr(netdev->dev_addr)) {
3224                 /* If the netdev mac is non-zero and doesn't match the default
3225                  * device address, it was set by something earlier and we're
3226                  * likely here again after a fw-upgrade reset.  We need to be
3227                  * sure the netdev mac is in our filter list.
3228                  */
3229                 if (!ether_addr_equal(ctx.comp.lif_getattr.mac,
3230                                       netdev->dev_addr))
3231                         ionic_lif_addr_add(lif, netdev->dev_addr);
3232         } else {
3233                 /* Update the netdev mac with the device's mac */
3234                 memcpy(addr.sa_data, ctx.comp.lif_getattr.mac, netdev->addr_len);
3235                 addr.sa_family = AF_INET;
3236                 err = eth_prepare_mac_addr_change(netdev, &addr);
3237                 if (err) {
3238                         netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
3239                                     addr.sa_data, err);
3240                         return 0;
3241                 }
3242
3243                 eth_commit_mac_addr_change(netdev, &addr);
3244         }
3245
3246         netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
3247                    netdev->dev_addr);
3248         ionic_lif_addr_add(lif, netdev->dev_addr);
3249
3250         return 0;
3251 }
3252
3253 int ionic_lif_init(struct ionic_lif *lif)
3254 {
3255         struct ionic_dev *idev = &lif->ionic->idev;
3256         struct device *dev = lif->ionic->dev;
3257         struct ionic_lif_init_comp comp;
3258         int dbpage_num;
3259         int err;
3260
3261         mutex_lock(&lif->ionic->dev_cmd_lock);
3262         ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
3263         err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3264         ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3265         mutex_unlock(&lif->ionic->dev_cmd_lock);
3266         if (err)
3267                 return err;
3268
3269         lif->hw_index = le16_to_cpu(comp.hw_index);
3270         mutex_init(&lif->queue_lock);
3271         mutex_init(&lif->config_lock);
3272
3273         /* now that we have the hw_index we can figure out our doorbell page */
3274         lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
3275         if (!lif->dbid_count) {
3276                 dev_err(dev, "No doorbell pages, aborting\n");
3277                 return -EINVAL;
3278         }
3279
3280         lif->dbid_inuse = bitmap_alloc(lif->dbid_count, GFP_KERNEL);
3281         if (!lif->dbid_inuse) {
3282                 dev_err(dev, "Failed alloc doorbell id bitmap, aborting\n");
3283                 return -ENOMEM;
3284         }
3285
3286         /* first doorbell id reserved for kernel (dbid aka pid == zero) */
3287         set_bit(0, lif->dbid_inuse);
3288         lif->kern_pid = 0;
3289
3290         dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
3291         lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
3292         if (!lif->kern_dbpage) {
3293                 dev_err(dev, "Cannot map dbpage, aborting\n");
3294                 err = -ENOMEM;
3295                 goto err_out_free_dbid;
3296         }
3297
3298         err = ionic_lif_adminq_init(lif);
3299         if (err)
3300                 goto err_out_adminq_deinit;
3301
3302         if (lif->ionic->nnqs_per_lif) {
3303                 err = ionic_lif_notifyq_init(lif);
3304                 if (err)
3305                         goto err_out_notifyq_deinit;
3306         }
3307
3308         err = ionic_init_nic_features(lif);
3309         if (err)
3310                 goto err_out_notifyq_deinit;
3311
3312         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3313                 err = ionic_rx_filters_init(lif);
3314                 if (err)
3315                         goto err_out_notifyq_deinit;
3316         }
3317
3318         err = ionic_station_set(lif);
3319         if (err)
3320                 goto err_out_notifyq_deinit;
3321
3322         lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
3323
3324         set_bit(IONIC_LIF_F_INITED, lif->state);
3325
3326         INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
3327
3328         return 0;
3329
3330 err_out_notifyq_deinit:
3331         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3332 err_out_adminq_deinit:
3333         ionic_lif_qcq_deinit(lif, lif->adminqcq);
3334         ionic_lif_reset(lif);
3335         ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3336         lif->kern_dbpage = NULL;
3337 err_out_free_dbid:
3338         kfree(lif->dbid_inuse);
3339         lif->dbid_inuse = NULL;
3340
3341         return err;
3342 }
3343
3344 static void ionic_lif_notify_work(struct work_struct *ws)
3345 {
3346 }
3347
3348 static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
3349 {
3350         struct ionic_admin_ctx ctx = {
3351                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3352                 .cmd.lif_setattr = {
3353                         .opcode = IONIC_CMD_LIF_SETATTR,
3354                         .index = cpu_to_le16(lif->index),
3355                         .attr = IONIC_LIF_ATTR_NAME,
3356                 },
3357         };
3358
3359         strlcpy(ctx.cmd.lif_setattr.name, lif->netdev->name,
3360                 sizeof(ctx.cmd.lif_setattr.name));
3361
3362         ionic_adminq_post_wait(lif, &ctx);
3363 }
3364
3365 static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
3366 {
3367         if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
3368                 return NULL;
3369
3370         return netdev_priv(netdev);
3371 }
3372
3373 static int ionic_lif_notify(struct notifier_block *nb,
3374                             unsigned long event, void *info)
3375 {
3376         struct net_device *ndev = netdev_notifier_info_to_dev(info);
3377         struct ionic *ionic = container_of(nb, struct ionic, nb);
3378         struct ionic_lif *lif = ionic_netdev_lif(ndev);
3379
3380         if (!lif || lif->ionic != ionic)
3381                 return NOTIFY_DONE;
3382
3383         switch (event) {
3384         case NETDEV_CHANGENAME:
3385                 ionic_lif_set_netdev_info(lif);
3386                 break;
3387         }
3388
3389         return NOTIFY_DONE;
3390 }
3391
3392 int ionic_lif_register(struct ionic_lif *lif)
3393 {
3394         int err;
3395
3396         ionic_lif_register_phc(lif);
3397
3398         INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
3399
3400         lif->ionic->nb.notifier_call = ionic_lif_notify;
3401
3402         err = register_netdevice_notifier(&lif->ionic->nb);
3403         if (err)
3404                 lif->ionic->nb.notifier_call = NULL;
3405
3406         /* only register LIF0 for now */
3407         err = register_netdev(lif->netdev);
3408         if (err) {
3409                 dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
3410                 ionic_lif_unregister_phc(lif);
3411                 return err;
3412         }
3413
3414         ionic_link_status_check_request(lif, CAN_SLEEP);
3415         lif->registered = true;
3416         ionic_lif_set_netdev_info(lif);
3417
3418         return 0;
3419 }
3420
3421 void ionic_lif_unregister(struct ionic_lif *lif)
3422 {
3423         if (lif->ionic->nb.notifier_call) {
3424                 unregister_netdevice_notifier(&lif->ionic->nb);
3425                 cancel_work_sync(&lif->ionic->nb_work);
3426                 lif->ionic->nb.notifier_call = NULL;
3427         }
3428
3429         if (lif->netdev->reg_state == NETREG_REGISTERED)
3430                 unregister_netdev(lif->netdev);
3431
3432         ionic_lif_unregister_phc(lif);
3433
3434         lif->registered = false;
3435 }
3436
3437 static void ionic_lif_queue_identify(struct ionic_lif *lif)
3438 {
3439         union ionic_q_identity __iomem *q_ident;
3440         struct ionic *ionic = lif->ionic;
3441         struct ionic_dev *idev;
3442         int qtype;
3443         int err;
3444
3445         idev = &lif->ionic->idev;
3446         q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
3447
3448         for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
3449                 struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
3450
3451                 /* filter out the ones we know about */
3452                 switch (qtype) {
3453                 case IONIC_QTYPE_ADMINQ:
3454                 case IONIC_QTYPE_NOTIFYQ:
3455                 case IONIC_QTYPE_RXQ:
3456                 case IONIC_QTYPE_TXQ:
3457                         break;
3458                 default:
3459                         continue;
3460                 }
3461
3462                 memset(qti, 0, sizeof(*qti));
3463
3464                 mutex_lock(&ionic->dev_cmd_lock);
3465                 ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
3466                                              ionic_qtype_versions[qtype]);
3467                 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3468                 if (!err) {
3469                         qti->version   = readb(&q_ident->version);
3470                         qti->supported = readb(&q_ident->supported);
3471                         qti->features  = readq(&q_ident->features);
3472                         qti->desc_sz   = readw(&q_ident->desc_sz);
3473                         qti->comp_sz   = readw(&q_ident->comp_sz);
3474                         qti->sg_desc_sz   = readw(&q_ident->sg_desc_sz);
3475                         qti->max_sg_elems = readw(&q_ident->max_sg_elems);
3476                         qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
3477                 }
3478                 mutex_unlock(&ionic->dev_cmd_lock);
3479
3480                 if (err == -EINVAL) {
3481                         dev_err(ionic->dev, "qtype %d not supported\n", qtype);
3482                         continue;
3483                 } else if (err == -EIO) {
3484                         dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
3485                         return;
3486                 } else if (err) {
3487                         dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
3488                                 qtype, err);
3489                         return;
3490                 }
3491
3492                 dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
3493                         qtype, qti->version);
3494                 dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
3495                         qtype, qti->supported);
3496                 dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
3497                         qtype, qti->features);
3498                 dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
3499                         qtype, qti->desc_sz);
3500                 dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
3501                         qtype, qti->comp_sz);
3502                 dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
3503                         qtype, qti->sg_desc_sz);
3504                 dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
3505                         qtype, qti->max_sg_elems);
3506                 dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
3507                         qtype, qti->sg_desc_stride);
3508         }
3509 }
3510
3511 int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
3512                        union ionic_lif_identity *lid)
3513 {
3514         struct ionic_dev *idev = &ionic->idev;
3515         size_t sz;
3516         int err;
3517
3518         sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
3519
3520         mutex_lock(&ionic->dev_cmd_lock);
3521         ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
3522         err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3523         memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
3524         mutex_unlock(&ionic->dev_cmd_lock);
3525         if (err)
3526                 return (err);
3527
3528         dev_dbg(ionic->dev, "capabilities 0x%llx\n",
3529                 le64_to_cpu(lid->capabilities));
3530
3531         dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
3532                 le32_to_cpu(lid->eth.max_ucast_filters));
3533         dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
3534                 le32_to_cpu(lid->eth.max_mcast_filters));
3535         dev_dbg(ionic->dev, "eth.features 0x%llx\n",
3536                 le64_to_cpu(lid->eth.config.features));
3537         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
3538                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
3539         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
3540                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
3541         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
3542                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
3543         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
3544                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
3545         dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
3546         dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
3547         dev_dbg(ionic->dev, "eth.config.mtu %d\n",
3548                 le32_to_cpu(lid->eth.config.mtu));
3549
3550         return 0;
3551 }
3552
3553 int ionic_lif_size(struct ionic *ionic)
3554 {
3555         struct ionic_identity *ident = &ionic->ident;
3556         unsigned int nintrs, dev_nintrs;
3557         union ionic_lif_config *lc;
3558         unsigned int ntxqs_per_lif;
3559         unsigned int nrxqs_per_lif;
3560         unsigned int neqs_per_lif;
3561         unsigned int nnqs_per_lif;
3562         unsigned int nxqs, neqs;
3563         unsigned int min_intrs;
3564         int err;
3565
3566         /* retrieve basic values from FW */
3567         lc = &ident->lif.eth.config;
3568         dev_nintrs = le32_to_cpu(ident->dev.nintrs);
3569         neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
3570         nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
3571         ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
3572         nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
3573
3574         /* limit values to play nice with kdump */
3575         if (is_kdump_kernel()) {
3576                 dev_nintrs = 2;
3577                 neqs_per_lif = 0;
3578                 nnqs_per_lif = 0;
3579                 ntxqs_per_lif = 1;
3580                 nrxqs_per_lif = 1;
3581         }
3582
3583         /* reserve last queue id for hardware timestamping */
3584         if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
3585                 if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
3586                         lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
3587                 } else {
3588                         ntxqs_per_lif -= 1;
3589                         nrxqs_per_lif -= 1;
3590                 }
3591         }
3592
3593         nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
3594         nxqs = min(nxqs, num_online_cpus());
3595         neqs = min(neqs_per_lif, num_online_cpus());
3596
3597 try_again:
3598         /* interrupt usage:
3599          *    1 for master lif adminq/notifyq
3600          *    1 for each CPU for master lif TxRx queue pairs
3601          *    whatever's left is for RDMA queues
3602          */
3603         nintrs = 1 + nxqs + neqs;
3604         min_intrs = 2;  /* adminq + 1 TxRx queue pair */
3605
3606         if (nintrs > dev_nintrs)
3607                 goto try_fewer;
3608
3609         err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
3610         if (err < 0 && err != -ENOSPC) {
3611                 dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
3612                 return err;
3613         }
3614         if (err == -ENOSPC)
3615                 goto try_fewer;
3616
3617         if (err != nintrs) {
3618                 ionic_bus_free_irq_vectors(ionic);
3619                 goto try_fewer;
3620         }
3621
3622         ionic->nnqs_per_lif = nnqs_per_lif;
3623         ionic->neqs_per_lif = neqs;
3624         ionic->ntxqs_per_lif = nxqs;
3625         ionic->nrxqs_per_lif = nxqs;
3626         ionic->nintrs = nintrs;
3627
3628         ionic_debugfs_add_sizes(ionic);
3629
3630         return 0;
3631
3632 try_fewer:
3633         if (nnqs_per_lif > 1) {
3634                 nnqs_per_lif >>= 1;
3635                 goto try_again;
3636         }
3637         if (neqs > 1) {
3638                 neqs >>= 1;
3639                 goto try_again;
3640         }
3641         if (nxqs > 1) {
3642                 nxqs >>= 1;
3643                 goto try_again;
3644         }
3645         dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
3646         return -ENOSPC;
3647 }