1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
4 #include <linux/ethtool.h>
5 #include <linux/printk.h>
6 #include <linux/dynamic_debug.h>
7 #include <linux/netdevice.h>
8 #include <linux/etherdevice.h>
9 #include <linux/if_vlan.h>
10 #include <linux/rtnetlink.h>
11 #include <linux/interrupt.h>
12 #include <linux/pci.h>
13 #include <linux/cpumask.h>
16 #include "ionic_bus.h"
17 #include "ionic_lif.h"
18 #include "ionic_txrx.h"
19 #include "ionic_ethtool.h"
20 #include "ionic_debugfs.h"
22 /* queuetype support level */
23 static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
24 [IONIC_QTYPE_ADMINQ] = 0, /* 0 = Base version with CQ support */
25 [IONIC_QTYPE_NOTIFYQ] = 0, /* 0 = Base version */
26 [IONIC_QTYPE_RXQ] = 0, /* 0 = Base version with CQ+SG support */
27 [IONIC_QTYPE_TXQ] = 1, /* 0 = Base version with CQ+SG support
28 * 1 = ... with Tx SG version 1
32 static void ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode);
33 static int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr);
34 static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr);
35 static void ionic_link_status_check(struct ionic_lif *lif);
36 static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
37 static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
38 static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
40 static void ionic_txrx_deinit(struct ionic_lif *lif);
41 static int ionic_txrx_init(struct ionic_lif *lif);
42 static int ionic_start_queues(struct ionic_lif *lif);
43 static void ionic_stop_queues(struct ionic_lif *lif);
44 static void ionic_lif_queue_identify(struct ionic_lif *lif);
46 static void ionic_dim_work(struct work_struct *work)
48 struct dim *dim = container_of(work, struct dim, work);
49 struct dim_cq_moder cur_moder;
50 struct ionic_qcq *qcq;
53 cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
54 qcq = container_of(dim, struct ionic_qcq, dim);
55 new_coal = ionic_coal_usec_to_hw(qcq->q.lif->ionic, cur_moder.usec);
56 qcq->intr.dim_coal_hw = new_coal ? new_coal : 1;
57 dim->state = DIM_START_MEASURE;
60 static void ionic_lif_deferred_work(struct work_struct *work)
62 struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
63 struct ionic_deferred *def = &lif->deferred;
64 struct ionic_deferred_work *w = NULL;
67 spin_lock_bh(&def->lock);
68 if (!list_empty(&def->list)) {
69 w = list_first_entry(&def->list,
70 struct ionic_deferred_work, list);
73 spin_unlock_bh(&def->lock);
79 case IONIC_DW_TYPE_RX_MODE:
80 ionic_lif_rx_mode(lif, w->rx_mode);
82 case IONIC_DW_TYPE_RX_ADDR_ADD:
83 ionic_lif_addr_add(lif, w->addr);
85 case IONIC_DW_TYPE_RX_ADDR_DEL:
86 ionic_lif_addr_del(lif, w->addr);
88 case IONIC_DW_TYPE_LINK_STATUS:
89 ionic_link_status_check(lif);
91 case IONIC_DW_TYPE_LIF_RESET:
93 ionic_lif_handle_fw_up(lif);
95 ionic_lif_handle_fw_down(lif);
105 void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
106 struct ionic_deferred_work *work)
108 spin_lock_bh(&def->lock);
109 list_add_tail(&work->list, &def->list);
110 spin_unlock_bh(&def->lock);
111 schedule_work(&def->work);
114 static void ionic_link_status_check(struct ionic_lif *lif)
116 struct net_device *netdev = lif->netdev;
120 if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
123 /* Don't put carrier back up if we're in a broken state */
124 if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
125 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
129 link_status = le16_to_cpu(lif->info->status.link_status);
130 link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
135 if (netdev->flags & IFF_UP && netif_running(netdev)) {
136 mutex_lock(&lif->queue_lock);
137 err = ionic_start_queues(lif);
138 if (err && err != -EBUSY) {
139 netdev_err(lif->netdev,
140 "Failed to start queues: %d\n", err);
141 set_bit(IONIC_LIF_F_BROKEN, lif->state);
142 netif_carrier_off(lif->netdev);
144 mutex_unlock(&lif->queue_lock);
147 if (!err && !netif_carrier_ok(netdev)) {
148 ionic_port_identify(lif->ionic);
149 netdev_info(netdev, "Link up - %d Gbps\n",
150 le32_to_cpu(lif->info->status.link_speed) / 1000);
151 netif_carrier_on(netdev);
154 if (netif_carrier_ok(netdev)) {
155 netdev_info(netdev, "Link down\n");
156 netif_carrier_off(netdev);
159 if (netdev->flags & IFF_UP && netif_running(netdev)) {
160 mutex_lock(&lif->queue_lock);
161 ionic_stop_queues(lif);
162 mutex_unlock(&lif->queue_lock);
166 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
169 void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
171 struct ionic_deferred_work *work;
173 /* we only need one request outstanding at a time */
174 if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
178 work = kzalloc(sizeof(*work), GFP_ATOMIC);
180 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
184 work->type = IONIC_DW_TYPE_LINK_STATUS;
185 ionic_lif_deferred_enqueue(&lif->deferred, work);
187 ionic_link_status_check(lif);
191 static irqreturn_t ionic_isr(int irq, void *data)
193 struct napi_struct *napi = data;
195 napi_schedule_irqoff(napi);
200 static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
202 struct ionic_intr_info *intr = &qcq->intr;
203 struct device *dev = lif->ionic->dev;
204 struct ionic_queue *q = &qcq->q;
208 name = lif->netdev->name;
210 name = dev_name(dev);
212 snprintf(intr->name, sizeof(intr->name),
213 "%s-%s-%s", IONIC_DRV_NAME, name, q->name);
215 return devm_request_irq(dev, intr->vector, ionic_isr,
216 0, intr->name, &qcq->napi);
219 static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
221 struct ionic *ionic = lif->ionic;
224 index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
225 if (index == ionic->nintrs) {
226 netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
227 __func__, index, ionic->nintrs);
231 set_bit(index, ionic->intrs);
232 ionic_intr_init(&ionic->idev, intr, index);
237 static void ionic_intr_free(struct ionic *ionic, int index)
239 if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
240 clear_bit(index, ionic->intrs);
243 static int ionic_qcq_enable(struct ionic_qcq *qcq)
245 struct ionic_queue *q = &qcq->q;
246 struct ionic_lif *lif = q->lif;
247 struct ionic_dev *idev;
250 struct ionic_admin_ctx ctx = {
251 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
253 .opcode = IONIC_CMD_Q_CONTROL,
254 .lif_index = cpu_to_le16(lif->index),
256 .index = cpu_to_le32(q->index),
257 .oper = IONIC_Q_ENABLE,
261 idev = &lif->ionic->idev;
262 dev = lif->ionic->dev;
264 dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
265 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
267 if (qcq->flags & IONIC_QCQ_F_INTR) {
268 irq_set_affinity_hint(qcq->intr.vector,
269 &qcq->intr.affinity_mask);
270 napi_enable(&qcq->napi);
271 ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
272 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
273 IONIC_INTR_MASK_CLEAR);
276 return ionic_adminq_post_wait(lif, &ctx);
279 static int ionic_qcq_disable(struct ionic_qcq *qcq, bool send_to_hw)
281 struct ionic_queue *q;
282 struct ionic_lif *lif;
285 struct ionic_admin_ctx ctx = {
286 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
288 .opcode = IONIC_CMD_Q_CONTROL,
289 .oper = IONIC_Q_DISABLE,
299 if (qcq->flags & IONIC_QCQ_F_INTR) {
300 struct ionic_dev *idev = &lif->ionic->idev;
302 cancel_work_sync(&qcq->dim.work);
303 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
304 IONIC_INTR_MASK_SET);
305 synchronize_irq(qcq->intr.vector);
306 irq_set_affinity_hint(qcq->intr.vector, NULL);
307 napi_disable(&qcq->napi);
311 ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
312 ctx.cmd.q_control.type = q->type;
313 ctx.cmd.q_control.index = cpu_to_le32(q->index);
314 dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
315 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
317 err = ionic_adminq_post_wait(lif, &ctx);
323 static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
325 struct ionic_dev *idev = &lif->ionic->idev;
330 if (!(qcq->flags & IONIC_QCQ_F_INITED))
333 if (qcq->flags & IONIC_QCQ_F_INTR) {
334 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
335 IONIC_INTR_MASK_SET);
336 netif_napi_del(&qcq->napi);
339 qcq->flags &= ~IONIC_QCQ_F_INITED;
342 static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
344 if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
347 irq_set_affinity_hint(qcq->intr.vector, NULL);
348 devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
349 qcq->intr.vector = 0;
350 ionic_intr_free(lif->ionic, qcq->intr.index);
351 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
354 static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
356 struct device *dev = lif->ionic->dev;
361 ionic_debugfs_del_qcq(qcq);
364 dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
370 dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
376 dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
381 ionic_qcq_intr_free(lif, qcq);
384 devm_kfree(dev, qcq->cq.info);
388 devm_kfree(dev, qcq->q.info);
393 static void ionic_qcqs_free(struct ionic_lif *lif)
395 struct device *dev = lif->ionic->dev;
396 struct ionic_qcq *adminqcq;
397 unsigned long irqflags;
399 if (lif->notifyqcq) {
400 ionic_qcq_free(lif, lif->notifyqcq);
401 devm_kfree(dev, lif->notifyqcq);
402 lif->notifyqcq = NULL;
406 spin_lock_irqsave(&lif->adminq_lock, irqflags);
407 adminqcq = READ_ONCE(lif->adminqcq);
408 lif->adminqcq = NULL;
409 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
411 ionic_qcq_free(lif, adminqcq);
412 devm_kfree(dev, adminqcq);
417 devm_kfree(dev, lif->rxqstats);
418 lif->rxqstats = NULL;
419 devm_kfree(dev, lif->rxqcqs);
424 devm_kfree(dev, lif->txqstats);
425 lif->txqstats = NULL;
426 devm_kfree(dev, lif->txqcqs);
431 static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
432 struct ionic_qcq *n_qcq)
434 if (WARN_ON(n_qcq->flags & IONIC_QCQ_F_INTR)) {
435 ionic_intr_free(n_qcq->cq.lif->ionic, n_qcq->intr.index);
436 n_qcq->flags &= ~IONIC_QCQ_F_INTR;
439 n_qcq->intr.vector = src_qcq->intr.vector;
440 n_qcq->intr.index = src_qcq->intr.index;
443 static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
447 if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
448 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
452 err = ionic_intr_alloc(lif, &qcq->intr);
454 netdev_warn(lif->netdev, "no intr for %s: %d\n",
459 err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
461 netdev_warn(lif->netdev, "no vector for %s: %d\n",
463 goto err_out_free_intr;
465 qcq->intr.vector = err;
466 ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
467 IONIC_INTR_MASK_SET);
469 err = ionic_request_irq(lif, qcq);
471 netdev_warn(lif->netdev, "irq request failed %d\n", err);
472 goto err_out_free_intr;
475 /* try to get the irq on the local numa node first */
476 qcq->intr.cpu = cpumask_local_spread(qcq->intr.index,
477 dev_to_node(lif->ionic->dev));
478 if (qcq->intr.cpu != -1)
479 cpumask_set_cpu(qcq->intr.cpu, &qcq->intr.affinity_mask);
481 netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
485 ionic_intr_free(lif->ionic, qcq->intr.index);
490 static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
492 const char *name, unsigned int flags,
493 unsigned int num_descs, unsigned int desc_size,
494 unsigned int cq_desc_size,
495 unsigned int sg_desc_size,
496 unsigned int pid, struct ionic_qcq **qcq)
498 struct ionic_dev *idev = &lif->ionic->idev;
499 struct device *dev = lif->ionic->dev;
500 void *q_base, *cq_base, *sg_base;
501 dma_addr_t cq_base_pa = 0;
502 dma_addr_t sg_base_pa = 0;
503 dma_addr_t q_base_pa = 0;
504 struct ionic_qcq *new;
509 new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
511 netdev_err(lif->netdev, "Cannot allocate queue structure\n");
519 new->q.info = devm_kcalloc(dev, num_descs, sizeof(*new->q.info),
522 netdev_err(lif->netdev, "Cannot allocate queue info\n");
524 goto err_out_free_qcq;
528 new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
530 err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
531 desc_size, sg_desc_size, pid);
533 netdev_err(lif->netdev, "Cannot initialize queue\n");
534 goto err_out_free_q_info;
537 err = ionic_alloc_qcq_interrupt(lif, new);
541 new->cq.info = devm_kcalloc(dev, num_descs, sizeof(*new->cq.info),
544 netdev_err(lif->netdev, "Cannot allocate completion queue info\n");
546 goto err_out_free_irq;
549 err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
551 netdev_err(lif->netdev, "Cannot initialize completion queue\n");
552 goto err_out_free_cq_info;
555 if (flags & IONIC_QCQ_F_NOTIFYQ) {
558 /* q & cq need to be contiguous in case of notifyq */
559 q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
560 cq_size = ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
562 new->q_size = PAGE_SIZE + q_size + cq_size;
563 new->q_base = dma_alloc_coherent(dev, new->q_size,
564 &new->q_base_pa, GFP_KERNEL);
566 netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
568 goto err_out_free_cq_info;
570 q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
571 q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
572 ionic_q_map(&new->q, q_base, q_base_pa);
574 cq_base = PTR_ALIGN(q_base + q_size, PAGE_SIZE);
575 cq_base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
576 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
577 ionic_cq_bind(&new->cq, &new->q);
579 new->q_size = PAGE_SIZE + (num_descs * desc_size);
580 new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
583 netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
585 goto err_out_free_cq_info;
587 q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
588 q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
589 ionic_q_map(&new->q, q_base, q_base_pa);
591 new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
592 new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
595 netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
599 cq_base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
600 cq_base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
601 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
602 ionic_cq_bind(&new->cq, &new->q);
605 if (flags & IONIC_QCQ_F_SG) {
606 new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
607 new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
610 netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
612 goto err_out_free_cq;
614 sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
615 sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
616 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
619 INIT_WORK(&new->dim.work, ionic_dim_work);
620 new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
627 dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
629 dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
630 err_out_free_cq_info:
631 devm_kfree(dev, new->cq.info);
633 if (flags & IONIC_QCQ_F_INTR) {
634 devm_free_irq(dev, new->intr.vector, &new->napi);
635 ionic_intr_free(lif->ionic, new->intr.index);
638 devm_kfree(dev, new->q.info);
640 devm_kfree(dev, new);
642 dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
646 static int ionic_qcqs_alloc(struct ionic_lif *lif)
648 struct device *dev = lif->ionic->dev;
652 flags = IONIC_QCQ_F_INTR;
653 err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
655 sizeof(struct ionic_admin_cmd),
656 sizeof(struct ionic_admin_comp),
657 0, lif->kern_pid, &lif->adminqcq);
660 ionic_debugfs_add_qcq(lif, lif->adminqcq);
662 if (lif->ionic->nnqs_per_lif) {
663 flags = IONIC_QCQ_F_NOTIFYQ;
664 err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
665 flags, IONIC_NOTIFYQ_LENGTH,
666 sizeof(struct ionic_notifyq_cmd),
667 sizeof(union ionic_notifyq_comp),
668 0, lif->kern_pid, &lif->notifyqcq);
671 ionic_debugfs_add_qcq(lif, lif->notifyqcq);
673 /* Let the notifyq ride on the adminq interrupt */
674 ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
678 lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
679 sizeof(*lif->txqcqs), GFP_KERNEL);
682 lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
683 sizeof(*lif->rxqcqs), GFP_KERNEL);
687 lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
688 sizeof(*lif->txqstats), GFP_KERNEL);
691 lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
692 sizeof(*lif->rxqstats), GFP_KERNEL);
699 ionic_qcqs_free(lif);
703 static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
707 qcq->cq.tail_idx = 0;
708 qcq->cq.done_color = 1;
709 memset(qcq->q_base, 0, qcq->q_size);
710 memset(qcq->cq_base, 0, qcq->cq_size);
711 memset(qcq->sg_base, 0, qcq->sg_size);
714 static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
716 struct device *dev = lif->ionic->dev;
717 struct ionic_queue *q = &qcq->q;
718 struct ionic_cq *cq = &qcq->cq;
719 struct ionic_admin_ctx ctx = {
720 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
722 .opcode = IONIC_CMD_Q_INIT,
723 .lif_index = cpu_to_le16(lif->index),
725 .ver = lif->qtype_info[q->type].version,
726 .index = cpu_to_le32(q->index),
727 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
729 .pid = cpu_to_le16(q->pid),
730 .ring_size = ilog2(q->num_descs),
731 .ring_base = cpu_to_le64(q->base_pa),
732 .cq_ring_base = cpu_to_le64(cq->base_pa),
733 .sg_ring_base = cpu_to_le64(q->sg_base_pa),
734 .features = cpu_to_le64(q->features),
737 unsigned int intr_index;
740 intr_index = qcq->intr.index;
742 ctx.cmd.q_init.intr_index = cpu_to_le16(intr_index);
744 dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
745 dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
746 dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
747 dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
748 dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
749 dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
750 dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
752 ionic_qcq_sanitize(qcq);
754 err = ionic_adminq_post_wait(lif, &ctx);
758 q->hw_type = ctx.comp.q_init.hw_type;
759 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
760 q->dbval = IONIC_DBELL_QID(q->hw_index);
762 dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
763 dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
765 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
766 netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi,
769 qcq->flags |= IONIC_QCQ_F_INITED;
774 static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
776 struct device *dev = lif->ionic->dev;
777 struct ionic_queue *q = &qcq->q;
778 struct ionic_cq *cq = &qcq->cq;
779 struct ionic_admin_ctx ctx = {
780 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
782 .opcode = IONIC_CMD_Q_INIT,
783 .lif_index = cpu_to_le16(lif->index),
785 .ver = lif->qtype_info[q->type].version,
786 .index = cpu_to_le32(q->index),
787 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
789 .intr_index = cpu_to_le16(cq->bound_intr->index),
790 .pid = cpu_to_le16(q->pid),
791 .ring_size = ilog2(q->num_descs),
792 .ring_base = cpu_to_le64(q->base_pa),
793 .cq_ring_base = cpu_to_le64(cq->base_pa),
794 .sg_ring_base = cpu_to_le64(q->sg_base_pa),
795 .features = cpu_to_le64(q->features),
800 dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
801 dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
802 dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
803 dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
804 dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
805 dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
806 dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
808 ionic_qcq_sanitize(qcq);
810 err = ionic_adminq_post_wait(lif, &ctx);
814 q->hw_type = ctx.comp.q_init.hw_type;
815 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
816 q->dbval = IONIC_DBELL_QID(q->hw_index);
818 dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
819 dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
821 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
822 netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi,
825 netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi,
828 qcq->flags |= IONIC_QCQ_F_INITED;
833 int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
835 unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
836 unsigned int txq_i, flags;
837 struct ionic_qcq *txq;
841 mutex_lock(&lif->queue_lock);
843 if (lif->hwstamp_txq)
846 features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
848 num_desc = IONIC_MIN_TXRX_DESC;
849 desc_sz = sizeof(struct ionic_txq_desc);
850 comp_sz = 2 * sizeof(struct ionic_txq_comp);
852 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
853 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
854 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
856 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
858 txq_i = lif->ionic->ntxqs_per_lif;
859 flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
861 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
862 num_desc, desc_sz, comp_sz, sg_desc_sz,
863 lif->kern_pid, &txq);
867 txq->q.features = features;
869 ionic_link_qcq_interrupts(lif->adminqcq, txq);
870 ionic_debugfs_add_qcq(lif, txq);
872 lif->hwstamp_txq = txq;
874 if (netif_running(lif->netdev)) {
875 err = ionic_lif_txq_init(lif, txq);
879 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
880 err = ionic_qcq_enable(txq);
887 mutex_unlock(&lif->queue_lock);
892 ionic_lif_qcq_deinit(lif, txq);
894 lif->hwstamp_txq = NULL;
895 ionic_debugfs_del_qcq(txq);
896 ionic_qcq_free(lif, txq);
897 devm_kfree(lif->ionic->dev, txq);
899 mutex_unlock(&lif->queue_lock);
903 int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
905 unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
906 unsigned int rxq_i, flags;
907 struct ionic_qcq *rxq;
911 mutex_lock(&lif->queue_lock);
913 if (lif->hwstamp_rxq)
916 features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
918 num_desc = IONIC_MIN_TXRX_DESC;
919 desc_sz = sizeof(struct ionic_rxq_desc);
920 comp_sz = 2 * sizeof(struct ionic_rxq_comp);
921 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
923 rxq_i = lif->ionic->nrxqs_per_lif;
924 flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
926 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
927 num_desc, desc_sz, comp_sz, sg_desc_sz,
928 lif->kern_pid, &rxq);
932 rxq->q.features = features;
934 ionic_link_qcq_interrupts(lif->adminqcq, rxq);
935 ionic_debugfs_add_qcq(lif, rxq);
937 lif->hwstamp_rxq = rxq;
939 if (netif_running(lif->netdev)) {
940 err = ionic_lif_rxq_init(lif, rxq);
944 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
945 ionic_rx_fill(&rxq->q);
946 err = ionic_qcq_enable(rxq);
953 mutex_unlock(&lif->queue_lock);
958 ionic_lif_qcq_deinit(lif, rxq);
960 lif->hwstamp_rxq = NULL;
961 ionic_debugfs_del_qcq(rxq);
962 ionic_qcq_free(lif, rxq);
963 devm_kfree(lif->ionic->dev, rxq);
965 mutex_unlock(&lif->queue_lock);
969 int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
971 struct ionic_queue_params qparam;
973 ionic_init_queue_params(lif, &qparam);
976 qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
978 qparam.rxq_features = 0;
980 /* if we're not running, just set the values and return */
981 if (!netif_running(lif->netdev)) {
982 lif->rxq_features = qparam.rxq_features;
986 return ionic_reconfigure_queues(lif, &qparam);
989 int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
991 struct ionic_admin_ctx ctx = {
992 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
994 .opcode = IONIC_CMD_LIF_SETATTR,
995 .index = cpu_to_le16(lif->index),
996 .attr = IONIC_LIF_ATTR_TXSTAMP,
997 .txstamp_mode = cpu_to_le16(txstamp_mode),
1001 return ionic_adminq_post_wait(lif, &ctx);
1004 static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
1006 struct ionic_admin_ctx ctx = {
1007 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1008 .cmd.rx_filter_del = {
1009 .opcode = IONIC_CMD_RX_FILTER_DEL,
1010 .lif_index = cpu_to_le16(lif->index),
1013 struct ionic_rx_filter *f;
1017 spin_lock_bh(&lif->rx_filters.lock);
1019 f = ionic_rx_filter_rxsteer(lif);
1021 spin_unlock_bh(&lif->rx_filters.lock);
1025 filter_id = f->filter_id;
1026 ionic_rx_filter_free(lif, f);
1028 spin_unlock_bh(&lif->rx_filters.lock);
1030 netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
1032 ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
1034 err = ionic_adminq_post_wait(lif, &ctx);
1035 if (err && err != -EEXIST)
1036 netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
1039 static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1041 struct ionic_admin_ctx ctx = {
1042 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1043 .cmd.rx_filter_add = {
1044 .opcode = IONIC_CMD_RX_FILTER_ADD,
1045 .lif_index = cpu_to_le16(lif->index),
1046 .match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
1047 .pkt_class = cpu_to_le64(pkt_class),
1054 if (!lif->hwstamp_rxq)
1057 qtype = lif->hwstamp_rxq->q.type;
1058 ctx.cmd.rx_filter_add.qtype = qtype;
1060 qid = lif->hwstamp_rxq->q.index;
1061 ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
1063 netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
1064 err = ionic_adminq_post_wait(lif, &ctx);
1065 if (err && err != -EEXIST)
1068 return ionic_rx_filter_save(lif, 0, qid, 0, &ctx);
1071 int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1073 ionic_lif_del_hwstamp_rxfilt(lif);
1078 return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
1081 static bool ionic_notifyq_service(struct ionic_cq *cq,
1082 struct ionic_cq_info *cq_info)
1084 union ionic_notifyq_comp *comp = cq_info->cq_desc;
1085 struct ionic_deferred_work *work;
1086 struct net_device *netdev;
1087 struct ionic_queue *q;
1088 struct ionic_lif *lif;
1092 lif = q->info[0].cb_arg;
1093 netdev = lif->netdev;
1094 eid = le64_to_cpu(comp->event.eid);
1096 /* Have we run out of new completions to process? */
1097 if ((s64)(eid - lif->last_eid) <= 0)
1100 lif->last_eid = eid;
1102 dev_dbg(lif->ionic->dev, "notifyq event:\n");
1103 dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET, 16, 1,
1104 comp, sizeof(*comp), true);
1106 switch (le16_to_cpu(comp->event.ecode)) {
1107 case IONIC_EVENT_LINK_CHANGE:
1108 ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1110 case IONIC_EVENT_RESET:
1111 work = kzalloc(sizeof(*work), GFP_ATOMIC);
1113 netdev_err(lif->netdev, "Reset event dropped\n");
1115 work->type = IONIC_DW_TYPE_LIF_RESET;
1116 ionic_lif_deferred_enqueue(&lif->deferred, work);
1120 netdev_warn(netdev, "Notifyq event ecode=%d eid=%lld\n",
1121 comp->event.ecode, eid);
1128 static bool ionic_adminq_service(struct ionic_cq *cq,
1129 struct ionic_cq_info *cq_info)
1131 struct ionic_admin_comp *comp = cq_info->cq_desc;
1133 if (!color_match(comp->color, cq->done_color))
1136 ionic_q_service(cq->bound_q, cq_info, le16_to_cpu(comp->comp_index));
1141 static int ionic_adminq_napi(struct napi_struct *napi, int budget)
1143 struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
1144 struct ionic_lif *lif = napi_to_cq(napi)->lif;
1145 struct ionic_dev *idev = &lif->ionic->idev;
1146 unsigned long irqflags;
1147 unsigned int flags = 0;
1155 if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
1156 n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
1157 ionic_notifyq_service, NULL, NULL);
1159 spin_lock_irqsave(&lif->adminq_lock, irqflags);
1160 if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
1161 a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
1162 ionic_adminq_service, NULL, NULL);
1163 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
1165 if (lif->hwstamp_rxq)
1166 rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
1167 ionic_rx_service, NULL, NULL);
1169 if (lif->hwstamp_txq)
1170 tx_work = ionic_cq_service(&lif->hwstamp_txq->cq, budget,
1171 ionic_tx_service, NULL, NULL);
1173 work_done = max(max(n_work, a_work), max(rx_work, tx_work));
1174 if (work_done < budget && napi_complete_done(napi, work_done)) {
1175 flags |= IONIC_INTR_CRED_UNMASK;
1176 intr->rearm_count++;
1179 if (work_done || flags) {
1180 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1181 credits = n_work + a_work + rx_work + tx_work;
1182 ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
1188 void ionic_get_stats64(struct net_device *netdev,
1189 struct rtnl_link_stats64 *ns)
1191 struct ionic_lif *lif = netdev_priv(netdev);
1192 struct ionic_lif_stats *ls;
1194 memset(ns, 0, sizeof(*ns));
1195 ls = &lif->info->stats;
1197 ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
1198 le64_to_cpu(ls->rx_mcast_packets) +
1199 le64_to_cpu(ls->rx_bcast_packets);
1201 ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
1202 le64_to_cpu(ls->tx_mcast_packets) +
1203 le64_to_cpu(ls->tx_bcast_packets);
1205 ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
1206 le64_to_cpu(ls->rx_mcast_bytes) +
1207 le64_to_cpu(ls->rx_bcast_bytes);
1209 ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
1210 le64_to_cpu(ls->tx_mcast_bytes) +
1211 le64_to_cpu(ls->tx_bcast_bytes);
1213 ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
1214 le64_to_cpu(ls->rx_mcast_drop_packets) +
1215 le64_to_cpu(ls->rx_bcast_drop_packets);
1217 ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
1218 le64_to_cpu(ls->tx_mcast_drop_packets) +
1219 le64_to_cpu(ls->tx_bcast_drop_packets);
1221 ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
1223 ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
1225 ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
1226 le64_to_cpu(ls->rx_queue_disabled) +
1227 le64_to_cpu(ls->rx_desc_fetch_error) +
1228 le64_to_cpu(ls->rx_desc_data_error);
1230 ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
1231 le64_to_cpu(ls->tx_queue_disabled) +
1232 le64_to_cpu(ls->tx_desc_fetch_error) +
1233 le64_to_cpu(ls->tx_desc_data_error);
1235 ns->rx_errors = ns->rx_over_errors +
1236 ns->rx_missed_errors;
1238 ns->tx_errors = ns->tx_aborted_errors;
1241 static int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr)
1243 struct ionic_admin_ctx ctx = {
1244 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1245 .cmd.rx_filter_add = {
1246 .opcode = IONIC_CMD_RX_FILTER_ADD,
1247 .lif_index = cpu_to_le16(lif->index),
1248 .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_MAC),
1251 struct ionic_rx_filter *f;
1254 /* don't bother if we already have it */
1255 spin_lock_bh(&lif->rx_filters.lock);
1256 f = ionic_rx_filter_by_addr(lif, addr);
1257 spin_unlock_bh(&lif->rx_filters.lock);
1261 netdev_dbg(lif->netdev, "rx_filter add ADDR %pM\n", addr);
1263 memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, ETH_ALEN);
1264 err = ionic_adminq_post_wait(lif, &ctx);
1265 if (err && err != -EEXIST)
1268 return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx);
1271 static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr)
1273 struct ionic_admin_ctx ctx = {
1274 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1275 .cmd.rx_filter_del = {
1276 .opcode = IONIC_CMD_RX_FILTER_DEL,
1277 .lif_index = cpu_to_le16(lif->index),
1280 struct ionic_rx_filter *f;
1283 spin_lock_bh(&lif->rx_filters.lock);
1284 f = ionic_rx_filter_by_addr(lif, addr);
1286 spin_unlock_bh(&lif->rx_filters.lock);
1290 netdev_dbg(lif->netdev, "rx_filter del ADDR %pM (id %d)\n",
1291 addr, f->filter_id);
1293 ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1294 ionic_rx_filter_free(lif, f);
1295 spin_unlock_bh(&lif->rx_filters.lock);
1297 err = ionic_adminq_post_wait(lif, &ctx);
1298 if (err && err != -EEXIST)
1304 static int ionic_lif_addr(struct ionic_lif *lif, const u8 *addr, bool add,
1307 struct ionic_deferred_work *work;
1308 unsigned int nmfilters;
1309 unsigned int nufilters;
1312 /* Do we have space for this filter? We test the counters
1313 * here before checking the need for deferral so that we
1314 * can return an overflow error to the stack.
1316 nmfilters = le32_to_cpu(lif->identity->eth.max_mcast_filters);
1317 nufilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1319 if ((is_multicast_ether_addr(addr) && lif->nmcast < nmfilters))
1321 else if (!is_multicast_ether_addr(addr) &&
1322 lif->nucast < nufilters)
1327 if (is_multicast_ether_addr(addr) && lif->nmcast)
1329 else if (!is_multicast_ether_addr(addr) && lif->nucast)
1334 work = kzalloc(sizeof(*work), GFP_ATOMIC);
1337 work->type = add ? IONIC_DW_TYPE_RX_ADDR_ADD :
1338 IONIC_DW_TYPE_RX_ADDR_DEL;
1339 memcpy(work->addr, addr, ETH_ALEN);
1340 netdev_dbg(lif->netdev, "deferred: rx_filter %s %pM\n",
1341 add ? "add" : "del", addr);
1342 ionic_lif_deferred_enqueue(&lif->deferred, work);
1344 netdev_dbg(lif->netdev, "rx_filter %s %pM\n",
1345 add ? "add" : "del", addr);
1347 return ionic_lif_addr_add(lif, addr);
1349 return ionic_lif_addr_del(lif, addr);
1355 static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
1357 return ionic_lif_addr(netdev_priv(netdev), addr, ADD_ADDR, CAN_SLEEP);
1360 static int ionic_ndo_addr_add(struct net_device *netdev, const u8 *addr)
1362 return ionic_lif_addr(netdev_priv(netdev), addr, ADD_ADDR, CAN_NOT_SLEEP);
1365 static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
1367 return ionic_lif_addr(netdev_priv(netdev), addr, DEL_ADDR, CAN_SLEEP);
1370 static int ionic_ndo_addr_del(struct net_device *netdev, const u8 *addr)
1372 return ionic_lif_addr(netdev_priv(netdev), addr, DEL_ADDR, CAN_NOT_SLEEP);
1375 static void ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode)
1377 struct ionic_admin_ctx ctx = {
1378 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1379 .cmd.rx_mode_set = {
1380 .opcode = IONIC_CMD_RX_MODE_SET,
1381 .lif_index = cpu_to_le16(lif->index),
1382 .rx_mode = cpu_to_le16(rx_mode),
1388 #define REMAIN(__x) (sizeof(buf) - (__x))
1390 i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
1391 lif->rx_mode, rx_mode);
1392 if (rx_mode & IONIC_RX_MODE_F_UNICAST)
1393 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
1394 if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
1395 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
1396 if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
1397 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
1398 if (rx_mode & IONIC_RX_MODE_F_PROMISC)
1399 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
1400 if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
1401 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
1402 netdev_dbg(lif->netdev, "lif%d %s\n", lif->index, buf);
1404 err = ionic_adminq_post_wait(lif, &ctx);
1406 netdev_warn(lif->netdev, "set rx_mode 0x%04x failed: %d\n",
1409 lif->rx_mode = rx_mode;
1412 static void ionic_set_rx_mode(struct net_device *netdev, bool can_sleep)
1414 struct ionic_lif *lif = netdev_priv(netdev);
1415 struct ionic_deferred_work *work;
1416 unsigned int nfilters;
1417 unsigned int rx_mode;
1419 rx_mode = IONIC_RX_MODE_F_UNICAST;
1420 rx_mode |= (netdev->flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
1421 rx_mode |= (netdev->flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
1422 rx_mode |= (netdev->flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
1423 rx_mode |= (netdev->flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
1425 /* sync unicast addresses
1426 * next check to see if we're in an overflow state
1427 * if so, we track that we overflowed and enable NIC PROMISC
1428 * else if the overflow is set and not needed
1429 * we remove our overflow flag and check the netdev flags
1430 * to see if we can disable NIC PROMISC
1433 __dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
1435 __dev_uc_sync(netdev, ionic_ndo_addr_add, ionic_ndo_addr_del);
1436 nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1437 if (netdev_uc_count(netdev) + 1 > nfilters) {
1438 rx_mode |= IONIC_RX_MODE_F_PROMISC;
1439 lif->uc_overflow = true;
1440 } else if (lif->uc_overflow) {
1441 lif->uc_overflow = false;
1442 if (!(netdev->flags & IFF_PROMISC))
1443 rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
1446 /* same for multicast */
1448 __dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
1450 __dev_mc_sync(netdev, ionic_ndo_addr_add, ionic_ndo_addr_del);
1451 nfilters = le32_to_cpu(lif->identity->eth.max_mcast_filters);
1452 if (netdev_mc_count(netdev) > nfilters) {
1453 rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
1454 lif->mc_overflow = true;
1455 } else if (lif->mc_overflow) {
1456 lif->mc_overflow = false;
1457 if (!(netdev->flags & IFF_ALLMULTI))
1458 rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
1461 if (lif->rx_mode != rx_mode) {
1463 work = kzalloc(sizeof(*work), GFP_ATOMIC);
1465 netdev_err(lif->netdev, "rxmode change dropped\n");
1468 work->type = IONIC_DW_TYPE_RX_MODE;
1469 work->rx_mode = rx_mode;
1470 netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1471 ionic_lif_deferred_enqueue(&lif->deferred, work);
1473 ionic_lif_rx_mode(lif, rx_mode);
1478 static void ionic_ndo_set_rx_mode(struct net_device *netdev)
1480 ionic_set_rx_mode(netdev, CAN_NOT_SLEEP);
1483 static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
1487 if (features & NETIF_F_HW_VLAN_CTAG_TX)
1488 wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
1489 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1490 wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
1491 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1492 wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
1493 if (features & NETIF_F_RXHASH)
1494 wanted |= IONIC_ETH_HW_RX_HASH;
1495 if (features & NETIF_F_RXCSUM)
1496 wanted |= IONIC_ETH_HW_RX_CSUM;
1497 if (features & NETIF_F_SG)
1498 wanted |= IONIC_ETH_HW_TX_SG;
1499 if (features & NETIF_F_HW_CSUM)
1500 wanted |= IONIC_ETH_HW_TX_CSUM;
1501 if (features & NETIF_F_TSO)
1502 wanted |= IONIC_ETH_HW_TSO;
1503 if (features & NETIF_F_TSO6)
1504 wanted |= IONIC_ETH_HW_TSO_IPV6;
1505 if (features & NETIF_F_TSO_ECN)
1506 wanted |= IONIC_ETH_HW_TSO_ECN;
1507 if (features & NETIF_F_GSO_GRE)
1508 wanted |= IONIC_ETH_HW_TSO_GRE;
1509 if (features & NETIF_F_GSO_GRE_CSUM)
1510 wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
1511 if (features & NETIF_F_GSO_IPXIP4)
1512 wanted |= IONIC_ETH_HW_TSO_IPXIP4;
1513 if (features & NETIF_F_GSO_IPXIP6)
1514 wanted |= IONIC_ETH_HW_TSO_IPXIP6;
1515 if (features & NETIF_F_GSO_UDP_TUNNEL)
1516 wanted |= IONIC_ETH_HW_TSO_UDP;
1517 if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
1518 wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
1520 return cpu_to_le64(wanted);
1523 static int ionic_set_nic_features(struct ionic_lif *lif,
1524 netdev_features_t features)
1526 struct device *dev = lif->ionic->dev;
1527 struct ionic_admin_ctx ctx = {
1528 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1529 .cmd.lif_setattr = {
1530 .opcode = IONIC_CMD_LIF_SETATTR,
1531 .index = cpu_to_le16(lif->index),
1532 .attr = IONIC_LIF_ATTR_FEATURES,
1535 u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
1536 IONIC_ETH_HW_VLAN_RX_STRIP |
1537 IONIC_ETH_HW_VLAN_RX_FILTER;
1538 u64 old_hw_features;
1541 ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
1544 ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
1546 err = ionic_adminq_post_wait(lif, &ctx);
1550 old_hw_features = lif->hw_features;
1551 lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
1552 ctx.comp.lif_setattr.features);
1554 if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
1555 ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1557 if ((vlan_flags & features) &&
1558 !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
1559 dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1561 if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1562 dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
1563 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1564 dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
1565 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1566 dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
1567 if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1568 dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
1569 if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1570 dev_dbg(dev, "feature ETH_HW_TX_SG\n");
1571 if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1572 dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
1573 if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1574 dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
1575 if (lif->hw_features & IONIC_ETH_HW_TSO)
1576 dev_dbg(dev, "feature ETH_HW_TSO\n");
1577 if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1578 dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
1579 if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1580 dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
1581 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1582 dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
1583 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1584 dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
1585 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1586 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
1587 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1588 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
1589 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1590 dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
1591 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1592 dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
1593 if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
1594 dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
1599 static int ionic_init_nic_features(struct ionic_lif *lif)
1601 struct net_device *netdev = lif->netdev;
1602 netdev_features_t features;
1605 /* set up what we expect to support by default */
1606 features = NETIF_F_HW_VLAN_CTAG_TX |
1607 NETIF_F_HW_VLAN_CTAG_RX |
1608 NETIF_F_HW_VLAN_CTAG_FILTER |
1617 err = ionic_set_nic_features(lif, features);
1621 /* tell the netdev what we actually can support */
1622 netdev->features |= NETIF_F_HIGHDMA;
1624 if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1625 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
1626 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1627 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1628 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1629 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1630 if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1631 netdev->hw_features |= NETIF_F_RXHASH;
1632 if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1633 netdev->hw_features |= NETIF_F_SG;
1635 if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1636 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
1637 if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1638 netdev->hw_enc_features |= NETIF_F_RXCSUM;
1639 if (lif->hw_features & IONIC_ETH_HW_TSO)
1640 netdev->hw_enc_features |= NETIF_F_TSO;
1641 if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1642 netdev->hw_enc_features |= NETIF_F_TSO6;
1643 if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1644 netdev->hw_enc_features |= NETIF_F_TSO_ECN;
1645 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1646 netdev->hw_enc_features |= NETIF_F_GSO_GRE;
1647 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1648 netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
1649 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1650 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
1651 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1652 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
1653 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1654 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
1655 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1656 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1658 netdev->hw_features |= netdev->hw_enc_features;
1659 netdev->features |= netdev->hw_features;
1660 netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
1662 netdev->priv_flags |= IFF_UNICAST_FLT |
1663 IFF_LIVE_ADDR_CHANGE;
1668 static int ionic_set_features(struct net_device *netdev,
1669 netdev_features_t features)
1671 struct ionic_lif *lif = netdev_priv(netdev);
1674 netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1675 __func__, (u64)lif->netdev->features, (u64)features);
1677 err = ionic_set_nic_features(lif, features);
1682 static int ionic_set_mac_address(struct net_device *netdev, void *sa)
1684 struct sockaddr *addr = sa;
1688 mac = (u8 *)addr->sa_data;
1689 if (ether_addr_equal(netdev->dev_addr, mac))
1692 err = eth_prepare_mac_addr_change(netdev, addr);
1696 if (!is_zero_ether_addr(netdev->dev_addr)) {
1697 netdev_info(netdev, "deleting mac addr %pM\n",
1699 ionic_addr_del(netdev, netdev->dev_addr);
1702 eth_commit_mac_addr_change(netdev, addr);
1703 netdev_info(netdev, "updating mac addr %pM\n", mac);
1705 return ionic_addr_add(netdev, mac);
1708 static void ionic_stop_queues_reconfig(struct ionic_lif *lif)
1710 /* Stop and clean the queues before reconfiguration */
1711 mutex_lock(&lif->queue_lock);
1712 netif_device_detach(lif->netdev);
1713 ionic_stop_queues(lif);
1714 ionic_txrx_deinit(lif);
1717 static int ionic_start_queues_reconfig(struct ionic_lif *lif)
1721 /* Re-init the queues after reconfiguration */
1723 /* The only way txrx_init can fail here is if communication
1724 * with FW is suddenly broken. There's not much we can do
1725 * at this point - error messages have already been printed,
1726 * so we can continue on and the user can eventually do a
1727 * DOWN and UP to try to reset and clear the issue.
1729 err = ionic_txrx_init(lif);
1730 mutex_unlock(&lif->queue_lock);
1731 ionic_link_status_check_request(lif, CAN_SLEEP);
1732 netif_device_attach(lif->netdev);
1737 static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
1739 struct ionic_lif *lif = netdev_priv(netdev);
1740 struct ionic_admin_ctx ctx = {
1741 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1742 .cmd.lif_setattr = {
1743 .opcode = IONIC_CMD_LIF_SETATTR,
1744 .index = cpu_to_le16(lif->index),
1745 .attr = IONIC_LIF_ATTR_MTU,
1746 .mtu = cpu_to_le32(new_mtu),
1751 err = ionic_adminq_post_wait(lif, &ctx);
1755 /* if we're not running, nothing more to do */
1756 if (!netif_running(netdev)) {
1757 netdev->mtu = new_mtu;
1761 ionic_stop_queues_reconfig(lif);
1762 netdev->mtu = new_mtu;
1763 return ionic_start_queues_reconfig(lif);
1766 static void ionic_tx_timeout_work(struct work_struct *ws)
1768 struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
1770 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
1773 /* if we were stopped before this scheduled job was launched,
1774 * don't bother the queues as they are already stopped.
1776 if (!netif_running(lif->netdev))
1779 ionic_stop_queues_reconfig(lif);
1780 ionic_start_queues_reconfig(lif);
1783 static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1785 struct ionic_lif *lif = netdev_priv(netdev);
1787 netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
1788 schedule_work(&lif->tx_timeout_work);
1791 static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1794 struct ionic_lif *lif = netdev_priv(netdev);
1795 struct ionic_admin_ctx ctx = {
1796 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1797 .cmd.rx_filter_add = {
1798 .opcode = IONIC_CMD_RX_FILTER_ADD,
1799 .lif_index = cpu_to_le16(lif->index),
1800 .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_VLAN),
1801 .vlan.vlan = cpu_to_le16(vid),
1806 netdev_dbg(netdev, "rx_filter add VLAN %d\n", vid);
1807 err = ionic_adminq_post_wait(lif, &ctx);
1811 return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx);
1814 static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1817 struct ionic_lif *lif = netdev_priv(netdev);
1818 struct ionic_admin_ctx ctx = {
1819 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1820 .cmd.rx_filter_del = {
1821 .opcode = IONIC_CMD_RX_FILTER_DEL,
1822 .lif_index = cpu_to_le16(lif->index),
1825 struct ionic_rx_filter *f;
1827 spin_lock_bh(&lif->rx_filters.lock);
1829 f = ionic_rx_filter_by_vlan(lif, vid);
1831 spin_unlock_bh(&lif->rx_filters.lock);
1835 netdev_dbg(netdev, "rx_filter del VLAN %d (id %d)\n",
1838 ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1839 ionic_rx_filter_free(lif, f);
1840 spin_unlock_bh(&lif->rx_filters.lock);
1842 return ionic_adminq_post_wait(lif, &ctx);
1845 int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
1846 const u8 *key, const u32 *indir)
1848 struct ionic_admin_ctx ctx = {
1849 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1850 .cmd.lif_setattr = {
1851 .opcode = IONIC_CMD_LIF_SETATTR,
1852 .attr = IONIC_LIF_ATTR_RSS,
1853 .rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
1856 unsigned int i, tbl_sz;
1858 if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
1859 lif->rss_types = types;
1860 ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
1864 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
1867 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1868 for (i = 0; i < tbl_sz; i++)
1869 lif->rss_ind_tbl[i] = indir[i];
1872 memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
1873 IONIC_RSS_HASH_KEY_SIZE);
1875 return ionic_adminq_post_wait(lif, &ctx);
1878 static int ionic_lif_rss_init(struct ionic_lif *lif)
1880 unsigned int tbl_sz;
1883 lif->rss_types = IONIC_RSS_TYPE_IPV4 |
1884 IONIC_RSS_TYPE_IPV4_TCP |
1885 IONIC_RSS_TYPE_IPV4_UDP |
1886 IONIC_RSS_TYPE_IPV6 |
1887 IONIC_RSS_TYPE_IPV6_TCP |
1888 IONIC_RSS_TYPE_IPV6_UDP;
1890 /* Fill indirection table with 'default' values */
1891 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1892 for (i = 0; i < tbl_sz; i++)
1893 lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
1895 return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1898 static void ionic_lif_rss_deinit(struct ionic_lif *lif)
1902 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1903 memset(lif->rss_ind_tbl, 0, tbl_sz);
1904 memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
1906 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1909 static void ionic_lif_quiesce(struct ionic_lif *lif)
1911 struct ionic_admin_ctx ctx = {
1912 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1913 .cmd.lif_setattr = {
1914 .opcode = IONIC_CMD_LIF_SETATTR,
1915 .index = cpu_to_le16(lif->index),
1916 .attr = IONIC_LIF_ATTR_STATE,
1917 .state = IONIC_LIF_QUIESCE,
1922 err = ionic_adminq_post_wait(lif, &ctx);
1924 netdev_err(lif->netdev, "lif quiesce failed %d\n", err);
1927 static void ionic_txrx_disable(struct ionic_lif *lif)
1933 for (i = 0; i < lif->nxqs; i++)
1934 err = ionic_qcq_disable(lif->txqcqs[i], (err != -ETIMEDOUT));
1937 if (lif->hwstamp_txq)
1938 err = ionic_qcq_disable(lif->hwstamp_txq, (err != -ETIMEDOUT));
1941 for (i = 0; i < lif->nxqs; i++)
1942 err = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
1945 if (lif->hwstamp_rxq)
1946 err = ionic_qcq_disable(lif->hwstamp_rxq, (err != -ETIMEDOUT));
1948 ionic_lif_quiesce(lif);
1951 static void ionic_txrx_deinit(struct ionic_lif *lif)
1956 for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
1957 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1958 ionic_tx_flush(&lif->txqcqs[i]->cq);
1959 ionic_tx_empty(&lif->txqcqs[i]->q);
1964 for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
1965 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
1966 ionic_rx_empty(&lif->rxqcqs[i]->q);
1971 if (lif->hwstamp_txq) {
1972 ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
1973 ionic_tx_flush(&lif->hwstamp_txq->cq);
1974 ionic_tx_empty(&lif->hwstamp_txq->q);
1977 if (lif->hwstamp_rxq) {
1978 ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
1979 ionic_rx_empty(&lif->hwstamp_rxq->q);
1983 static void ionic_txrx_free(struct ionic_lif *lif)
1988 for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
1989 ionic_qcq_free(lif, lif->txqcqs[i]);
1990 devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
1991 lif->txqcqs[i] = NULL;
1996 for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
1997 ionic_qcq_free(lif, lif->rxqcqs[i]);
1998 devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
1999 lif->rxqcqs[i] = NULL;
2003 if (lif->hwstamp_txq) {
2004 ionic_qcq_free(lif, lif->hwstamp_txq);
2005 devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
2006 lif->hwstamp_txq = NULL;
2009 if (lif->hwstamp_rxq) {
2010 ionic_qcq_free(lif, lif->hwstamp_rxq);
2011 devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
2012 lif->hwstamp_rxq = NULL;
2016 static int ionic_txrx_alloc(struct ionic_lif *lif)
2018 unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2019 unsigned int flags, i;
2022 num_desc = lif->ntxq_descs;
2023 desc_sz = sizeof(struct ionic_txq_desc);
2024 comp_sz = sizeof(struct ionic_txq_comp);
2026 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2027 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2028 sizeof(struct ionic_txq_sg_desc_v1))
2029 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2031 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2033 flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2034 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2035 flags |= IONIC_QCQ_F_INTR;
2036 for (i = 0; i < lif->nxqs; i++) {
2037 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2038 num_desc, desc_sz, comp_sz, sg_desc_sz,
2039 lif->kern_pid, &lif->txqcqs[i]);
2043 if (flags & IONIC_QCQ_F_INTR) {
2044 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2045 lif->txqcqs[i]->intr.index,
2046 lif->tx_coalesce_hw);
2047 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2048 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2051 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2054 flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
2056 num_desc = lif->nrxq_descs;
2057 desc_sz = sizeof(struct ionic_rxq_desc);
2058 comp_sz = sizeof(struct ionic_rxq_comp);
2059 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2061 if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2064 for (i = 0; i < lif->nxqs; i++) {
2065 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2066 num_desc, desc_sz, comp_sz, sg_desc_sz,
2067 lif->kern_pid, &lif->rxqcqs[i]);
2071 lif->rxqcqs[i]->q.features = lif->rxq_features;
2073 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2074 lif->rxqcqs[i]->intr.index,
2075 lif->rx_coalesce_hw);
2076 if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
2077 lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
2079 if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2080 ionic_link_qcq_interrupts(lif->rxqcqs[i],
2083 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2089 ionic_txrx_free(lif);
2094 static int ionic_txrx_init(struct ionic_lif *lif)
2099 for (i = 0; i < lif->nxqs; i++) {
2100 err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
2104 err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
2106 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2111 if (lif->netdev->features & NETIF_F_RXHASH)
2112 ionic_lif_rss_init(lif);
2114 ionic_set_rx_mode(lif->netdev, CAN_SLEEP);
2120 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2121 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2127 static int ionic_txrx_enable(struct ionic_lif *lif)
2132 for (i = 0; i < lif->nxqs; i++) {
2133 if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
2134 dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
2139 ionic_rx_fill(&lif->rxqcqs[i]->q);
2140 err = ionic_qcq_enable(lif->rxqcqs[i]);
2144 err = ionic_qcq_enable(lif->txqcqs[i]);
2146 derr = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
2151 if (lif->hwstamp_rxq) {
2152 ionic_rx_fill(&lif->hwstamp_rxq->q);
2153 err = ionic_qcq_enable(lif->hwstamp_rxq);
2155 goto err_out_hwstamp_rx;
2158 if (lif->hwstamp_txq) {
2159 err = ionic_qcq_enable(lif->hwstamp_txq);
2161 goto err_out_hwstamp_tx;
2167 if (lif->hwstamp_rxq)
2168 derr = ionic_qcq_disable(lif->hwstamp_rxq, (derr != -ETIMEDOUT));
2173 derr = ionic_qcq_disable(lif->txqcqs[i], (derr != -ETIMEDOUT));
2174 derr = ionic_qcq_disable(lif->rxqcqs[i], (derr != -ETIMEDOUT));
2180 static int ionic_start_queues(struct ionic_lif *lif)
2184 if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
2187 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2190 if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
2193 err = ionic_txrx_enable(lif);
2195 clear_bit(IONIC_LIF_F_UP, lif->state);
2198 netif_tx_wake_all_queues(lif->netdev);
2203 static int ionic_open(struct net_device *netdev)
2205 struct ionic_lif *lif = netdev_priv(netdev);
2208 /* If recovering from a broken state, clear the bit and we'll try again */
2209 if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
2210 netdev_info(netdev, "clearing broken state\n");
2212 err = ionic_txrx_alloc(lif);
2216 err = ionic_txrx_init(lif);
2220 err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
2222 goto err_txrx_deinit;
2224 err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
2226 goto err_txrx_deinit;
2228 /* don't start the queues until we have link */
2229 if (netif_carrier_ok(netdev)) {
2230 err = ionic_start_queues(lif);
2232 goto err_txrx_deinit;
2238 ionic_txrx_deinit(lif);
2240 ionic_txrx_free(lif);
2244 static void ionic_stop_queues(struct ionic_lif *lif)
2246 if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
2249 netif_tx_disable(lif->netdev);
2250 ionic_txrx_disable(lif);
2253 static int ionic_stop(struct net_device *netdev)
2255 struct ionic_lif *lif = netdev_priv(netdev);
2257 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2260 ionic_stop_queues(lif);
2261 ionic_txrx_deinit(lif);
2262 ionic_txrx_free(lif);
2267 static int ionic_do_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2269 struct ionic_lif *lif = netdev_priv(netdev);
2273 return ionic_lif_hwstamp_set(lif, ifr);
2275 return ionic_lif_hwstamp_get(lif, ifr);
2281 static int ionic_get_vf_config(struct net_device *netdev,
2282 int vf, struct ifla_vf_info *ivf)
2284 struct ionic_lif *lif = netdev_priv(netdev);
2285 struct ionic *ionic = lif->ionic;
2288 if (!netif_device_present(netdev))
2291 down_read(&ionic->vf_op_lock);
2293 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2297 ivf->vlan = le16_to_cpu(ionic->vfs[vf].vlanid);
2299 ivf->spoofchk = ionic->vfs[vf].spoofchk;
2300 ivf->linkstate = ionic->vfs[vf].linkstate;
2301 ivf->max_tx_rate = le32_to_cpu(ionic->vfs[vf].maxrate);
2302 ivf->trusted = ionic->vfs[vf].trusted;
2303 ether_addr_copy(ivf->mac, ionic->vfs[vf].macaddr);
2306 up_read(&ionic->vf_op_lock);
2310 static int ionic_get_vf_stats(struct net_device *netdev, int vf,
2311 struct ifla_vf_stats *vf_stats)
2313 struct ionic_lif *lif = netdev_priv(netdev);
2314 struct ionic *ionic = lif->ionic;
2315 struct ionic_lif_stats *vs;
2318 if (!netif_device_present(netdev))
2321 down_read(&ionic->vf_op_lock);
2323 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2326 memset(vf_stats, 0, sizeof(*vf_stats));
2327 vs = &ionic->vfs[vf].stats;
2329 vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
2330 vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
2331 vf_stats->rx_bytes = le64_to_cpu(vs->rx_ucast_bytes);
2332 vf_stats->tx_bytes = le64_to_cpu(vs->tx_ucast_bytes);
2333 vf_stats->broadcast = le64_to_cpu(vs->rx_bcast_packets);
2334 vf_stats->multicast = le64_to_cpu(vs->rx_mcast_packets);
2335 vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
2336 le64_to_cpu(vs->rx_mcast_drop_packets) +
2337 le64_to_cpu(vs->rx_bcast_drop_packets);
2338 vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
2339 le64_to_cpu(vs->tx_mcast_drop_packets) +
2340 le64_to_cpu(vs->tx_bcast_drop_packets);
2343 up_read(&ionic->vf_op_lock);
2347 static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2349 struct ionic_lif *lif = netdev_priv(netdev);
2350 struct ionic *ionic = lif->ionic;
2353 if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
2356 if (!netif_device_present(netdev))
2359 down_write(&ionic->vf_op_lock);
2361 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2364 ret = ionic_set_vf_config(ionic, vf, IONIC_VF_ATTR_MAC, mac);
2366 ether_addr_copy(ionic->vfs[vf].macaddr, mac);
2369 up_write(&ionic->vf_op_lock);
2373 static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2374 u8 qos, __be16 proto)
2376 struct ionic_lif *lif = netdev_priv(netdev);
2377 struct ionic *ionic = lif->ionic;
2380 /* until someday when we support qos */
2387 if (proto != htons(ETH_P_8021Q))
2388 return -EPROTONOSUPPORT;
2390 if (!netif_device_present(netdev))
2393 down_write(&ionic->vf_op_lock);
2395 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2398 ret = ionic_set_vf_config(ionic, vf,
2399 IONIC_VF_ATTR_VLAN, (u8 *)&vlan);
2401 ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
2404 up_write(&ionic->vf_op_lock);
2408 static int ionic_set_vf_rate(struct net_device *netdev, int vf,
2409 int tx_min, int tx_max)
2411 struct ionic_lif *lif = netdev_priv(netdev);
2412 struct ionic *ionic = lif->ionic;
2415 /* setting the min just seems silly */
2419 if (!netif_device_present(netdev))
2422 down_write(&ionic->vf_op_lock);
2424 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2427 ret = ionic_set_vf_config(ionic, vf,
2428 IONIC_VF_ATTR_RATE, (u8 *)&tx_max);
2430 lif->ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
2433 up_write(&ionic->vf_op_lock);
2437 static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
2439 struct ionic_lif *lif = netdev_priv(netdev);
2440 struct ionic *ionic = lif->ionic;
2441 u8 data = set; /* convert to u8 for config */
2444 if (!netif_device_present(netdev))
2447 down_write(&ionic->vf_op_lock);
2449 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2452 ret = ionic_set_vf_config(ionic, vf,
2453 IONIC_VF_ATTR_SPOOFCHK, &data);
2455 ionic->vfs[vf].spoofchk = data;
2458 up_write(&ionic->vf_op_lock);
2462 static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
2464 struct ionic_lif *lif = netdev_priv(netdev);
2465 struct ionic *ionic = lif->ionic;
2466 u8 data = set; /* convert to u8 for config */
2469 if (!netif_device_present(netdev))
2472 down_write(&ionic->vf_op_lock);
2474 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2477 ret = ionic_set_vf_config(ionic, vf,
2478 IONIC_VF_ATTR_TRUST, &data);
2480 ionic->vfs[vf].trusted = data;
2483 up_write(&ionic->vf_op_lock);
2487 static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
2489 struct ionic_lif *lif = netdev_priv(netdev);
2490 struct ionic *ionic = lif->ionic;
2495 case IFLA_VF_LINK_STATE_ENABLE:
2496 data = IONIC_VF_LINK_STATUS_UP;
2498 case IFLA_VF_LINK_STATE_DISABLE:
2499 data = IONIC_VF_LINK_STATUS_DOWN;
2501 case IFLA_VF_LINK_STATE_AUTO:
2502 data = IONIC_VF_LINK_STATUS_AUTO;
2508 if (!netif_device_present(netdev))
2511 down_write(&ionic->vf_op_lock);
2513 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2516 ret = ionic_set_vf_config(ionic, vf,
2517 IONIC_VF_ATTR_LINKSTATE, &data);
2519 ionic->vfs[vf].linkstate = set;
2522 up_write(&ionic->vf_op_lock);
2526 static const struct net_device_ops ionic_netdev_ops = {
2527 .ndo_open = ionic_open,
2528 .ndo_stop = ionic_stop,
2529 .ndo_do_ioctl = ionic_do_ioctl,
2530 .ndo_start_xmit = ionic_start_xmit,
2531 .ndo_get_stats64 = ionic_get_stats64,
2532 .ndo_set_rx_mode = ionic_ndo_set_rx_mode,
2533 .ndo_set_features = ionic_set_features,
2534 .ndo_set_mac_address = ionic_set_mac_address,
2535 .ndo_validate_addr = eth_validate_addr,
2536 .ndo_tx_timeout = ionic_tx_timeout,
2537 .ndo_change_mtu = ionic_change_mtu,
2538 .ndo_vlan_rx_add_vid = ionic_vlan_rx_add_vid,
2539 .ndo_vlan_rx_kill_vid = ionic_vlan_rx_kill_vid,
2540 .ndo_set_vf_vlan = ionic_set_vf_vlan,
2541 .ndo_set_vf_trust = ionic_set_vf_trust,
2542 .ndo_set_vf_mac = ionic_set_vf_mac,
2543 .ndo_set_vf_rate = ionic_set_vf_rate,
2544 .ndo_set_vf_spoofchk = ionic_set_vf_spoofchk,
2545 .ndo_get_vf_config = ionic_get_vf_config,
2546 .ndo_set_vf_link_state = ionic_set_vf_link_state,
2547 .ndo_get_vf_stats = ionic_get_vf_stats,
2550 static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2552 /* only swapping the queues, not the napi, flags, or other stuff */
2553 swap(a->q.features, b->q.features);
2554 swap(a->q.num_descs, b->q.num_descs);
2555 swap(a->q.desc_size, b->q.desc_size);
2556 swap(a->q.base, b->q.base);
2557 swap(a->q.base_pa, b->q.base_pa);
2558 swap(a->q.info, b->q.info);
2559 swap(a->q_base, b->q_base);
2560 swap(a->q_base_pa, b->q_base_pa);
2561 swap(a->q_size, b->q_size);
2563 swap(a->q.sg_desc_size, b->q.sg_desc_size);
2564 swap(a->q.sg_base, b->q.sg_base);
2565 swap(a->q.sg_base_pa, b->q.sg_base_pa);
2566 swap(a->sg_base, b->sg_base);
2567 swap(a->sg_base_pa, b->sg_base_pa);
2568 swap(a->sg_size, b->sg_size);
2570 swap(a->cq.num_descs, b->cq.num_descs);
2571 swap(a->cq.desc_size, b->cq.desc_size);
2572 swap(a->cq.base, b->cq.base);
2573 swap(a->cq.base_pa, b->cq.base_pa);
2574 swap(a->cq.info, b->cq.info);
2575 swap(a->cq_base, b->cq_base);
2576 swap(a->cq_base_pa, b->cq_base_pa);
2577 swap(a->cq_size, b->cq_size);
2579 ionic_debugfs_del_qcq(a);
2580 ionic_debugfs_add_qcq(a->q.lif, a);
2583 int ionic_reconfigure_queues(struct ionic_lif *lif,
2584 struct ionic_queue_params *qparam)
2586 unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2587 struct ionic_qcq **tx_qcqs = NULL;
2588 struct ionic_qcq **rx_qcqs = NULL;
2589 unsigned int flags, i;
2592 /* allocate temporary qcq arrays to hold new queue structs */
2593 if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
2594 tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
2595 sizeof(struct ionic_qcq *), GFP_KERNEL);
2599 if (qparam->nxqs != lif->nxqs ||
2600 qparam->nrxq_descs != lif->nrxq_descs ||
2601 qparam->rxq_features != lif->rxq_features) {
2602 rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
2603 sizeof(struct ionic_qcq *), GFP_KERNEL);
2608 /* allocate new desc_info and rings, but leave the interrupt setup
2609 * until later so as to not mess with the still-running queues
2612 num_desc = qparam->ntxq_descs;
2613 desc_sz = sizeof(struct ionic_txq_desc);
2614 comp_sz = sizeof(struct ionic_txq_comp);
2616 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2617 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2618 sizeof(struct ionic_txq_sg_desc_v1))
2619 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2621 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2623 for (i = 0; i < qparam->nxqs; i++) {
2624 flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2625 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2626 num_desc, desc_sz, comp_sz, sg_desc_sz,
2627 lif->kern_pid, &tx_qcqs[i]);
2634 num_desc = qparam->nrxq_descs;
2635 desc_sz = sizeof(struct ionic_rxq_desc);
2636 comp_sz = sizeof(struct ionic_rxq_comp);
2637 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2639 if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2642 for (i = 0; i < qparam->nxqs; i++) {
2643 flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2644 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2645 num_desc, desc_sz, comp_sz, sg_desc_sz,
2646 lif->kern_pid, &rx_qcqs[i]);
2650 rx_qcqs[i]->q.features = qparam->rxq_features;
2654 /* stop and clean the queues */
2655 ionic_stop_queues_reconfig(lif);
2657 if (qparam->nxqs != lif->nxqs) {
2658 err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
2660 goto err_out_reinit_unlock;
2661 err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
2663 netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
2664 goto err_out_reinit_unlock;
2668 /* swap new desc_info and rings, keeping existing interrupt config */
2670 lif->ntxq_descs = qparam->ntxq_descs;
2671 for (i = 0; i < qparam->nxqs; i++)
2672 ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
2676 lif->nrxq_descs = qparam->nrxq_descs;
2677 for (i = 0; i < qparam->nxqs; i++)
2678 ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
2681 /* if we need to change the interrupt layout, this is the time */
2682 if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
2683 qparam->nxqs != lif->nxqs) {
2684 if (qparam->intr_split) {
2685 set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2687 clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2688 lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2689 lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2692 /* clear existing interrupt assignments */
2693 for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
2694 ionic_qcq_intr_free(lif, lif->txqcqs[i]);
2695 ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
2698 /* re-assign the interrupts */
2699 for (i = 0; i < qparam->nxqs; i++) {
2700 lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2701 err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
2702 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2703 lif->rxqcqs[i]->intr.index,
2704 lif->rx_coalesce_hw);
2706 if (qparam->intr_split) {
2707 lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2708 err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
2709 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2710 lif->txqcqs[i]->intr.index,
2711 lif->tx_coalesce_hw);
2712 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2713 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2715 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2716 ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
2721 /* now we can rework the debugfs mappings */
2723 for (i = 0; i < qparam->nxqs; i++) {
2724 ionic_debugfs_del_qcq(lif->txqcqs[i]);
2725 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2730 for (i = 0; i < qparam->nxqs; i++) {
2731 ionic_debugfs_del_qcq(lif->rxqcqs[i]);
2732 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2736 swap(lif->nxqs, qparam->nxqs);
2737 swap(lif->rxq_features, qparam->rxq_features);
2739 err_out_reinit_unlock:
2740 /* re-init the queues, but don't lose an error code */
2742 ionic_start_queues_reconfig(lif);
2744 err = ionic_start_queues_reconfig(lif);
2747 /* free old allocs without cleaning intr */
2748 for (i = 0; i < qparam->nxqs; i++) {
2749 if (tx_qcqs && tx_qcqs[i]) {
2750 tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2751 ionic_qcq_free(lif, tx_qcqs[i]);
2752 devm_kfree(lif->ionic->dev, tx_qcqs[i]);
2755 if (rx_qcqs && rx_qcqs[i]) {
2756 rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2757 ionic_qcq_free(lif, rx_qcqs[i]);
2758 devm_kfree(lif->ionic->dev, rx_qcqs[i]);
2765 devm_kfree(lif->ionic->dev, rx_qcqs);
2769 devm_kfree(lif->ionic->dev, tx_qcqs);
2773 /* clean the unused dma and info allocations when new set is smaller
2774 * than the full array, but leave the qcq shells in place
2776 for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
2777 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2778 ionic_qcq_free(lif, lif->txqcqs[i]);
2780 lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2781 ionic_qcq_free(lif, lif->rxqcqs[i]);
2787 int ionic_lif_alloc(struct ionic *ionic)
2789 struct device *dev = ionic->dev;
2790 union ionic_lif_identity *lid;
2791 struct net_device *netdev;
2792 struct ionic_lif *lif;
2796 lid = kzalloc(sizeof(*lid), GFP_KERNEL);
2800 netdev = alloc_etherdev_mqs(sizeof(*lif),
2801 ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
2803 dev_err(dev, "Cannot allocate netdev, aborting\n");
2805 goto err_out_free_lid;
2808 SET_NETDEV_DEV(netdev, dev);
2810 lif = netdev_priv(netdev);
2811 lif->netdev = netdev;
2813 netdev->netdev_ops = &ionic_netdev_ops;
2814 ionic_ethtool_set_ops(netdev);
2816 netdev->watchdog_timeo = 2 * HZ;
2817 netif_carrier_off(netdev);
2819 lif->identity = lid;
2820 lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
2821 err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
2823 dev_err(ionic->dev, "Cannot identify type %d: %d\n",
2824 lif->lif_type, err);
2825 goto err_out_free_netdev;
2827 lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
2828 le32_to_cpu(lif->identity->eth.min_frame_size));
2829 lif->netdev->max_mtu =
2830 le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
2832 lif->neqs = ionic->neqs_per_lif;
2833 lif->nxqs = ionic->ntxqs_per_lif;
2837 lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
2838 lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
2840 /* Convert the default coalesce value to actual hw resolution */
2841 lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
2842 lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
2843 lif->rx_coalesce_usecs);
2844 lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2845 lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2846 set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
2847 set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
2849 snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
2851 spin_lock_init(&lif->adminq_lock);
2853 spin_lock_init(&lif->deferred.lock);
2854 INIT_LIST_HEAD(&lif->deferred.list);
2855 INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
2857 /* allocate lif info */
2858 lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
2859 lif->info = dma_alloc_coherent(dev, lif->info_sz,
2860 &lif->info_pa, GFP_KERNEL);
2862 dev_err(dev, "Failed to allocate lif info, aborting\n");
2864 goto err_out_free_netdev;
2867 ionic_debugfs_add_lif(lif);
2869 /* allocate control queues and txrx queue arrays */
2870 ionic_lif_queue_identify(lif);
2871 err = ionic_qcqs_alloc(lif);
2873 goto err_out_free_lif_info;
2875 /* allocate rss indirection table */
2876 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
2877 lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
2878 lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
2879 &lif->rss_ind_tbl_pa,
2882 if (!lif->rss_ind_tbl) {
2884 dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
2885 goto err_out_free_qcqs;
2887 netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
2889 ionic_lif_alloc_phc(lif);
2894 ionic_qcqs_free(lif);
2895 err_out_free_lif_info:
2896 dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
2899 err_out_free_netdev:
2900 free_netdev(lif->netdev);
2908 static void ionic_lif_reset(struct ionic_lif *lif)
2910 struct ionic_dev *idev = &lif->ionic->idev;
2912 mutex_lock(&lif->ionic->dev_cmd_lock);
2913 ionic_dev_cmd_lif_reset(idev, lif->index);
2914 ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
2915 mutex_unlock(&lif->ionic->dev_cmd_lock);
2918 static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
2920 struct ionic *ionic = lif->ionic;
2922 if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
2925 dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
2927 netif_device_detach(lif->netdev);
2929 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
2930 dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
2931 mutex_lock(&lif->queue_lock);
2932 ionic_stop_queues(lif);
2933 mutex_unlock(&lif->queue_lock);
2936 if (netif_running(lif->netdev)) {
2937 ionic_txrx_deinit(lif);
2938 ionic_txrx_free(lif);
2940 ionic_lif_deinit(lif);
2942 ionic_qcqs_free(lif);
2944 dev_info(ionic->dev, "FW Down: LIFs stopped\n");
2947 static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
2949 struct ionic *ionic = lif->ionic;
2952 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2955 dev_info(ionic->dev, "FW Up: restarting LIFs\n");
2957 ionic_init_devinfo(ionic);
2958 err = ionic_identify(ionic);
2961 err = ionic_port_identify(ionic);
2964 err = ionic_port_init(ionic);
2967 err = ionic_qcqs_alloc(lif);
2971 err = ionic_lif_init(lif);
2975 if (lif->registered)
2976 ionic_lif_set_netdev_info(lif);
2978 ionic_rx_filter_replay(lif);
2980 if (netif_running(lif->netdev)) {
2981 err = ionic_txrx_alloc(lif);
2983 goto err_lifs_deinit;
2985 err = ionic_txrx_init(lif);
2990 clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
2991 ionic_link_status_check_request(lif, CAN_SLEEP);
2992 netif_device_attach(lif->netdev);
2993 dev_info(ionic->dev, "FW Up: LIFs restarted\n");
2995 /* restore the hardware timestamping queues */
2996 ionic_lif_hwstamp_replay(lif);
3001 ionic_txrx_free(lif);
3003 ionic_lif_deinit(lif);
3005 ionic_qcqs_free(lif);
3007 dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
3010 void ionic_lif_free(struct ionic_lif *lif)
3012 struct device *dev = lif->ionic->dev;
3014 ionic_lif_free_phc(lif);
3016 /* free rss indirection table */
3017 dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
3018 lif->rss_ind_tbl_pa);
3019 lif->rss_ind_tbl = NULL;
3020 lif->rss_ind_tbl_pa = 0;
3023 ionic_qcqs_free(lif);
3024 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3025 ionic_lif_reset(lif);
3028 kfree(lif->identity);
3029 dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3033 /* unmap doorbell page */
3034 ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3035 lif->kern_dbpage = NULL;
3036 kfree(lif->dbid_inuse);
3037 lif->dbid_inuse = NULL;
3039 /* free netdev & lif */
3040 ionic_debugfs_del_lif(lif);
3041 free_netdev(lif->netdev);
3044 void ionic_lif_deinit(struct ionic_lif *lif)
3046 if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
3049 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3050 cancel_work_sync(&lif->deferred.work);
3051 cancel_work_sync(&lif->tx_timeout_work);
3052 ionic_rx_filters_deinit(lif);
3053 if (lif->netdev->features & NETIF_F_RXHASH)
3054 ionic_lif_rss_deinit(lif);
3057 napi_disable(&lif->adminqcq->napi);
3058 ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3059 ionic_lif_qcq_deinit(lif, lif->adminqcq);
3061 mutex_destroy(&lif->queue_lock);
3062 ionic_lif_reset(lif);
3065 static int ionic_lif_adminq_init(struct ionic_lif *lif)
3067 struct device *dev = lif->ionic->dev;
3068 struct ionic_q_init_comp comp;
3069 struct ionic_dev *idev;
3070 struct ionic_qcq *qcq;
3071 struct ionic_queue *q;
3074 idev = &lif->ionic->idev;
3075 qcq = lif->adminqcq;
3078 mutex_lock(&lif->ionic->dev_cmd_lock);
3079 ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
3080 err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3081 ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3082 mutex_unlock(&lif->ionic->dev_cmd_lock);
3084 netdev_err(lif->netdev, "adminq init failed %d\n", err);
3088 q->hw_type = comp.hw_type;
3089 q->hw_index = le32_to_cpu(comp.hw_index);
3090 q->dbval = IONIC_DBELL_QID(q->hw_index);
3092 dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
3093 dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
3095 netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi,
3098 napi_enable(&qcq->napi);
3100 if (qcq->flags & IONIC_QCQ_F_INTR)
3101 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
3102 IONIC_INTR_MASK_CLEAR);
3104 qcq->flags |= IONIC_QCQ_F_INITED;
3109 static int ionic_lif_notifyq_init(struct ionic_lif *lif)
3111 struct ionic_qcq *qcq = lif->notifyqcq;
3112 struct device *dev = lif->ionic->dev;
3113 struct ionic_queue *q = &qcq->q;
3116 struct ionic_admin_ctx ctx = {
3117 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3119 .opcode = IONIC_CMD_Q_INIT,
3120 .lif_index = cpu_to_le16(lif->index),
3122 .ver = lif->qtype_info[q->type].version,
3123 .index = cpu_to_le32(q->index),
3124 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
3126 .intr_index = cpu_to_le16(lif->adminqcq->intr.index),
3127 .pid = cpu_to_le16(q->pid),
3128 .ring_size = ilog2(q->num_descs),
3129 .ring_base = cpu_to_le64(q->base_pa),
3133 dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
3134 dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
3135 dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
3136 dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
3138 err = ionic_adminq_post_wait(lif, &ctx);
3143 q->hw_type = ctx.comp.q_init.hw_type;
3144 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
3145 q->dbval = IONIC_DBELL_QID(q->hw_index);
3147 dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
3148 dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
3150 /* preset the callback info */
3151 q->info[0].cb_arg = lif;
3153 qcq->flags |= IONIC_QCQ_F_INITED;
3158 static int ionic_station_set(struct ionic_lif *lif)
3160 struct net_device *netdev = lif->netdev;
3161 struct ionic_admin_ctx ctx = {
3162 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3163 .cmd.lif_getattr = {
3164 .opcode = IONIC_CMD_LIF_GETATTR,
3165 .index = cpu_to_le16(lif->index),
3166 .attr = IONIC_LIF_ATTR_MAC,
3169 struct sockaddr addr;
3172 err = ionic_adminq_post_wait(lif, &ctx);
3175 netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
3176 ctx.comp.lif_getattr.mac);
3177 if (is_zero_ether_addr(ctx.comp.lif_getattr.mac))
3180 if (!is_zero_ether_addr(netdev->dev_addr)) {
3181 /* If the netdev mac is non-zero and doesn't match the default
3182 * device address, it was set by something earlier and we're
3183 * likely here again after a fw-upgrade reset. We need to be
3184 * sure the netdev mac is in our filter list.
3186 if (!ether_addr_equal(ctx.comp.lif_getattr.mac,
3188 ionic_lif_addr(lif, netdev->dev_addr, ADD_ADDR, CAN_SLEEP);
3190 /* Update the netdev mac with the device's mac */
3191 memcpy(addr.sa_data, ctx.comp.lif_getattr.mac, netdev->addr_len);
3192 addr.sa_family = AF_INET;
3193 err = eth_prepare_mac_addr_change(netdev, &addr);
3195 netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
3200 eth_commit_mac_addr_change(netdev, &addr);
3203 netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
3205 ionic_lif_addr(lif, netdev->dev_addr, ADD_ADDR, CAN_SLEEP);
3210 int ionic_lif_init(struct ionic_lif *lif)
3212 struct ionic_dev *idev = &lif->ionic->idev;
3213 struct device *dev = lif->ionic->dev;
3214 struct ionic_lif_init_comp comp;
3218 mutex_lock(&lif->ionic->dev_cmd_lock);
3219 ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
3220 err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3221 ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3222 mutex_unlock(&lif->ionic->dev_cmd_lock);
3226 lif->hw_index = le16_to_cpu(comp.hw_index);
3227 mutex_init(&lif->queue_lock);
3229 /* now that we have the hw_index we can figure out our doorbell page */
3230 lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
3231 if (!lif->dbid_count) {
3232 dev_err(dev, "No doorbell pages, aborting\n");
3236 lif->dbid_inuse = bitmap_alloc(lif->dbid_count, GFP_KERNEL);
3237 if (!lif->dbid_inuse) {
3238 dev_err(dev, "Failed alloc doorbell id bitmap, aborting\n");
3242 /* first doorbell id reserved for kernel (dbid aka pid == zero) */
3243 set_bit(0, lif->dbid_inuse);
3246 dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
3247 lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
3248 if (!lif->kern_dbpage) {
3249 dev_err(dev, "Cannot map dbpage, aborting\n");
3251 goto err_out_free_dbid;
3254 err = ionic_lif_adminq_init(lif);
3256 goto err_out_adminq_deinit;
3258 if (lif->ionic->nnqs_per_lif) {
3259 err = ionic_lif_notifyq_init(lif);
3261 goto err_out_notifyq_deinit;
3264 err = ionic_init_nic_features(lif);
3266 goto err_out_notifyq_deinit;
3268 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3269 err = ionic_rx_filters_init(lif);
3271 goto err_out_notifyq_deinit;
3274 err = ionic_station_set(lif);
3276 goto err_out_notifyq_deinit;
3278 lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
3280 set_bit(IONIC_LIF_F_INITED, lif->state);
3282 INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
3286 err_out_notifyq_deinit:
3287 ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3288 err_out_adminq_deinit:
3289 ionic_lif_qcq_deinit(lif, lif->adminqcq);
3290 ionic_lif_reset(lif);
3291 ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3292 lif->kern_dbpage = NULL;
3294 kfree(lif->dbid_inuse);
3295 lif->dbid_inuse = NULL;
3300 static void ionic_lif_notify_work(struct work_struct *ws)
3304 static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
3306 struct ionic_admin_ctx ctx = {
3307 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3308 .cmd.lif_setattr = {
3309 .opcode = IONIC_CMD_LIF_SETATTR,
3310 .index = cpu_to_le16(lif->index),
3311 .attr = IONIC_LIF_ATTR_NAME,
3315 strlcpy(ctx.cmd.lif_setattr.name, lif->netdev->name,
3316 sizeof(ctx.cmd.lif_setattr.name));
3318 ionic_adminq_post_wait(lif, &ctx);
3321 static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
3323 if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
3326 return netdev_priv(netdev);
3329 static int ionic_lif_notify(struct notifier_block *nb,
3330 unsigned long event, void *info)
3332 struct net_device *ndev = netdev_notifier_info_to_dev(info);
3333 struct ionic *ionic = container_of(nb, struct ionic, nb);
3334 struct ionic_lif *lif = ionic_netdev_lif(ndev);
3336 if (!lif || lif->ionic != ionic)
3340 case NETDEV_CHANGENAME:
3341 ionic_lif_set_netdev_info(lif);
3348 int ionic_lif_register(struct ionic_lif *lif)
3352 ionic_lif_register_phc(lif);
3354 INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
3356 lif->ionic->nb.notifier_call = ionic_lif_notify;
3358 err = register_netdevice_notifier(&lif->ionic->nb);
3360 lif->ionic->nb.notifier_call = NULL;
3362 /* only register LIF0 for now */
3363 err = register_netdev(lif->netdev);
3365 dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
3366 ionic_lif_unregister_phc(lif);
3370 ionic_link_status_check_request(lif, CAN_SLEEP);
3371 lif->registered = true;
3372 ionic_lif_set_netdev_info(lif);
3377 void ionic_lif_unregister(struct ionic_lif *lif)
3379 if (lif->ionic->nb.notifier_call) {
3380 unregister_netdevice_notifier(&lif->ionic->nb);
3381 cancel_work_sync(&lif->ionic->nb_work);
3382 lif->ionic->nb.notifier_call = NULL;
3385 if (lif->netdev->reg_state == NETREG_REGISTERED)
3386 unregister_netdev(lif->netdev);
3388 ionic_lif_unregister_phc(lif);
3390 lif->registered = false;
3393 static void ionic_lif_queue_identify(struct ionic_lif *lif)
3395 union ionic_q_identity __iomem *q_ident;
3396 struct ionic *ionic = lif->ionic;
3397 struct ionic_dev *idev;
3401 idev = &lif->ionic->idev;
3402 q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
3404 for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
3405 struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
3407 /* filter out the ones we know about */
3409 case IONIC_QTYPE_ADMINQ:
3410 case IONIC_QTYPE_NOTIFYQ:
3411 case IONIC_QTYPE_RXQ:
3412 case IONIC_QTYPE_TXQ:
3418 memset(qti, 0, sizeof(*qti));
3420 mutex_lock(&ionic->dev_cmd_lock);
3421 ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
3422 ionic_qtype_versions[qtype]);
3423 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3425 qti->version = readb(&q_ident->version);
3426 qti->supported = readb(&q_ident->supported);
3427 qti->features = readq(&q_ident->features);
3428 qti->desc_sz = readw(&q_ident->desc_sz);
3429 qti->comp_sz = readw(&q_ident->comp_sz);
3430 qti->sg_desc_sz = readw(&q_ident->sg_desc_sz);
3431 qti->max_sg_elems = readw(&q_ident->max_sg_elems);
3432 qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
3434 mutex_unlock(&ionic->dev_cmd_lock);
3436 if (err == -EINVAL) {
3437 dev_err(ionic->dev, "qtype %d not supported\n", qtype);
3439 } else if (err == -EIO) {
3440 dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
3443 dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
3448 dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
3449 qtype, qti->version);
3450 dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
3451 qtype, qti->supported);
3452 dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
3453 qtype, qti->features);
3454 dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
3455 qtype, qti->desc_sz);
3456 dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
3457 qtype, qti->comp_sz);
3458 dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
3459 qtype, qti->sg_desc_sz);
3460 dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
3461 qtype, qti->max_sg_elems);
3462 dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
3463 qtype, qti->sg_desc_stride);
3467 int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
3468 union ionic_lif_identity *lid)
3470 struct ionic_dev *idev = &ionic->idev;
3474 sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
3476 mutex_lock(&ionic->dev_cmd_lock);
3477 ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
3478 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3479 memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
3480 mutex_unlock(&ionic->dev_cmd_lock);
3484 dev_dbg(ionic->dev, "capabilities 0x%llx\n",
3485 le64_to_cpu(lid->capabilities));
3487 dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
3488 le32_to_cpu(lid->eth.max_ucast_filters));
3489 dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
3490 le32_to_cpu(lid->eth.max_mcast_filters));
3491 dev_dbg(ionic->dev, "eth.features 0x%llx\n",
3492 le64_to_cpu(lid->eth.config.features));
3493 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
3494 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
3495 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
3496 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
3497 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
3498 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
3499 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
3500 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
3501 dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
3502 dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
3503 dev_dbg(ionic->dev, "eth.config.mtu %d\n",
3504 le32_to_cpu(lid->eth.config.mtu));
3509 int ionic_lif_size(struct ionic *ionic)
3511 struct ionic_identity *ident = &ionic->ident;
3512 unsigned int nintrs, dev_nintrs;
3513 union ionic_lif_config *lc;
3514 unsigned int ntxqs_per_lif;
3515 unsigned int nrxqs_per_lif;
3516 unsigned int neqs_per_lif;
3517 unsigned int nnqs_per_lif;
3518 unsigned int nxqs, neqs;
3519 unsigned int min_intrs;
3522 lc = &ident->lif.eth.config;
3523 dev_nintrs = le32_to_cpu(ident->dev.nintrs);
3524 neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
3525 nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
3526 ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
3527 nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
3529 /* reserve last queue id for hardware timestamping */
3530 if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
3531 if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
3532 lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
3539 nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
3540 nxqs = min(nxqs, num_online_cpus());
3541 neqs = min(neqs_per_lif, num_online_cpus());
3545 * 1 for master lif adminq/notifyq
3546 * 1 for each CPU for master lif TxRx queue pairs
3547 * whatever's left is for RDMA queues
3549 nintrs = 1 + nxqs + neqs;
3550 min_intrs = 2; /* adminq + 1 TxRx queue pair */
3552 if (nintrs > dev_nintrs)
3555 err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
3556 if (err < 0 && err != -ENOSPC) {
3557 dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
3563 if (err != nintrs) {
3564 ionic_bus_free_irq_vectors(ionic);
3568 ionic->nnqs_per_lif = nnqs_per_lif;
3569 ionic->neqs_per_lif = neqs;
3570 ionic->ntxqs_per_lif = nxqs;
3571 ionic->nrxqs_per_lif = nxqs;
3572 ionic->nintrs = nintrs;
3574 ionic_debugfs_add_sizes(ionic);
3579 if (nnqs_per_lif > 1) {
3591 dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);