1 /* bnx2x_cmn.c: Broadcom Everest network driver.
3 * Copyright (c) 2007-2013 Broadcom Corporation
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
9 * Maintained by: Ariel Elior <ariel.elior@qlogic.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
20 #include <linux/etherdevice.h>
21 #include <linux/if_vlan.h>
22 #include <linux/interrupt.h>
24 #include <linux/crash_dump.h>
27 #include <net/ip6_checksum.h>
28 #include <net/busy_poll.h>
29 #include <linux/prefetch.h>
30 #include "bnx2x_cmn.h"
31 #include "bnx2x_init.h"
34 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp);
35 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp);
36 static int bnx2x_alloc_fp_mem(struct bnx2x *bp);
37 static int bnx2x_poll(struct napi_struct *napi, int budget);
39 static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
43 /* Add NAPI objects */
44 for_each_rx_queue_cnic(bp, i) {
45 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
46 bnx2x_poll, NAPI_POLL_WEIGHT);
47 napi_hash_add(&bnx2x_fp(bp, i, napi));
51 static void bnx2x_add_all_napi(struct bnx2x *bp)
55 /* Add NAPI objects */
56 for_each_eth_queue(bp, i) {
57 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
58 bnx2x_poll, NAPI_POLL_WEIGHT);
59 napi_hash_add(&bnx2x_fp(bp, i, napi));
63 static int bnx2x_calc_num_queues(struct bnx2x *bp)
65 int nq = bnx2x_num_queues ? : netif_get_num_default_rss_queues();
67 /* Reduce memory usage in kdump environment by using only one queue */
68 if (is_kdump_kernel())
71 nq = clamp(nq, 1, BNX2X_MAX_QUEUES(bp));
76 * bnx2x_move_fp - move content of the fastpath structure.
79 * @from: source FP index
80 * @to: destination FP index
82 * Makes sure the contents of the bp->fp[to].napi is kept
83 * intact. This is done by first copying the napi struct from
84 * the target to the source, and then mem copying the entire
85 * source onto the target. Update txdata pointers and related
88 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
90 struct bnx2x_fastpath *from_fp = &bp->fp[from];
91 struct bnx2x_fastpath *to_fp = &bp->fp[to];
92 struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
93 struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
94 struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
95 struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
96 int old_max_eth_txqs, new_max_eth_txqs;
97 int old_txdata_index = 0, new_txdata_index = 0;
98 struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
100 /* Copy the NAPI object as it has been already initialized */
101 from_fp->napi = to_fp->napi;
103 /* Move bnx2x_fastpath contents */
104 memcpy(to_fp, from_fp, sizeof(*to_fp));
107 /* Retain the tpa_info of the original `to' version as we don't want
108 * 2 FPs to contain the same tpa_info pointer.
110 to_fp->tpa_info = old_tpa_info;
112 /* move sp_objs contents as well, as their indices match fp ones */
113 memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
115 /* move fp_stats contents as well, as their indices match fp ones */
116 memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
118 /* Update txdata pointers in fp and move txdata content accordingly:
119 * Each fp consumes 'max_cos' txdata structures, so the index should be
120 * decremented by max_cos x delta.
123 old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
124 new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
126 if (from == FCOE_IDX(bp)) {
127 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
128 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
131 memcpy(&bp->bnx2x_txq[new_txdata_index],
132 &bp->bnx2x_txq[old_txdata_index],
133 sizeof(struct bnx2x_fp_txdata));
134 to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
138 * bnx2x_fill_fw_str - Fill buffer with FW version string.
141 * @buf: character buffer to fill with the fw name
142 * @buf_len: length of the above buffer
145 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
148 u8 phy_fw_ver[PHY_FW_VER_LEN];
150 phy_fw_ver[0] = '\0';
151 bnx2x_get_ext_phy_fw_version(&bp->link_params,
152 phy_fw_ver, PHY_FW_VER_LEN);
153 strlcpy(buf, bp->fw_ver, buf_len);
154 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
156 (bp->common.bc_ver & 0xff0000) >> 16,
157 (bp->common.bc_ver & 0xff00) >> 8,
158 (bp->common.bc_ver & 0xff),
159 ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
161 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
166 * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
169 * @delta: number of eth queues which were not allocated
171 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
173 int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
175 /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
176 * backward along the array could cause memory to be overridden
178 for (cos = 1; cos < bp->max_cos; cos++) {
179 for (i = 0; i < old_eth_num - delta; i++) {
180 struct bnx2x_fastpath *fp = &bp->fp[i];
181 int new_idx = cos * (old_eth_num - delta) + i;
183 memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
184 sizeof(struct bnx2x_fp_txdata));
185 fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
190 int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
192 /* free skb in the packet ring at pos idx
193 * return idx of last bd freed
195 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
196 u16 idx, unsigned int *pkts_compl,
197 unsigned int *bytes_compl)
199 struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
200 struct eth_tx_start_bd *tx_start_bd;
201 struct eth_tx_bd *tx_data_bd;
202 struct sk_buff *skb = tx_buf->skb;
203 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
205 u16 split_bd_len = 0;
207 /* prefetch skb end pointer to speedup dev_kfree_skb() */
210 DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d buff @(%p)->skb %p\n",
211 txdata->txq_index, idx, tx_buf, skb);
213 tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
215 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
216 #ifdef BNX2X_STOP_ON_ERROR
217 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
218 BNX2X_ERR("BAD nbd!\n");
222 new_cons = nbd + tx_buf->first_bd;
224 /* Get the next bd */
225 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
227 /* Skip a parse bd... */
229 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
231 if (tx_buf->flags & BNX2X_HAS_SECOND_PBD) {
232 /* Skip second parse bd... */
234 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
237 /* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
238 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
239 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
240 split_bd_len = BD_UNMAP_LEN(tx_data_bd);
242 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
246 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
247 BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
253 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
254 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
255 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
257 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
264 (*bytes_compl) += skb->len;
267 dev_kfree_skb_any(skb);
268 tx_buf->first_bd = 0;
274 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
276 struct netdev_queue *txq;
277 u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
278 unsigned int pkts_compl = 0, bytes_compl = 0;
280 #ifdef BNX2X_STOP_ON_ERROR
281 if (unlikely(bp->panic))
285 txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
286 hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
287 sw_cons = txdata->tx_pkt_cons;
289 while (sw_cons != hw_cons) {
292 pkt_cons = TX_BD(sw_cons);
294 DP(NETIF_MSG_TX_DONE,
295 "queue[%d]: hw_cons %u sw_cons %u pkt_cons %u\n",
296 txdata->txq_index, hw_cons, sw_cons, pkt_cons);
298 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
299 &pkts_compl, &bytes_compl);
304 netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
306 txdata->tx_pkt_cons = sw_cons;
307 txdata->tx_bd_cons = bd_cons;
309 /* Need to make the tx_bd_cons update visible to start_xmit()
310 * before checking for netif_tx_queue_stopped(). Without the
311 * memory barrier, there is a small possibility that
312 * start_xmit() will miss it and cause the queue to be stopped
314 * On the other hand we need an rmb() here to ensure the proper
315 * ordering of bit testing in the following
316 * netif_tx_queue_stopped(txq) call.
320 if (unlikely(netif_tx_queue_stopped(txq))) {
321 /* Taking tx_lock() is needed to prevent re-enabling the queue
322 * while it's empty. This could have happen if rx_action() gets
323 * suspended in bnx2x_tx_int() after the condition before
324 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
326 * stops the queue->sees fresh tx_bd_cons->releases the queue->
327 * sends some packets consuming the whole queue again->
331 __netif_tx_lock(txq, smp_processor_id());
333 if ((netif_tx_queue_stopped(txq)) &&
334 (bp->state == BNX2X_STATE_OPEN) &&
335 (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
336 netif_tx_wake_queue(txq);
338 __netif_tx_unlock(txq);
343 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
346 u16 last_max = fp->last_max_sge;
348 if (SUB_S16(idx, last_max) > 0)
349 fp->last_max_sge = idx;
352 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
354 struct eth_end_agg_rx_cqe *cqe)
356 struct bnx2x *bp = fp->bp;
357 u16 last_max, last_elem, first_elem;
364 /* First mark all used pages */
365 for (i = 0; i < sge_len; i++)
366 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
367 RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
369 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
370 sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
372 /* Here we assume that the last SGE index is the biggest */
373 prefetch((void *)(fp->sge_mask));
374 bnx2x_update_last_max_sge(fp,
375 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
377 last_max = RX_SGE(fp->last_max_sge);
378 last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
379 first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
381 /* If ring is not full */
382 if (last_elem + 1 != first_elem)
385 /* Now update the prod */
386 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
387 if (likely(fp->sge_mask[i]))
390 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
391 delta += BIT_VEC64_ELEM_SZ;
395 fp->rx_sge_prod += delta;
396 /* clear page-end entries */
397 bnx2x_clear_sge_mask_next_elems(fp);
400 DP(NETIF_MSG_RX_STATUS,
401 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
402 fp->last_max_sge, fp->rx_sge_prod);
405 /* Get Toeplitz hash value in the skb using the value from the
406 * CQE (calculated by HW).
408 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
409 const struct eth_fast_path_rx_cqe *cqe,
410 enum pkt_hash_types *rxhash_type)
412 /* Get Toeplitz hash from CQE */
413 if ((bp->dev->features & NETIF_F_RXHASH) &&
414 (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
415 enum eth_rss_hash_type htype;
417 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
418 *rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) ||
419 (htype == TCP_IPV6_HASH_TYPE)) ?
420 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
422 return le32_to_cpu(cqe->rss_hash_result);
424 *rxhash_type = PKT_HASH_TYPE_NONE;
428 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
430 struct eth_fast_path_rx_cqe *cqe)
432 struct bnx2x *bp = fp->bp;
433 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
434 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
435 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
437 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
438 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
440 /* print error if current state != stop */
441 if (tpa_info->tpa_state != BNX2X_TPA_STOP)
442 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
444 /* Try to map an empty data buffer from the aggregation info */
445 mapping = dma_map_single(&bp->pdev->dev,
446 first_buf->data + NET_SKB_PAD,
447 fp->rx_buf_size, DMA_FROM_DEVICE);
449 * ...if it fails - move the skb from the consumer to the producer
450 * and set the current aggregation state as ERROR to drop it
451 * when TPA_STOP arrives.
454 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
455 /* Move the BD from the consumer to the producer */
456 bnx2x_reuse_rx_data(fp, cons, prod);
457 tpa_info->tpa_state = BNX2X_TPA_ERROR;
461 /* move empty data from pool to prod */
462 prod_rx_buf->data = first_buf->data;
463 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
464 /* point prod_bd to new data */
465 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
466 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
468 /* move partial skb from cons to pool (don't unmap yet) */
469 *first_buf = *cons_rx_buf;
471 /* mark bin state as START */
472 tpa_info->parsing_flags =
473 le16_to_cpu(cqe->pars_flags.flags);
474 tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
475 tpa_info->tpa_state = BNX2X_TPA_START;
476 tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
477 tpa_info->placement_offset = cqe->placement_offset;
478 tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type);
479 if (fp->mode == TPA_MODE_GRO) {
480 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
481 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
482 tpa_info->gro_size = gro_size;
485 #ifdef BNX2X_STOP_ON_ERROR
486 fp->tpa_queue_used |= (1 << queue);
487 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
492 /* Timestamp option length allowed for TPA aggregation:
494 * nop nop kind length echo val
496 #define TPA_TSTAMP_OPT_LEN 12
498 * bnx2x_set_gro_params - compute GRO values
501 * @parsing_flags: parsing flags from the START CQE
502 * @len_on_bd: total length of the first packet for the
504 * @pkt_len: length of all segments
506 * Approximate value of the MSS for this aggregation calculated using
507 * the first packet of it.
508 * Compute number of aggregated segments, and gso_type.
510 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
511 u16 len_on_bd, unsigned int pkt_len,
512 u16 num_of_coalesced_segs)
514 /* TPA aggregation won't have either IP options or TCP options
515 * other than timestamp or IPv6 extension headers.
517 u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
519 if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
520 PRS_FLAG_OVERETH_IPV6) {
521 hdrs_len += sizeof(struct ipv6hdr);
522 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
524 hdrs_len += sizeof(struct iphdr);
525 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
528 /* Check if there was a TCP timestamp, if there is it's will
529 * always be 12 bytes length: nop nop kind length echo val.
531 * Otherwise FW would close the aggregation.
533 if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
534 hdrs_len += TPA_TSTAMP_OPT_LEN;
536 skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
538 /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
539 * to skb_shinfo(skb)->gso_segs
541 NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
544 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
545 u16 index, gfp_t gfp_mask)
547 struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
548 struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
549 struct bnx2x_alloc_pool *pool = &fp->page_pool;
552 if (!pool->page || (PAGE_SIZE - pool->offset) < SGE_PAGE_SIZE) {
554 /* put page reference used by the memory pool, since we
555 * won't be using this page as the mempool anymore.
558 put_page(pool->page);
560 pool->page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
561 if (unlikely(!pool->page)) {
562 BNX2X_ERR("Can't alloc sge\n");
566 pool->dma = dma_map_page(&bp->pdev->dev, pool->page, 0,
567 PAGE_SIZE, DMA_FROM_DEVICE);
568 if (unlikely(dma_mapping_error(&bp->pdev->dev,
570 __free_pages(pool->page, PAGES_PER_SGE_SHIFT);
572 BNX2X_ERR("Can't map sge\n");
578 get_page(pool->page);
579 sw_buf->page = pool->page;
580 sw_buf->offset = pool->offset;
582 mapping = pool->dma + sw_buf->offset;
583 dma_unmap_addr_set(sw_buf, mapping, mapping);
585 sge->addr_hi = cpu_to_le32(U64_HI(mapping));
586 sge->addr_lo = cpu_to_le32(U64_LO(mapping));
588 pool->offset += SGE_PAGE_SIZE;
593 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
594 struct bnx2x_agg_info *tpa_info,
597 struct eth_end_agg_rx_cqe *cqe,
600 struct sw_rx_page *rx_pg, old_rx_pg;
601 u32 i, frag_len, frag_size;
602 int err, j, frag_id = 0;
603 u16 len_on_bd = tpa_info->len_on_bd;
604 u16 full_page = 0, gro_size = 0;
606 frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
608 if (fp->mode == TPA_MODE_GRO) {
609 gro_size = tpa_info->gro_size;
610 full_page = tpa_info->full_page;
613 /* This is needed in order to enable forwarding support */
615 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
616 le16_to_cpu(cqe->pkt_len),
617 le16_to_cpu(cqe->num_of_coalesced_segs));
619 #ifdef BNX2X_STOP_ON_ERROR
620 if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
621 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
623 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
629 /* Run through the SGL and compose the fragmented skb */
630 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
631 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
633 /* FW gives the indices of the SGE as if the ring is an array
634 (meaning that "next" element will consume 2 indices) */
635 if (fp->mode == TPA_MODE_GRO)
636 frag_len = min_t(u32, frag_size, (u32)full_page);
638 frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
640 rx_pg = &fp->rx_page_ring[sge_idx];
643 /* If we fail to allocate a substitute page, we simply stop
644 where we are and drop the whole packet */
645 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
647 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
651 dma_unmap_single(&bp->pdev->dev,
652 dma_unmap_addr(&old_rx_pg, mapping),
653 SGE_PAGE_SIZE, DMA_FROM_DEVICE);
654 /* Add one frag and update the appropriate fields in the skb */
655 if (fp->mode == TPA_MODE_LRO)
656 skb_fill_page_desc(skb, j, old_rx_pg.page,
657 old_rx_pg.offset, frag_len);
661 for (rem = frag_len; rem > 0; rem -= gro_size) {
662 int len = rem > gro_size ? gro_size : rem;
663 skb_fill_page_desc(skb, frag_id++,
665 old_rx_pg.offset + offset,
668 get_page(old_rx_pg.page);
673 skb->data_len += frag_len;
674 skb->truesize += SGE_PAGES;
675 skb->len += frag_len;
677 frag_size -= frag_len;
683 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
685 if (fp->rx_frag_size)
691 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
693 if (fp->rx_frag_size) {
694 /* GFP_KERNEL allocations are used only during initialization */
695 if (unlikely(gfp_mask & __GFP_WAIT))
696 return (void *)__get_free_page(gfp_mask);
698 return netdev_alloc_frag(fp->rx_frag_size);
701 return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
705 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
707 const struct iphdr *iph = ip_hdr(skb);
710 skb_set_transport_header(skb, sizeof(struct iphdr));
713 th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
714 iph->saddr, iph->daddr, 0);
717 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
719 struct ipv6hdr *iph = ipv6_hdr(skb);
722 skb_set_transport_header(skb, sizeof(struct ipv6hdr));
725 th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
726 &iph->saddr, &iph->daddr, 0);
729 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
730 void (*gro_func)(struct bnx2x*, struct sk_buff*))
732 skb_set_network_header(skb, 0);
734 tcp_gro_complete(skb);
738 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
742 if (skb_shinfo(skb)->gso_size) {
743 switch (be16_to_cpu(skb->protocol)) {
745 bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
748 bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
751 BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
752 be16_to_cpu(skb->protocol));
756 skb_record_rx_queue(skb, fp->rx_queue);
757 napi_gro_receive(&fp->napi, skb);
760 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
761 struct bnx2x_agg_info *tpa_info,
763 struct eth_end_agg_rx_cqe *cqe,
766 struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
767 u8 pad = tpa_info->placement_offset;
768 u16 len = tpa_info->len_on_bd;
769 struct sk_buff *skb = NULL;
770 u8 *new_data, *data = rx_buf->data;
771 u8 old_tpa_state = tpa_info->tpa_state;
773 tpa_info->tpa_state = BNX2X_TPA_STOP;
775 /* If we there was an error during the handling of the TPA_START -
776 * drop this aggregation.
778 if (old_tpa_state == BNX2X_TPA_ERROR)
781 /* Try to allocate the new data */
782 new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
783 /* Unmap skb in the pool anyway, as we are going to change
784 pool entry status to BNX2X_TPA_STOP even if new skb allocation
786 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
787 fp->rx_buf_size, DMA_FROM_DEVICE);
788 if (likely(new_data))
789 skb = build_skb(data, fp->rx_frag_size);
792 #ifdef BNX2X_STOP_ON_ERROR
793 if (pad + len > fp->rx_buf_size) {
794 BNX2X_ERR("skb_put is about to fail... pad %d len %d rx_buf_size %d\n",
795 pad, len, fp->rx_buf_size);
801 skb_reserve(skb, pad + NET_SKB_PAD);
803 skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type);
805 skb->protocol = eth_type_trans(skb, bp->dev);
806 skb->ip_summed = CHECKSUM_UNNECESSARY;
808 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
809 skb, cqe, cqe_idx)) {
810 if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
811 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
812 bnx2x_gro_receive(bp, fp, skb);
814 DP(NETIF_MSG_RX_STATUS,
815 "Failed to allocate new pages - dropping packet!\n");
816 dev_kfree_skb_any(skb);
819 /* put new data in bin */
820 rx_buf->data = new_data;
825 bnx2x_frag_free(fp, new_data);
827 /* drop the packet and keep the buffer in the bin */
828 DP(NETIF_MSG_RX_STATUS,
829 "Failed to allocate or map a new skb - dropping packet!\n");
830 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
833 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
834 u16 index, gfp_t gfp_mask)
837 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
838 struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
841 data = bnx2x_frag_alloc(fp, gfp_mask);
842 if (unlikely(data == NULL))
845 mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
848 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
849 bnx2x_frag_free(fp, data);
850 BNX2X_ERR("Can't map rx data\n");
855 dma_unmap_addr_set(rx_buf, mapping, mapping);
857 rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
858 rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
864 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
865 struct bnx2x_fastpath *fp,
866 struct bnx2x_eth_q_stats *qstats)
868 /* Do nothing if no L4 csum validation was done.
869 * We do not check whether IP csum was validated. For IPv4 we assume
870 * that if the card got as far as validating the L4 csum, it also
871 * validated the IP csum. IPv6 has no IP csum.
873 if (cqe->fast_path_cqe.status_flags &
874 ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
877 /* If L4 validation was done, check if an error was found. */
879 if (cqe->fast_path_cqe.type_error_flags &
880 (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
881 ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
882 qstats->hw_csum_err++;
884 skb->ip_summed = CHECKSUM_UNNECESSARY;
887 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
889 struct bnx2x *bp = fp->bp;
890 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
891 u16 sw_comp_cons, sw_comp_prod;
893 union eth_rx_cqe *cqe;
894 struct eth_fast_path_rx_cqe *cqe_fp;
896 #ifdef BNX2X_STOP_ON_ERROR
897 if (unlikely(bp->panic))
903 bd_cons = fp->rx_bd_cons;
904 bd_prod = fp->rx_bd_prod;
905 bd_prod_fw = bd_prod;
906 sw_comp_cons = fp->rx_comp_cons;
907 sw_comp_prod = fp->rx_comp_prod;
909 comp_ring_cons = RCQ_BD(sw_comp_cons);
910 cqe = &fp->rx_comp_ring[comp_ring_cons];
911 cqe_fp = &cqe->fast_path_cqe;
913 DP(NETIF_MSG_RX_STATUS,
914 "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
916 while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
917 struct sw_rx_bd *rx_buf = NULL;
920 enum eth_rx_cqe_type cqe_fp_type;
924 enum pkt_hash_types rxhash_type;
926 #ifdef BNX2X_STOP_ON_ERROR
927 if (unlikely(bp->panic))
931 bd_prod = RX_BD(bd_prod);
932 bd_cons = RX_BD(bd_cons);
934 /* A rmb() is required to ensure that the CQE is not read
935 * before it is written by the adapter DMA. PCI ordering
936 * rules will make sure the other fields are written before
937 * the marker at the end of struct eth_fast_path_rx_cqe
938 * but without rmb() a weakly ordered processor can process
939 * stale data. Without the barrier TPA state-machine might
940 * enter inconsistent state and kernel stack might be
941 * provided with incorrect packet description - these lead
942 * to various kernel crashed.
946 cqe_fp_flags = cqe_fp->type_error_flags;
947 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
949 DP(NETIF_MSG_RX_STATUS,
950 "CQE type %x err %x status %x queue %x vlan %x len %u\n",
951 CQE_TYPE(cqe_fp_flags),
952 cqe_fp_flags, cqe_fp->status_flags,
953 le32_to_cpu(cqe_fp->rss_hash_result),
954 le16_to_cpu(cqe_fp->vlan_tag),
955 le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
957 /* is this a slowpath msg? */
958 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
959 bnx2x_sp_event(fp, cqe);
963 rx_buf = &fp->rx_buf_ring[bd_cons];
966 if (!CQE_TYPE_FAST(cqe_fp_type)) {
967 struct bnx2x_agg_info *tpa_info;
968 u16 frag_size, pages;
969 #ifdef BNX2X_STOP_ON_ERROR
971 if (fp->mode == TPA_MODE_DISABLED &&
972 (CQE_TYPE_START(cqe_fp_type) ||
973 CQE_TYPE_STOP(cqe_fp_type)))
974 BNX2X_ERR("START/STOP packet while TPA disabled, type %x\n",
975 CQE_TYPE(cqe_fp_type));
978 if (CQE_TYPE_START(cqe_fp_type)) {
979 u16 queue = cqe_fp->queue_index;
980 DP(NETIF_MSG_RX_STATUS,
981 "calling tpa_start on queue %d\n",
984 bnx2x_tpa_start(fp, queue,
990 queue = cqe->end_agg_cqe.queue_index;
991 tpa_info = &fp->tpa_info[queue];
992 DP(NETIF_MSG_RX_STATUS,
993 "calling tpa_stop on queue %d\n",
996 frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
999 if (fp->mode == TPA_MODE_GRO)
1000 pages = (frag_size + tpa_info->full_page - 1) /
1001 tpa_info->full_page;
1003 pages = SGE_PAGE_ALIGN(frag_size) >>
1006 bnx2x_tpa_stop(bp, fp, tpa_info, pages,
1007 &cqe->end_agg_cqe, comp_ring_cons);
1008 #ifdef BNX2X_STOP_ON_ERROR
1013 bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
1017 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
1018 pad = cqe_fp->placement_offset;
1019 dma_sync_single_for_cpu(&bp->pdev->dev,
1020 dma_unmap_addr(rx_buf, mapping),
1021 pad + RX_COPY_THRESH,
1024 prefetch(data + pad); /* speedup eth_type_trans() */
1025 /* is this an error packet? */
1026 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
1027 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1028 "ERROR flags %x rx packet %u\n",
1029 cqe_fp_flags, sw_comp_cons);
1030 bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
1034 /* Since we don't have a jumbo ring
1035 * copy small packets if mtu > 1500
1037 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
1038 (len <= RX_COPY_THRESH)) {
1039 skb = napi_alloc_skb(&fp->napi, len);
1041 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1042 "ERROR packet dropped because of alloc failure\n");
1043 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1046 memcpy(skb->data, data + pad, len);
1047 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1049 if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
1050 GFP_ATOMIC) == 0)) {
1051 dma_unmap_single(&bp->pdev->dev,
1052 dma_unmap_addr(rx_buf, mapping),
1055 skb = build_skb(data, fp->rx_frag_size);
1056 if (unlikely(!skb)) {
1057 bnx2x_frag_free(fp, data);
1058 bnx2x_fp_qstats(bp, fp)->
1059 rx_skb_alloc_failed++;
1062 skb_reserve(skb, pad);
1064 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1065 "ERROR packet dropped because of alloc failure\n");
1066 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1068 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1074 skb->protocol = eth_type_trans(skb, bp->dev);
1076 /* Set Toeplitz hash for a none-LRO skb */
1077 rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type);
1078 skb_set_hash(skb, rxhash, rxhash_type);
1080 skb_checksum_none_assert(skb);
1082 if (bp->dev->features & NETIF_F_RXCSUM)
1083 bnx2x_csum_validate(skb, cqe, fp,
1084 bnx2x_fp_qstats(bp, fp));
1086 skb_record_rx_queue(skb, fp->rx_queue);
1088 /* Check if this packet was timestamped */
1089 if (unlikely(cqe->fast_path_cqe.type_error_flags &
1090 (1 << ETH_FAST_PATH_RX_CQE_PTP_PKT_SHIFT)))
1091 bnx2x_set_rx_ts(bp, skb);
1093 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1095 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1096 le16_to_cpu(cqe_fp->vlan_tag));
1098 skb_mark_napi_id(skb, &fp->napi);
1100 if (bnx2x_fp_ll_polling(fp))
1101 netif_receive_skb(skb);
1103 napi_gro_receive(&fp->napi, skb);
1105 rx_buf->data = NULL;
1107 bd_cons = NEXT_RX_IDX(bd_cons);
1108 bd_prod = NEXT_RX_IDX(bd_prod);
1109 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1112 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1113 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1115 /* mark CQE as free */
1116 BNX2X_SEED_CQE(cqe_fp);
1118 if (rx_pkt == budget)
1121 comp_ring_cons = RCQ_BD(sw_comp_cons);
1122 cqe = &fp->rx_comp_ring[comp_ring_cons];
1123 cqe_fp = &cqe->fast_path_cqe;
1126 fp->rx_bd_cons = bd_cons;
1127 fp->rx_bd_prod = bd_prod_fw;
1128 fp->rx_comp_cons = sw_comp_cons;
1129 fp->rx_comp_prod = sw_comp_prod;
1131 /* Update producers */
1132 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1135 fp->rx_pkt += rx_pkt;
1141 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1143 struct bnx2x_fastpath *fp = fp_cookie;
1144 struct bnx2x *bp = fp->bp;
1148 "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1149 fp->index, fp->fw_sb_id, fp->igu_sb_id);
1151 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1153 #ifdef BNX2X_STOP_ON_ERROR
1154 if (unlikely(bp->panic))
1158 /* Handle Rx and Tx according to MSI-X vector */
1159 for_each_cos_in_tx_queue(fp, cos)
1160 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1162 prefetch(&fp->sb_running_index[SM_RX_ID]);
1163 napi_schedule_irqoff(&bnx2x_fp(bp, fp->index, napi));
1168 /* HW Lock for shared dual port PHYs */
1169 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1171 mutex_lock(&bp->port.phy_mutex);
1173 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1176 void bnx2x_release_phy_lock(struct bnx2x *bp)
1178 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1180 mutex_unlock(&bp->port.phy_mutex);
1183 /* calculates MF speed according to current linespeed and MF configuration */
1184 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1186 u16 line_speed = bp->link_vars.line_speed;
1188 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1189 bp->mf_config[BP_VN(bp)]);
1191 /* Calculate the current MAX line speed limit for the MF
1195 line_speed = (line_speed * maxCfg) / 100;
1196 else { /* SD mode */
1197 u16 vn_max_rate = maxCfg * 100;
1199 if (vn_max_rate < line_speed)
1200 line_speed = vn_max_rate;
1208 * bnx2x_fill_report_data - fill link report data to report
1210 * @bp: driver handle
1211 * @data: link state to update
1213 * It uses a none-atomic bit operations because is called under the mutex.
1215 static void bnx2x_fill_report_data(struct bnx2x *bp,
1216 struct bnx2x_link_report_data *data)
1218 memset(data, 0, sizeof(*data));
1221 /* Fill the report data: effective line speed */
1222 data->line_speed = bnx2x_get_mf_speed(bp);
1225 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1226 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1227 &data->link_report_flags);
1229 if (!BNX2X_NUM_ETH_QUEUES(bp))
1230 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1231 &data->link_report_flags);
1234 if (bp->link_vars.duplex == DUPLEX_FULL)
1235 __set_bit(BNX2X_LINK_REPORT_FD,
1236 &data->link_report_flags);
1238 /* Rx Flow Control is ON */
1239 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1240 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1241 &data->link_report_flags);
1243 /* Tx Flow Control is ON */
1244 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1245 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1246 &data->link_report_flags);
1248 *data = bp->vf_link_vars;
1253 * bnx2x_link_report - report link status to OS.
1255 * @bp: driver handle
1257 * Calls the __bnx2x_link_report() under the same locking scheme
1258 * as a link/PHY state managing code to ensure a consistent link
1262 void bnx2x_link_report(struct bnx2x *bp)
1264 bnx2x_acquire_phy_lock(bp);
1265 __bnx2x_link_report(bp);
1266 bnx2x_release_phy_lock(bp);
1270 * __bnx2x_link_report - report link status to OS.
1272 * @bp: driver handle
1274 * None atomic implementation.
1275 * Should be called under the phy_lock.
1277 void __bnx2x_link_report(struct bnx2x *bp)
1279 struct bnx2x_link_report_data cur_data;
1282 if (IS_PF(bp) && !CHIP_IS_E1(bp))
1283 bnx2x_read_mf_cfg(bp);
1285 /* Read the current link report info */
1286 bnx2x_fill_report_data(bp, &cur_data);
1288 /* Don't report link down or exactly the same link status twice */
1289 if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1290 (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1291 &bp->last_reported_link.link_report_flags) &&
1292 test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1293 &cur_data.link_report_flags)))
1298 /* We are going to report a new link parameters now -
1299 * remember the current data for the next time.
1301 memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1303 /* propagate status to VFs */
1305 bnx2x_iov_link_update(bp);
1307 if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1308 &cur_data.link_report_flags)) {
1309 netif_carrier_off(bp->dev);
1310 netdev_err(bp->dev, "NIC Link is Down\n");
1316 netif_carrier_on(bp->dev);
1318 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1319 &cur_data.link_report_flags))
1324 /* Handle the FC at the end so that only these flags would be
1325 * possibly set. This way we may easily check if there is no FC
1328 if (cur_data.link_report_flags) {
1329 if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1330 &cur_data.link_report_flags)) {
1331 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1332 &cur_data.link_report_flags))
1333 flow = "ON - receive & transmit";
1335 flow = "ON - receive";
1337 flow = "ON - transmit";
1342 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1343 cur_data.line_speed, duplex, flow);
1347 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1351 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1352 struct eth_rx_sge *sge;
1354 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1356 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1357 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1360 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1361 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1365 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1366 struct bnx2x_fastpath *fp, int last)
1370 for (i = 0; i < last; i++) {
1371 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1372 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1373 u8 *data = first_buf->data;
1376 DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1379 if (tpa_info->tpa_state == BNX2X_TPA_START)
1380 dma_unmap_single(&bp->pdev->dev,
1381 dma_unmap_addr(first_buf, mapping),
1382 fp->rx_buf_size, DMA_FROM_DEVICE);
1383 bnx2x_frag_free(fp, data);
1384 first_buf->data = NULL;
1388 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1392 for_each_rx_queue_cnic(bp, j) {
1393 struct bnx2x_fastpath *fp = &bp->fp[j];
1397 /* Activate BD ring */
1399 * this will generate an interrupt (to the TSTORM)
1400 * must only be done after chip is initialized
1402 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1407 void bnx2x_init_rx_rings(struct bnx2x *bp)
1409 int func = BP_FUNC(bp);
1413 /* Allocate TPA resources */
1414 for_each_eth_queue(bp, j) {
1415 struct bnx2x_fastpath *fp = &bp->fp[j];
1418 "mtu %d rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1420 if (fp->mode != TPA_MODE_DISABLED) {
1421 /* Fill the per-aggregation pool */
1422 for (i = 0; i < MAX_AGG_QS(bp); i++) {
1423 struct bnx2x_agg_info *tpa_info =
1425 struct sw_rx_bd *first_buf =
1426 &tpa_info->first_buf;
1429 bnx2x_frag_alloc(fp, GFP_KERNEL);
1430 if (!first_buf->data) {
1431 BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1433 bnx2x_free_tpa_pool(bp, fp, i);
1434 fp->mode = TPA_MODE_DISABLED;
1437 dma_unmap_addr_set(first_buf, mapping, 0);
1438 tpa_info->tpa_state = BNX2X_TPA_STOP;
1441 /* "next page" elements initialization */
1442 bnx2x_set_next_page_sgl(fp);
1444 /* set SGEs bit mask */
1445 bnx2x_init_sge_ring_bit_mask(fp);
1447 /* Allocate SGEs and initialize the ring elements */
1448 for (i = 0, ring_prod = 0;
1449 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1451 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1453 BNX2X_ERR("was only able to allocate %d rx sges\n",
1455 BNX2X_ERR("disabling TPA for queue[%d]\n",
1457 /* Cleanup already allocated elements */
1458 bnx2x_free_rx_sge_range(bp, fp,
1460 bnx2x_free_tpa_pool(bp, fp,
1462 fp->mode = TPA_MODE_DISABLED;
1466 ring_prod = NEXT_SGE_IDX(ring_prod);
1469 fp->rx_sge_prod = ring_prod;
1473 for_each_eth_queue(bp, j) {
1474 struct bnx2x_fastpath *fp = &bp->fp[j];
1478 /* Activate BD ring */
1480 * this will generate an interrupt (to the TSTORM)
1481 * must only be done after chip is initialized
1483 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1489 if (CHIP_IS_E1(bp)) {
1490 REG_WR(bp, BAR_USTRORM_INTMEM +
1491 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1492 U64_LO(fp->rx_comp_mapping));
1493 REG_WR(bp, BAR_USTRORM_INTMEM +
1494 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1495 U64_HI(fp->rx_comp_mapping));
1500 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1503 struct bnx2x *bp = fp->bp;
1505 for_each_cos_in_tx_queue(fp, cos) {
1506 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1507 unsigned pkts_compl = 0, bytes_compl = 0;
1509 u16 sw_prod = txdata->tx_pkt_prod;
1510 u16 sw_cons = txdata->tx_pkt_cons;
1512 while (sw_cons != sw_prod) {
1513 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1514 &pkts_compl, &bytes_compl);
1518 netdev_tx_reset_queue(
1519 netdev_get_tx_queue(bp->dev,
1520 txdata->txq_index));
1524 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1528 for_each_tx_queue_cnic(bp, i) {
1529 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1533 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1537 for_each_eth_queue(bp, i) {
1538 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1542 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1544 struct bnx2x *bp = fp->bp;
1547 /* ring wasn't allocated */
1548 if (fp->rx_buf_ring == NULL)
1551 for (i = 0; i < NUM_RX_BD; i++) {
1552 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1553 u8 *data = rx_buf->data;
1557 dma_unmap_single(&bp->pdev->dev,
1558 dma_unmap_addr(rx_buf, mapping),
1559 fp->rx_buf_size, DMA_FROM_DEVICE);
1561 rx_buf->data = NULL;
1562 bnx2x_frag_free(fp, data);
1566 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1570 for_each_rx_queue_cnic(bp, j) {
1571 bnx2x_free_rx_bds(&bp->fp[j]);
1575 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1579 for_each_eth_queue(bp, j) {
1580 struct bnx2x_fastpath *fp = &bp->fp[j];
1582 bnx2x_free_rx_bds(fp);
1584 if (fp->mode != TPA_MODE_DISABLED)
1585 bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1589 static void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1591 bnx2x_free_tx_skbs_cnic(bp);
1592 bnx2x_free_rx_skbs_cnic(bp);
1595 void bnx2x_free_skbs(struct bnx2x *bp)
1597 bnx2x_free_tx_skbs(bp);
1598 bnx2x_free_rx_skbs(bp);
1601 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1603 /* load old values */
1604 u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1606 if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1607 /* leave all but MAX value */
1608 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1610 /* set new MAX value */
1611 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1612 & FUNC_MF_CFG_MAX_BW_MASK;
1614 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1619 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1621 * @bp: driver handle
1622 * @nvecs: number of vectors to be released
1624 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1628 if (nvecs == offset)
1631 /* VFs don't have a default SB */
1633 free_irq(bp->msix_table[offset].vector, bp->dev);
1634 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1635 bp->msix_table[offset].vector);
1639 if (CNIC_SUPPORT(bp)) {
1640 if (nvecs == offset)
1645 for_each_eth_queue(bp, i) {
1646 if (nvecs == offset)
1648 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1649 i, bp->msix_table[offset].vector);
1651 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1655 void bnx2x_free_irq(struct bnx2x *bp)
1657 if (bp->flags & USING_MSIX_FLAG &&
1658 !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1659 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1661 /* vfs don't have a default status block */
1665 bnx2x_free_msix_irqs(bp, nvecs);
1667 free_irq(bp->dev->irq, bp->dev);
1671 int bnx2x_enable_msix(struct bnx2x *bp)
1673 int msix_vec = 0, i, rc;
1675 /* VFs don't have a default status block */
1677 bp->msix_table[msix_vec].entry = msix_vec;
1678 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1679 bp->msix_table[0].entry);
1683 /* Cnic requires an msix vector for itself */
1684 if (CNIC_SUPPORT(bp)) {
1685 bp->msix_table[msix_vec].entry = msix_vec;
1686 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1687 msix_vec, bp->msix_table[msix_vec].entry);
1691 /* We need separate vectors for ETH queues only (not FCoE) */
1692 for_each_eth_queue(bp, i) {
1693 bp->msix_table[msix_vec].entry = msix_vec;
1694 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1695 msix_vec, msix_vec, i);
1699 DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1702 rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0],
1703 BNX2X_MIN_MSIX_VEC_CNT(bp), msix_vec);
1705 * reconfigure number of tx/rx queues according to available
1708 if (rc == -ENOSPC) {
1709 /* Get by with single vector */
1710 rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1, 1);
1712 BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1717 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1718 bp->flags |= USING_SINGLE_MSIX_FLAG;
1720 BNX2X_DEV_INFO("set number of queues to 1\n");
1721 bp->num_ethernet_queues = 1;
1722 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1723 } else if (rc < 0) {
1724 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1726 } else if (rc < msix_vec) {
1727 /* how less vectors we will have? */
1728 int diff = msix_vec - rc;
1730 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1733 * decrease number of queues by number of unallocated entries
1735 bp->num_ethernet_queues -= diff;
1736 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1738 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1742 bp->flags |= USING_MSIX_FLAG;
1747 /* fall to INTx if not enough memory */
1749 bp->flags |= DISABLE_MSI_FLAG;
1754 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1756 int i, rc, offset = 0;
1758 /* no default status block for vf */
1760 rc = request_irq(bp->msix_table[offset++].vector,
1761 bnx2x_msix_sp_int, 0,
1762 bp->dev->name, bp->dev);
1764 BNX2X_ERR("request sp irq failed\n");
1769 if (CNIC_SUPPORT(bp))
1772 for_each_eth_queue(bp, i) {
1773 struct bnx2x_fastpath *fp = &bp->fp[i];
1774 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1777 rc = request_irq(bp->msix_table[offset].vector,
1778 bnx2x_msix_fp_int, 0, fp->name, fp);
1780 BNX2X_ERR("request fp #%d irq (%d) failed rc %d\n", i,
1781 bp->msix_table[offset].vector, rc);
1782 bnx2x_free_msix_irqs(bp, offset);
1789 i = BNX2X_NUM_ETH_QUEUES(bp);
1791 offset = 1 + CNIC_SUPPORT(bp);
1792 netdev_info(bp->dev,
1793 "using MSI-X IRQs: sp %d fp[%d] %d ... fp[%d] %d\n",
1794 bp->msix_table[0].vector,
1795 0, bp->msix_table[offset].vector,
1796 i - 1, bp->msix_table[offset + i - 1].vector);
1798 offset = CNIC_SUPPORT(bp);
1799 netdev_info(bp->dev,
1800 "using MSI-X IRQs: fp[%d] %d ... fp[%d] %d\n",
1801 0, bp->msix_table[offset].vector,
1802 i - 1, bp->msix_table[offset + i - 1].vector);
1807 int bnx2x_enable_msi(struct bnx2x *bp)
1811 rc = pci_enable_msi(bp->pdev);
1813 BNX2X_DEV_INFO("MSI is not attainable\n");
1816 bp->flags |= USING_MSI_FLAG;
1821 static int bnx2x_req_irq(struct bnx2x *bp)
1823 unsigned long flags;
1826 if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1829 flags = IRQF_SHARED;
1831 if (bp->flags & USING_MSIX_FLAG)
1832 irq = bp->msix_table[0].vector;
1834 irq = bp->pdev->irq;
1836 return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1839 static int bnx2x_setup_irqs(struct bnx2x *bp)
1842 if (bp->flags & USING_MSIX_FLAG &&
1843 !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1844 rc = bnx2x_req_msix_irqs(bp);
1848 rc = bnx2x_req_irq(bp);
1850 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1853 if (bp->flags & USING_MSI_FLAG) {
1854 bp->dev->irq = bp->pdev->irq;
1855 netdev_info(bp->dev, "using MSI IRQ %d\n",
1858 if (bp->flags & USING_MSIX_FLAG) {
1859 bp->dev->irq = bp->msix_table[0].vector;
1860 netdev_info(bp->dev, "using MSIX IRQ %d\n",
1868 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1872 for_each_rx_queue_cnic(bp, i) {
1873 bnx2x_fp_busy_poll_init(&bp->fp[i]);
1874 napi_enable(&bnx2x_fp(bp, i, napi));
1878 static void bnx2x_napi_enable(struct bnx2x *bp)
1882 for_each_eth_queue(bp, i) {
1883 bnx2x_fp_busy_poll_init(&bp->fp[i]);
1884 napi_enable(&bnx2x_fp(bp, i, napi));
1888 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1892 for_each_rx_queue_cnic(bp, i) {
1893 napi_disable(&bnx2x_fp(bp, i, napi));
1894 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1895 usleep_range(1000, 2000);
1899 static void bnx2x_napi_disable(struct bnx2x *bp)
1903 for_each_eth_queue(bp, i) {
1904 napi_disable(&bnx2x_fp(bp, i, napi));
1905 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1906 usleep_range(1000, 2000);
1910 void bnx2x_netif_start(struct bnx2x *bp)
1912 if (netif_running(bp->dev)) {
1913 bnx2x_napi_enable(bp);
1914 if (CNIC_LOADED(bp))
1915 bnx2x_napi_enable_cnic(bp);
1916 bnx2x_int_enable(bp);
1917 if (bp->state == BNX2X_STATE_OPEN)
1918 netif_tx_wake_all_queues(bp->dev);
1922 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1924 bnx2x_int_disable_sync(bp, disable_hw);
1925 bnx2x_napi_disable(bp);
1926 if (CNIC_LOADED(bp))
1927 bnx2x_napi_disable_cnic(bp);
1930 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
1931 void *accel_priv, select_queue_fallback_t fallback)
1933 struct bnx2x *bp = netdev_priv(dev);
1935 if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1936 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1937 u16 ether_type = ntohs(hdr->h_proto);
1939 /* Skip VLAN tag if present */
1940 if (ether_type == ETH_P_8021Q) {
1941 struct vlan_ethhdr *vhdr =
1942 (struct vlan_ethhdr *)skb->data;
1944 ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1947 /* If ethertype is FCoE or FIP - use FCoE ring */
1948 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1949 return bnx2x_fcoe_tx(bp, txq_index);
1952 /* select a non-FCoE queue */
1953 return fallback(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
1956 void bnx2x_set_num_queues(struct bnx2x *bp)
1959 bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1961 /* override in STORAGE SD modes */
1962 if (IS_MF_STORAGE_ONLY(bp))
1963 bp->num_ethernet_queues = 1;
1965 /* Add special queues */
1966 bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1967 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1969 BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1973 * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1975 * @bp: Driver handle
1977 * We currently support for at most 16 Tx queues for each CoS thus we will
1978 * allocate a multiple of 16 for ETH L2 rings according to the value of the
1981 * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1982 * index after all ETH L2 indices.
1984 * If the actual number of Tx queues (for each CoS) is less than 16 then there
1985 * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1986 * 16..31,...) with indices that are not coupled with any real Tx queue.
1988 * The proper configuration of skb->queue_mapping is handled by
1989 * bnx2x_select_queue() and __skb_tx_hash().
1991 * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1992 * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1994 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1998 tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1999 rx = BNX2X_NUM_ETH_QUEUES(bp);
2001 /* account for fcoe queue */
2002 if (include_cnic && !NO_FCOE(bp)) {
2007 rc = netif_set_real_num_tx_queues(bp->dev, tx);
2009 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
2012 rc = netif_set_real_num_rx_queues(bp->dev, rx);
2014 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
2018 DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
2024 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
2028 for_each_queue(bp, i) {
2029 struct bnx2x_fastpath *fp = &bp->fp[i];
2032 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
2035 * Although there are no IP frames expected to arrive to
2036 * this ring we still want to add an
2037 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
2040 mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
2043 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
2044 IP_HEADER_ALIGNMENT_PADDING +
2047 BNX2X_FW_RX_ALIGN_END;
2048 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
2049 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
2050 fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
2052 fp->rx_frag_size = 0;
2056 static int bnx2x_init_rss(struct bnx2x *bp)
2059 u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
2061 /* Prepare the initial contents for the indirection table if RSS is
2064 for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
2065 bp->rss_conf_obj.ind_table[i] =
2067 ethtool_rxfh_indir_default(i, num_eth_queues);
2070 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
2071 * per-port, so if explicit configuration is needed , do it only
2074 * For 57712 and newer on the other hand it's a per-function
2077 return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
2080 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
2081 bool config_hash, bool enable)
2083 struct bnx2x_config_rss_params params = {NULL};
2085 /* Although RSS is meaningless when there is a single HW queue we
2086 * still need it enabled in order to have HW Rx hash generated.
2088 * if (!is_eth_multi(bp))
2089 * bp->multi_mode = ETH_RSS_MODE_DISABLED;
2092 params.rss_obj = rss_obj;
2094 __set_bit(RAMROD_COMP_WAIT, ¶ms.ramrod_flags);
2097 __set_bit(BNX2X_RSS_MODE_REGULAR, ¶ms.rss_flags);
2099 /* RSS configuration */
2100 __set_bit(BNX2X_RSS_IPV4, ¶ms.rss_flags);
2101 __set_bit(BNX2X_RSS_IPV4_TCP, ¶ms.rss_flags);
2102 __set_bit(BNX2X_RSS_IPV6, ¶ms.rss_flags);
2103 __set_bit(BNX2X_RSS_IPV6_TCP, ¶ms.rss_flags);
2104 if (rss_obj->udp_rss_v4)
2105 __set_bit(BNX2X_RSS_IPV4_UDP, ¶ms.rss_flags);
2106 if (rss_obj->udp_rss_v6)
2107 __set_bit(BNX2X_RSS_IPV6_UDP, ¶ms.rss_flags);
2109 if (!CHIP_IS_E1x(bp))
2110 /* valid only for TUNN_MODE_GRE tunnel mode */
2111 __set_bit(BNX2X_RSS_GRE_INNER_HDRS, ¶ms.rss_flags);
2113 __set_bit(BNX2X_RSS_MODE_DISABLED, ¶ms.rss_flags);
2117 params.rss_result_mask = MULTI_MASK;
2119 memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2123 netdev_rss_key_fill(params.rss_key, T_ETH_RSS_KEY * 4);
2124 __set_bit(BNX2X_RSS_SET_SRCH, ¶ms.rss_flags);
2128 return bnx2x_config_rss(bp, ¶ms);
2130 return bnx2x_vfpf_config_rss(bp, ¶ms);
2133 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2135 struct bnx2x_func_state_params func_params = {NULL};
2137 /* Prepare parameters for function state transitions */
2138 __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2140 func_params.f_obj = &bp->func_obj;
2141 func_params.cmd = BNX2X_F_CMD_HW_INIT;
2143 func_params.params.hw_init.load_phase = load_code;
2145 return bnx2x_func_state_change(bp, &func_params);
2149 * Cleans the object that have internal lists without sending
2150 * ramrods. Should be run when interrupts are disabled.
2152 void bnx2x_squeeze_objects(struct bnx2x *bp)
2155 unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2156 struct bnx2x_mcast_ramrod_params rparam = {NULL};
2157 struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2159 /***************** Cleanup MACs' object first *************************/
2161 /* Wait for completion of requested */
2162 __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2163 /* Perform a dry cleanup */
2164 __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2166 /* Clean ETH primary MAC */
2167 __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2168 rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2171 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2173 /* Cleanup UC list */
2175 __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2176 rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2179 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2181 /***************** Now clean mcast object *****************************/
2182 rparam.mcast_obj = &bp->mcast_obj;
2183 __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2185 /* Add a DEL command... - Since we're doing a driver cleanup only,
2186 * we take a lock surrounding both the initial send and the CONTs,
2187 * as we don't want a true completion to disrupt us in the middle.
2189 netif_addr_lock_bh(bp->dev);
2190 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2192 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2195 /* ...and wait until all pending commands are cleared */
2196 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2199 BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2201 netif_addr_unlock_bh(bp->dev);
2205 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2207 netif_addr_unlock_bh(bp->dev);
2210 #ifndef BNX2X_STOP_ON_ERROR
2211 #define LOAD_ERROR_EXIT(bp, label) \
2213 (bp)->state = BNX2X_STATE_ERROR; \
2217 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2219 bp->cnic_loaded = false; \
2222 #else /*BNX2X_STOP_ON_ERROR*/
2223 #define LOAD_ERROR_EXIT(bp, label) \
2225 (bp)->state = BNX2X_STATE_ERROR; \
2229 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2231 bp->cnic_loaded = false; \
2235 #endif /*BNX2X_STOP_ON_ERROR*/
2237 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2239 BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2240 bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2244 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2246 int num_groups, vf_headroom = 0;
2247 int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2249 /* number of queues for statistics is number of eth queues + FCoE */
2250 u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2252 /* Total number of FW statistics requests =
2253 * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2254 * and fcoe l2 queue) stats + num of queues (which includes another 1
2255 * for fcoe l2 queue if applicable)
2257 bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2259 /* vf stats appear in the request list, but their data is allocated by
2260 * the VFs themselves. We don't include them in the bp->fw_stats_num as
2261 * it is used to determine where to place the vf stats queries in the
2265 vf_headroom = bnx2x_vf_headroom(bp);
2267 /* Request is built from stats_query_header and an array of
2268 * stats_query_cmd_group each of which contains
2269 * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2270 * configured in the stats_query_header.
2273 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2274 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2277 DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2278 bp->fw_stats_num, vf_headroom, num_groups);
2279 bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2280 num_groups * sizeof(struct stats_query_cmd_group);
2282 /* Data for statistics requests + stats_counter
2283 * stats_counter holds per-STORM counters that are incremented
2284 * when STORM has finished with the current request.
2285 * memory for FCoE offloaded statistics are counted anyway,
2286 * even if they will not be sent.
2287 * VF stats are not accounted for here as the data of VF stats is stored
2288 * in memory allocated by the VF, not here.
2290 bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2291 sizeof(struct per_pf_stats) +
2292 sizeof(struct fcoe_statistics_params) +
2293 sizeof(struct per_queue_stats) * num_queue_stats +
2294 sizeof(struct stats_counter);
2296 bp->fw_stats = BNX2X_PCI_ALLOC(&bp->fw_stats_mapping,
2297 bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2302 bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2303 bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2304 bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2305 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2306 bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2307 bp->fw_stats_req_sz;
2309 DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2310 U64_HI(bp->fw_stats_req_mapping),
2311 U64_LO(bp->fw_stats_req_mapping));
2312 DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2313 U64_HI(bp->fw_stats_data_mapping),
2314 U64_LO(bp->fw_stats_data_mapping));
2318 bnx2x_free_fw_stats_mem(bp);
2319 BNX2X_ERR("Can't allocate FW stats memory\n");
2323 /* send load request to mcp and analyze response */
2324 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2330 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2331 DRV_MSG_SEQ_NUMBER_MASK);
2332 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2334 /* Get current FW pulse sequence */
2335 bp->fw_drv_pulse_wr_seq =
2336 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2337 DRV_PULSE_SEQ_MASK);
2338 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2340 param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2342 if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2343 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2346 (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2348 /* if mcp fails to respond we must abort */
2349 if (!(*load_code)) {
2350 BNX2X_ERR("MCP response failure, aborting\n");
2354 /* If mcp refused (e.g. other port is in diagnostic mode) we
2357 if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2358 BNX2X_ERR("MCP refused load request, aborting\n");
2364 /* check whether another PF has already loaded FW to chip. In
2365 * virtualized environments a pf from another VM may have already
2366 * initialized the device including loading FW
2368 int bnx2x_compare_fw_ver(struct bnx2x *bp, u32 load_code, bool print_err)
2370 /* is another pf loaded on this engine? */
2371 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2372 load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2373 /* build my FW version dword */
2374 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2375 (BCM_5710_FW_MINOR_VERSION << 8) +
2376 (BCM_5710_FW_REVISION_VERSION << 16) +
2377 (BCM_5710_FW_ENGINEERING_VERSION << 24);
2379 /* read loaded FW from chip */
2380 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2382 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2385 /* abort nic load if version mismatch */
2386 if (my_fw != loaded_fw) {
2388 BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2391 BNX2X_DEV_INFO("bnx2x with FW %x was already loaded which mismatches my %x FW, possibly due to MF UNDI\n",
2399 /* returns the "mcp load_code" according to global load_count array */
2400 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2402 int path = BP_PATH(bp);
2404 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d] %d, %d, %d\n",
2405 path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2406 bnx2x_load_count[path][2]);
2407 bnx2x_load_count[path][0]++;
2408 bnx2x_load_count[path][1 + port]++;
2409 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d] %d, %d, %d\n",
2410 path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2411 bnx2x_load_count[path][2]);
2412 if (bnx2x_load_count[path][0] == 1)
2413 return FW_MSG_CODE_DRV_LOAD_COMMON;
2414 else if (bnx2x_load_count[path][1 + port] == 1)
2415 return FW_MSG_CODE_DRV_LOAD_PORT;
2417 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2420 /* mark PMF if applicable */
2421 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2423 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2424 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2425 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2427 /* We need the barrier to ensure the ordering between the
2428 * writing to bp->port.pmf here and reading it from the
2429 * bnx2x_periodic_task().
2436 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2439 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2441 if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2442 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2443 (bp->common.shmem2_base)) {
2444 if (SHMEM2_HAS(bp, dcc_support))
2445 SHMEM2_WR(bp, dcc_support,
2446 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2447 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2448 if (SHMEM2_HAS(bp, afex_driver_support))
2449 SHMEM2_WR(bp, afex_driver_support,
2450 SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2453 /* Set AFEX default VLAN tag to an invalid value */
2454 bp->afex_def_vlan_tag = -1;
2458 * bnx2x_bz_fp - zero content of the fastpath structure.
2460 * @bp: driver handle
2461 * @index: fastpath index to be zeroed
2463 * Makes sure the contents of the bp->fp[index].napi is kept
2466 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2468 struct bnx2x_fastpath *fp = &bp->fp[index];
2470 struct napi_struct orig_napi = fp->napi;
2471 struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2473 /* bzero bnx2x_fastpath contents */
2475 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2476 sizeof(struct bnx2x_agg_info));
2477 memset(fp, 0, sizeof(*fp));
2479 /* Restore the NAPI object as it has been already initialized */
2480 fp->napi = orig_napi;
2481 fp->tpa_info = orig_tpa_info;
2485 fp->max_cos = bp->max_cos;
2487 /* Special queues support only one CoS */
2490 /* Init txdata pointers */
2492 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2494 for_each_cos_in_tx_queue(fp, cos)
2495 fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2496 BNX2X_NUM_ETH_QUEUES(bp) + index];
2498 /* set the tpa flag for each queue. The tpa flag determines the queue
2499 * minimal size so it must be set prior to queue memory allocation
2501 if (bp->dev->features & NETIF_F_LRO)
2502 fp->mode = TPA_MODE_LRO;
2503 else if (bp->dev->features & NETIF_F_GRO &&
2504 bnx2x_mtu_allows_gro(bp->dev->mtu))
2505 fp->mode = TPA_MODE_GRO;
2507 fp->mode = TPA_MODE_DISABLED;
2509 /* We don't want TPA if it's disabled in bp
2510 * or if this is an FCoE L2 ring.
2512 if (bp->disable_tpa || IS_FCOE_FP(fp))
2513 fp->mode = TPA_MODE_DISABLED;
2516 int bnx2x_load_cnic(struct bnx2x *bp)
2518 int i, rc, port = BP_PORT(bp);
2520 DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2522 mutex_init(&bp->cnic_mutex);
2525 rc = bnx2x_alloc_mem_cnic(bp);
2527 BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2528 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2532 rc = bnx2x_alloc_fp_mem_cnic(bp);
2534 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2535 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2538 /* Update the number of queues with the cnic queues */
2539 rc = bnx2x_set_real_num_queues(bp, 1);
2541 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2542 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2545 /* Add all CNIC NAPI objects */
2546 bnx2x_add_all_napi_cnic(bp);
2547 DP(NETIF_MSG_IFUP, "cnic napi added\n");
2548 bnx2x_napi_enable_cnic(bp);
2550 rc = bnx2x_init_hw_func_cnic(bp);
2552 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2554 bnx2x_nic_init_cnic(bp);
2557 /* Enable Timer scan */
2558 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2560 /* setup cnic queues */
2561 for_each_cnic_queue(bp, i) {
2562 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2564 BNX2X_ERR("Queue setup failed\n");
2565 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2570 /* Initialize Rx filter. */
2571 bnx2x_set_rx_mode_inner(bp);
2573 /* re-read iscsi info */
2574 bnx2x_get_iscsi_info(bp);
2575 bnx2x_setup_cnic_irq_info(bp);
2576 bnx2x_setup_cnic_info(bp);
2577 bp->cnic_loaded = true;
2578 if (bp->state == BNX2X_STATE_OPEN)
2579 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2581 DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2585 #ifndef BNX2X_STOP_ON_ERROR
2587 /* Disable Timer scan */
2588 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2591 bnx2x_napi_disable_cnic(bp);
2592 /* Update the number of queues without the cnic queues */
2593 if (bnx2x_set_real_num_queues(bp, 0))
2594 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2596 BNX2X_ERR("CNIC-related load failed\n");
2597 bnx2x_free_fp_mem_cnic(bp);
2598 bnx2x_free_mem_cnic(bp);
2600 #endif /* ! BNX2X_STOP_ON_ERROR */
2603 /* must be called with rtnl_lock */
2604 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2606 int port = BP_PORT(bp);
2607 int i, rc = 0, load_code = 0;
2609 DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2611 "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2613 #ifdef BNX2X_STOP_ON_ERROR
2614 if (unlikely(bp->panic)) {
2615 BNX2X_ERR("Can't load NIC when there is panic\n");
2620 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2622 /* zero the structure w/o any lock, before SP handler is initialized */
2623 memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2624 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2625 &bp->last_reported_link.link_report_flags);
2628 /* must be called before memory allocation and HW init */
2629 bnx2x_ilt_set_info(bp);
2632 * Zero fastpath structures preserving invariants like napi, which are
2633 * allocated only once, fp index, max_cos, bp pointer.
2634 * Also set fp->mode and txdata_ptr.
2636 DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2637 for_each_queue(bp, i)
2639 memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2640 bp->num_cnic_queues) *
2641 sizeof(struct bnx2x_fp_txdata));
2643 bp->fcoe_init = false;
2645 /* Set the receive queues buffer size */
2646 bnx2x_set_rx_buf_size(bp);
2649 rc = bnx2x_alloc_mem(bp);
2651 BNX2X_ERR("Unable to allocate bp memory\n");
2656 /* need to be done after alloc mem, since it's self adjusting to amount
2657 * of memory available for RSS queues
2659 rc = bnx2x_alloc_fp_mem(bp);
2661 BNX2X_ERR("Unable to allocate memory for fps\n");
2662 LOAD_ERROR_EXIT(bp, load_error0);
2665 /* Allocated memory for FW statistics */
2666 if (bnx2x_alloc_fw_stats_mem(bp))
2667 LOAD_ERROR_EXIT(bp, load_error0);
2669 /* request pf to initialize status blocks */
2671 rc = bnx2x_vfpf_init(bp);
2673 LOAD_ERROR_EXIT(bp, load_error0);
2676 /* As long as bnx2x_alloc_mem() may possibly update
2677 * bp->num_queues, bnx2x_set_real_num_queues() should always
2678 * come after it. At this stage cnic queues are not counted.
2680 rc = bnx2x_set_real_num_queues(bp, 0);
2682 BNX2X_ERR("Unable to set real_num_queues\n");
2683 LOAD_ERROR_EXIT(bp, load_error0);
2686 /* configure multi cos mappings in kernel.
2687 * this configuration may be overridden by a multi class queue
2688 * discipline or by a dcbx negotiation result.
2690 bnx2x_setup_tc(bp->dev, bp->max_cos);
2692 /* Add all NAPI objects */
2693 bnx2x_add_all_napi(bp);
2694 DP(NETIF_MSG_IFUP, "napi added\n");
2695 bnx2x_napi_enable(bp);
2698 /* set pf load just before approaching the MCP */
2699 bnx2x_set_pf_load(bp);
2701 /* if mcp exists send load request and analyze response */
2702 if (!BP_NOMCP(bp)) {
2703 /* attempt to load pf */
2704 rc = bnx2x_nic_load_request(bp, &load_code);
2706 LOAD_ERROR_EXIT(bp, load_error1);
2708 /* what did mcp say? */
2709 rc = bnx2x_compare_fw_ver(bp, load_code, true);
2711 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2712 LOAD_ERROR_EXIT(bp, load_error2);
2715 load_code = bnx2x_nic_load_no_mcp(bp, port);
2718 /* mark pmf if applicable */
2719 bnx2x_nic_load_pmf(bp, load_code);
2721 /* Init Function state controlling object */
2722 bnx2x__init_func_obj(bp);
2725 rc = bnx2x_init_hw(bp, load_code);
2727 BNX2X_ERR("HW init failed, aborting\n");
2728 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2729 LOAD_ERROR_EXIT(bp, load_error2);
2733 bnx2x_pre_irq_nic_init(bp);
2735 /* Connect to IRQs */
2736 rc = bnx2x_setup_irqs(bp);
2738 BNX2X_ERR("setup irqs failed\n");
2740 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2741 LOAD_ERROR_EXIT(bp, load_error2);
2744 /* Init per-function objects */
2746 /* Setup NIC internals and enable interrupts */
2747 bnx2x_post_irq_nic_init(bp, load_code);
2749 bnx2x_init_bp_objs(bp);
2750 bnx2x_iov_nic_init(bp);
2752 /* Set AFEX default VLAN tag to an invalid value */
2753 bp->afex_def_vlan_tag = -1;
2754 bnx2x_nic_load_afex_dcc(bp, load_code);
2755 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2756 rc = bnx2x_func_start(bp);
2758 BNX2X_ERR("Function start failed!\n");
2759 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2761 LOAD_ERROR_EXIT(bp, load_error3);
2764 /* Send LOAD_DONE command to MCP */
2765 if (!BP_NOMCP(bp)) {
2766 load_code = bnx2x_fw_command(bp,
2767 DRV_MSG_CODE_LOAD_DONE, 0);
2769 BNX2X_ERR("MCP response failure, aborting\n");
2771 LOAD_ERROR_EXIT(bp, load_error3);
2775 /* initialize FW coalescing state machines in RAM */
2776 bnx2x_update_coalesce(bp);
2779 /* setup the leading queue */
2780 rc = bnx2x_setup_leading(bp);
2782 BNX2X_ERR("Setup leading failed!\n");
2783 LOAD_ERROR_EXIT(bp, load_error3);
2786 /* set up the rest of the queues */
2787 for_each_nondefault_eth_queue(bp, i) {
2789 rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2791 rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2793 BNX2X_ERR("Queue %d setup failed\n", i);
2794 LOAD_ERROR_EXIT(bp, load_error3);
2799 rc = bnx2x_init_rss(bp);
2801 BNX2X_ERR("PF RSS init failed\n");
2802 LOAD_ERROR_EXIT(bp, load_error3);
2805 /* Now when Clients are configured we are ready to work */
2806 bp->state = BNX2X_STATE_OPEN;
2808 /* Configure a ucast MAC */
2810 rc = bnx2x_set_eth_mac(bp, true);
2812 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2815 BNX2X_ERR("Setting Ethernet MAC failed\n");
2816 LOAD_ERROR_EXIT(bp, load_error3);
2819 if (IS_PF(bp) && bp->pending_max) {
2820 bnx2x_update_max_mf_config(bp, bp->pending_max);
2821 bp->pending_max = 0;
2825 rc = bnx2x_initial_phy_init(bp, load_mode);
2827 LOAD_ERROR_EXIT(bp, load_error3);
2829 bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2831 /* Start fast path */
2833 /* Initialize Rx filter. */
2834 bnx2x_set_rx_mode_inner(bp);
2836 if (bp->flags & PTP_SUPPORTED) {
2838 bnx2x_configure_ptp_filters(bp);
2841 switch (load_mode) {
2843 /* Tx queue should be only re-enabled */
2844 netif_tx_wake_all_queues(bp->dev);
2848 netif_tx_start_all_queues(bp->dev);
2849 smp_mb__after_atomic();
2853 case LOAD_LOOPBACK_EXT:
2854 bp->state = BNX2X_STATE_DIAG;
2862 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2864 bnx2x__link_status_update(bp);
2866 /* start the timer */
2867 mod_timer(&bp->timer, jiffies + bp->current_interval);
2869 if (CNIC_ENABLED(bp))
2870 bnx2x_load_cnic(bp);
2873 bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_GET_DRV_VERSION, 0);
2875 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2876 /* mark driver is loaded in shmem2 */
2878 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2879 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2880 val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2881 DRV_FLAGS_CAPABILITIES_LOADED_L2);
2884 /* Wait for all pending SP commands to complete */
2885 if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2886 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2887 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2891 /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2892 if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2893 bnx2x_dcbx_init(bp, false);
2895 DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2899 #ifndef BNX2X_STOP_ON_ERROR
2902 bnx2x_int_disable_sync(bp, 1);
2904 /* Clean queueable objects */
2905 bnx2x_squeeze_objects(bp);
2908 /* Free SKBs, SGEs, TPA pool and driver internals */
2909 bnx2x_free_skbs(bp);
2910 for_each_rx_queue(bp, i)
2911 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2916 if (IS_PF(bp) && !BP_NOMCP(bp)) {
2917 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2918 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2923 bnx2x_napi_disable(bp);
2924 bnx2x_del_all_napi(bp);
2926 /* clear pf_load status, as it was already set */
2928 bnx2x_clear_pf_load(bp);
2930 bnx2x_free_fw_stats_mem(bp);
2931 bnx2x_free_fp_mem(bp);
2935 #endif /* ! BNX2X_STOP_ON_ERROR */
2938 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2942 /* Wait until tx fastpath tasks complete */
2943 for_each_tx_queue(bp, i) {
2944 struct bnx2x_fastpath *fp = &bp->fp[i];
2946 for_each_cos_in_tx_queue(fp, cos)
2947 rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2954 /* must be called with rtnl_lock */
2955 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2958 bool global = false;
2960 DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2962 /* mark driver is unloaded in shmem2 */
2963 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2965 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2966 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2967 val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2970 if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2971 (bp->state == BNX2X_STATE_CLOSED ||
2972 bp->state == BNX2X_STATE_ERROR)) {
2973 /* We can get here if the driver has been unloaded
2974 * during parity error recovery and is either waiting for a
2975 * leader to complete or for other functions to unload and
2976 * then ifdown has been issued. In this case we want to
2977 * unload and let other functions to complete a recovery
2980 bp->recovery_state = BNX2X_RECOVERY_DONE;
2982 bnx2x_release_leader_lock(bp);
2985 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2986 BNX2X_ERR("Can't unload in closed or error state\n");
2990 /* Nothing to do during unload if previous bnx2x_nic_load()
2991 * have not completed successfully - all resources are released.
2993 * we can get here only after unsuccessful ndo_* callback, during which
2994 * dev->IFF_UP flag is still on.
2996 if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2999 /* It's important to set the bp->state to the value different from
3000 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
3001 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
3003 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
3006 /* indicate to VFs that the PF is going down */
3007 bnx2x_iov_channel_down(bp);
3009 if (CNIC_LOADED(bp))
3010 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
3013 bnx2x_tx_disable(bp);
3014 netdev_reset_tc(bp->dev);
3016 bp->rx_mode = BNX2X_RX_MODE_NONE;
3018 del_timer_sync(&bp->timer);
3021 /* Set ALWAYS_ALIVE bit in shmem */
3022 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
3023 bnx2x_drv_pulse(bp);
3024 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
3025 bnx2x_save_statistics(bp);
3028 /* wait till consumers catch up with producers in all queues */
3029 bnx2x_drain_tx_queues(bp);
3031 /* if VF indicate to PF this function is going down (PF will delete sp
3032 * elements and clear initializations
3035 bnx2x_vfpf_close_vf(bp);
3036 else if (unload_mode != UNLOAD_RECOVERY)
3037 /* if this is a normal/close unload need to clean up chip*/
3038 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
3040 /* Send the UNLOAD_REQUEST to the MCP */
3041 bnx2x_send_unload_req(bp, unload_mode);
3043 /* Prevent transactions to host from the functions on the
3044 * engine that doesn't reset global blocks in case of global
3045 * attention once global blocks are reset and gates are opened
3046 * (the engine which leader will perform the recovery
3049 if (!CHIP_IS_E1x(bp))
3050 bnx2x_pf_disable(bp);
3052 /* Disable HW interrupts, NAPI */
3053 bnx2x_netif_stop(bp, 1);
3054 /* Delete all NAPI objects */
3055 bnx2x_del_all_napi(bp);
3056 if (CNIC_LOADED(bp))
3057 bnx2x_del_all_napi_cnic(bp);
3061 /* Report UNLOAD_DONE to MCP */
3062 bnx2x_send_unload_done(bp, false);
3066 * At this stage no more interrupts will arrive so we may safely clean
3067 * the queueable objects here in case they failed to get cleaned so far.
3070 bnx2x_squeeze_objects(bp);
3072 /* There should be no more pending SP commands at this stage */
3077 /* clear pending work in rtnl task */
3078 bp->sp_rtnl_state = 0;
3081 /* Free SKBs, SGEs, TPA pool and driver internals */
3082 bnx2x_free_skbs(bp);
3083 if (CNIC_LOADED(bp))
3084 bnx2x_free_skbs_cnic(bp);
3085 for_each_rx_queue(bp, i)
3086 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
3088 bnx2x_free_fp_mem(bp);
3089 if (CNIC_LOADED(bp))
3090 bnx2x_free_fp_mem_cnic(bp);
3093 if (CNIC_LOADED(bp))
3094 bnx2x_free_mem_cnic(bp);
3098 bp->state = BNX2X_STATE_CLOSED;
3099 bp->cnic_loaded = false;
3101 /* Clear driver version indication in shmem */
3103 bnx2x_update_mng_version(bp);
3105 /* Check if there are pending parity attentions. If there are - set
3106 * RECOVERY_IN_PROGRESS.
3108 if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
3109 bnx2x_set_reset_in_progress(bp);
3111 /* Set RESET_IS_GLOBAL if needed */
3113 bnx2x_set_reset_global(bp);
3116 /* The last driver must disable a "close the gate" if there is no
3117 * parity attention or "process kill" pending.
3120 !bnx2x_clear_pf_load(bp) &&
3121 bnx2x_reset_is_done(bp, BP_PATH(bp)))
3122 bnx2x_disable_close_the_gate(bp);
3124 DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3129 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3133 /* If there is no power capability, silently succeed */
3134 if (!bp->pdev->pm_cap) {
3135 BNX2X_DEV_INFO("No power capability. Breaking.\n");
3139 pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3143 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3144 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3145 PCI_PM_CTRL_PME_STATUS));
3147 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3148 /* delay required during transition out of D3hot */
3153 /* If there are other clients above don't
3154 shut down the power */
3155 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3157 /* Don't shut down the power for emulation and FPGA */
3158 if (CHIP_REV_IS_SLOW(bp))
3161 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3165 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3167 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3170 /* No more memory access after this point until
3171 * device is brought back to D0.
3176 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3183 * net_device service functions
3185 static int bnx2x_poll(struct napi_struct *napi, int budget)
3189 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3191 struct bnx2x *bp = fp->bp;
3194 #ifdef BNX2X_STOP_ON_ERROR
3195 if (unlikely(bp->panic)) {
3196 napi_complete(napi);
3200 if (!bnx2x_fp_lock_napi(fp))
3203 for_each_cos_in_tx_queue(fp, cos)
3204 if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3205 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3207 if (bnx2x_has_rx_work(fp)) {
3208 work_done += bnx2x_rx_int(fp, budget - work_done);
3210 /* must not complete if we consumed full budget */
3211 if (work_done >= budget) {
3212 bnx2x_fp_unlock_napi(fp);
3217 bnx2x_fp_unlock_napi(fp);
3219 /* Fall out from the NAPI loop if needed */
3220 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3222 /* No need to update SB for FCoE L2 ring as long as
3223 * it's connected to the default SB and the SB
3224 * has been updated when NAPI was scheduled.
3226 if (IS_FCOE_FP(fp)) {
3227 napi_complete(napi);
3230 bnx2x_update_fpsb_idx(fp);
3231 /* bnx2x_has_rx_work() reads the status block,
3232 * thus we need to ensure that status block indices
3233 * have been actually read (bnx2x_update_fpsb_idx)
3234 * prior to this check (bnx2x_has_rx_work) so that
3235 * we won't write the "newer" value of the status block
3236 * to IGU (if there was a DMA right after
3237 * bnx2x_has_rx_work and if there is no rmb, the memory
3238 * reading (bnx2x_update_fpsb_idx) may be postponed
3239 * to right before bnx2x_ack_sb). In this case there
3240 * will never be another interrupt until there is
3241 * another update of the status block, while there
3242 * is still unhandled work.
3246 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3247 napi_complete(napi);
3248 /* Re-enable interrupts */
3249 DP(NETIF_MSG_RX_STATUS,
3250 "Update index to %d\n", fp->fp_hc_idx);
3251 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3252 le16_to_cpu(fp->fp_hc_idx),
3262 #ifdef CONFIG_NET_RX_BUSY_POLL
3263 /* must be called with local_bh_disable()d */
3264 int bnx2x_low_latency_recv(struct napi_struct *napi)
3266 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3268 struct bnx2x *bp = fp->bp;
3271 if ((bp->state == BNX2X_STATE_CLOSED) ||
3272 (bp->state == BNX2X_STATE_ERROR) ||
3273 (bp->dev->features & (NETIF_F_LRO | NETIF_F_GRO)))
3274 return LL_FLUSH_FAILED;
3276 if (!bnx2x_fp_lock_poll(fp))
3277 return LL_FLUSH_BUSY;
3279 if (bnx2x_has_rx_work(fp))
3280 found = bnx2x_rx_int(fp, 4);
3282 bnx2x_fp_unlock_poll(fp);
3288 /* we split the first BD into headers and data BDs
3289 * to ease the pain of our fellow microcode engineers
3290 * we use one mapping for both BDs
3292 static u16 bnx2x_tx_split(struct bnx2x *bp,
3293 struct bnx2x_fp_txdata *txdata,
3294 struct sw_tx_bd *tx_buf,
3295 struct eth_tx_start_bd **tx_bd, u16 hlen,
3298 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3299 struct eth_tx_bd *d_tx_bd;
3301 int old_len = le16_to_cpu(h_tx_bd->nbytes);
3303 /* first fix first BD */
3304 h_tx_bd->nbytes = cpu_to_le16(hlen);
3306 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3307 h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3309 /* now get a new data BD
3310 * (after the pbd) and fill it */
3311 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3312 d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3314 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3315 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3317 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3318 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3319 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3321 /* this marks the BD as one that has no individual mapping */
3322 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3324 DP(NETIF_MSG_TX_QUEUED,
3325 "TSO split data size is %d (%x:%x)\n",
3326 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3329 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3334 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3335 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3336 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3338 __sum16 tsum = (__force __sum16) csum;
3341 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3342 csum_partial(t_header - fix, fix, 0)));
3345 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3346 csum_partial(t_header, -fix, 0)));
3348 return bswab16(tsum);
3351 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3357 if (skb->ip_summed != CHECKSUM_PARTIAL)
3360 protocol = vlan_get_protocol(skb);
3361 if (protocol == htons(ETH_P_IPV6)) {
3363 prot = ipv6_hdr(skb)->nexthdr;
3366 prot = ip_hdr(skb)->protocol;
3369 if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3370 if (inner_ip_hdr(skb)->version == 6) {
3371 rc |= XMIT_CSUM_ENC_V6;
3372 if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3373 rc |= XMIT_CSUM_TCP;
3375 rc |= XMIT_CSUM_ENC_V4;
3376 if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3377 rc |= XMIT_CSUM_TCP;
3380 if (prot == IPPROTO_TCP)
3381 rc |= XMIT_CSUM_TCP;
3383 if (skb_is_gso(skb)) {
3384 if (skb_is_gso_v6(skb)) {
3385 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3386 if (rc & XMIT_CSUM_ENC)
3387 rc |= XMIT_GSO_ENC_V6;
3389 rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3390 if (rc & XMIT_CSUM_ENC)
3391 rc |= XMIT_GSO_ENC_V4;
3398 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3399 /* check if packet requires linearization (packet is too fragmented)
3400 no need to check fragmentation if page size > 8K (there will be no
3401 violation to FW restrictions) */
3402 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3407 int first_bd_sz = 0;
3409 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3410 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3412 if (xmit_type & XMIT_GSO) {
3413 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3414 /* Check if LSO packet needs to be copied:
3415 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3416 int wnd_size = MAX_FETCH_BD - 3;
3417 /* Number of windows to check */
3418 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3423 /* Headers length */
3424 hlen = (int)(skb_transport_header(skb) - skb->data) +
3427 /* Amount of data (w/o headers) on linear part of SKB*/
3428 first_bd_sz = skb_headlen(skb) - hlen;
3430 wnd_sum = first_bd_sz;
3432 /* Calculate the first sum - it's special */
3433 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3435 skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3437 /* If there was data on linear skb data - check it */
3438 if (first_bd_sz > 0) {
3439 if (unlikely(wnd_sum < lso_mss)) {
3444 wnd_sum -= first_bd_sz;
3447 /* Others are easier: run through the frag list and
3448 check all windows */
3449 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3451 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3453 if (unlikely(wnd_sum < lso_mss)) {
3458 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3461 /* in non-LSO too fragmented packet should always
3468 if (unlikely(to_copy))
3469 DP(NETIF_MSG_TX_QUEUED,
3470 "Linearization IS REQUIRED for %s packet. num_frags %d hlen %d first_bd_sz %d\n",
3471 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3472 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3479 * bnx2x_set_pbd_gso - update PBD in GSO case.
3483 * @xmit_type: xmit flags
3485 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3486 struct eth_tx_parse_bd_e1x *pbd,
3489 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3490 pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3491 pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3493 if (xmit_type & XMIT_GSO_V4) {
3494 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3495 pbd->tcp_pseudo_csum =
3496 bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3498 0, IPPROTO_TCP, 0));
3500 pbd->tcp_pseudo_csum =
3501 bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3502 &ipv6_hdr(skb)->daddr,
3503 0, IPPROTO_TCP, 0));
3507 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3511 * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3513 * @bp: driver handle
3515 * @parsing_data: data to be updated
3516 * @xmit_type: xmit flags
3518 * 57712/578xx related, when skb has encapsulation
3520 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3521 u32 *parsing_data, u32 xmit_type)
3524 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3525 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3526 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3528 if (xmit_type & XMIT_CSUM_TCP) {
3529 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3530 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3531 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3533 return skb_inner_transport_header(skb) +
3534 inner_tcp_hdrlen(skb) - skb->data;
3537 /* We support checksum offload for TCP and UDP only.
3538 * No need to pass the UDP header length - it's a constant.
3540 return skb_inner_transport_header(skb) +
3541 sizeof(struct udphdr) - skb->data;
3545 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3547 * @bp: driver handle
3549 * @parsing_data: data to be updated
3550 * @xmit_type: xmit flags
3552 * 57712/578xx related
3554 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3555 u32 *parsing_data, u32 xmit_type)
3558 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3559 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3560 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3562 if (xmit_type & XMIT_CSUM_TCP) {
3563 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3564 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3565 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3567 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3569 /* We support checksum offload for TCP and UDP only.
3570 * No need to pass the UDP header length - it's a constant.
3572 return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3575 /* set FW indication according to inner or outer protocols if tunneled */
3576 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3577 struct eth_tx_start_bd *tx_start_bd,
3580 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3582 if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3583 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3585 if (!(xmit_type & XMIT_CSUM_TCP))
3586 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3590 * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3592 * @bp: driver handle
3594 * @pbd: parse BD to be updated
3595 * @xmit_type: xmit flags
3597 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3598 struct eth_tx_parse_bd_e1x *pbd,
3601 u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3603 /* for now NS flag is not used in Linux */
3606 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3607 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3609 pbd->ip_hlen_w = (skb_transport_header(skb) -
3610 skb_network_header(skb)) >> 1;
3612 hlen += pbd->ip_hlen_w;
3614 /* We support checksum offload for TCP and UDP only */
3615 if (xmit_type & XMIT_CSUM_TCP)
3616 hlen += tcp_hdrlen(skb) / 2;
3618 hlen += sizeof(struct udphdr) / 2;
3620 pbd->total_hlen_w = cpu_to_le16(hlen);
3623 if (xmit_type & XMIT_CSUM_TCP) {
3624 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3627 s8 fix = SKB_CS_OFF(skb); /* signed! */
3629 DP(NETIF_MSG_TX_QUEUED,
3630 "hlen %d fix %d csum before fix %x\n",
3631 le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3633 /* HW bug: fixup the CSUM */
3634 pbd->tcp_pseudo_csum =
3635 bnx2x_csum_fix(skb_transport_header(skb),
3638 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3639 pbd->tcp_pseudo_csum);
3645 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3646 struct eth_tx_parse_bd_e2 *pbd_e2,
3647 struct eth_tx_parse_2nd_bd *pbd2,
3652 u8 outerip_off, outerip_len = 0;
3654 /* from outer IP to transport */
3655 hlen_w = (skb_inner_transport_header(skb) -
3656 skb_network_header(skb)) >> 1;
3659 hlen_w += inner_tcp_hdrlen(skb) >> 1;
3661 pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3663 /* outer IP header info */
3664 if (xmit_type & XMIT_CSUM_V4) {
3665 struct iphdr *iph = ip_hdr(skb);
3666 u32 csum = (__force u32)(~iph->check) -
3667 (__force u32)iph->tot_len -
3668 (__force u32)iph->frag_off;
3670 outerip_len = iph->ihl << 1;
3672 pbd2->fw_ip_csum_wo_len_flags_frag =
3673 bswab16(csum_fold((__force __wsum)csum));
3675 pbd2->fw_ip_hdr_to_payload_w =
3676 hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3677 pbd_e2->data.tunnel_data.flags |=
3678 ETH_TUNNEL_DATA_IP_HDR_TYPE_OUTER;
3681 pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3683 pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3685 /* inner IP header info */
3686 if (xmit_type & XMIT_CSUM_ENC_V4) {
3687 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3689 pbd_e2->data.tunnel_data.pseudo_csum =
3690 bswab16(~csum_tcpudp_magic(
3691 inner_ip_hdr(skb)->saddr,
3692 inner_ip_hdr(skb)->daddr,
3693 0, IPPROTO_TCP, 0));
3695 pbd_e2->data.tunnel_data.pseudo_csum =
3696 bswab16(~csum_ipv6_magic(
3697 &inner_ipv6_hdr(skb)->saddr,
3698 &inner_ipv6_hdr(skb)->daddr,
3699 0, IPPROTO_TCP, 0));
3702 outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3707 ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3708 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3709 ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3711 if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3712 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3713 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3717 static inline void bnx2x_set_ipv6_ext_e2(struct sk_buff *skb, u32 *parsing_data,
3720 struct ipv6hdr *ipv6;
3722 if (!(xmit_type & (XMIT_GSO_ENC_V6 | XMIT_GSO_V6)))
3725 if (xmit_type & XMIT_GSO_ENC_V6)
3726 ipv6 = inner_ipv6_hdr(skb);
3727 else /* XMIT_GSO_V6 */
3728 ipv6 = ipv6_hdr(skb);
3730 if (ipv6->nexthdr == NEXTHDR_IPV6)
3731 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3734 /* called with netif_tx_lock
3735 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3736 * netif_wake_queue()
3738 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3740 struct bnx2x *bp = netdev_priv(dev);
3742 struct netdev_queue *txq;
3743 struct bnx2x_fp_txdata *txdata;
3744 struct sw_tx_bd *tx_buf;
3745 struct eth_tx_start_bd *tx_start_bd, *first_bd;
3746 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3747 struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3748 struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3749 struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3750 u32 pbd_e2_parsing_data = 0;
3751 u16 pkt_prod, bd_prod;
3754 u32 xmit_type = bnx2x_xmit_type(bp, skb);
3757 __le16 pkt_size = 0;
3759 u8 mac_type = UNICAST_ADDRESS;
3761 #ifdef BNX2X_STOP_ON_ERROR
3762 if (unlikely(bp->panic))
3763 return NETDEV_TX_BUSY;
3766 txq_index = skb_get_queue_mapping(skb);
3767 txq = netdev_get_tx_queue(dev, txq_index);
3769 BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3771 txdata = &bp->bnx2x_txq[txq_index];
3773 /* enable this debug print to view the transmission queue being used
3774 DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3775 txq_index, fp_index, txdata_index); */
3777 /* enable this debug print to view the transmission details
3778 DP(NETIF_MSG_TX_QUEUED,
3779 "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3780 txdata->cid, fp_index, txdata_index, txdata, fp); */
3782 if (unlikely(bnx2x_tx_avail(bp, txdata) <
3783 skb_shinfo(skb)->nr_frags +
3785 NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3786 /* Handle special storage cases separately */
3787 if (txdata->tx_ring_size == 0) {
3788 struct bnx2x_eth_q_stats *q_stats =
3789 bnx2x_fp_qstats(bp, txdata->parent_fp);
3790 q_stats->driver_filtered_tx_pkt++;
3792 return NETDEV_TX_OK;
3794 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3795 netif_tx_stop_queue(txq);
3796 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3798 return NETDEV_TX_BUSY;
3801 DP(NETIF_MSG_TX_QUEUED,
3802 "queue[%d]: SKB: summed %x protocol %x protocol(%x,%x) gso type %x xmit_type %x len %d\n",
3803 txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3804 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3807 eth = (struct ethhdr *)skb->data;
3809 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3810 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3811 if (is_broadcast_ether_addr(eth->h_dest))
3812 mac_type = BROADCAST_ADDRESS;
3814 mac_type = MULTICAST_ADDRESS;
3817 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3818 /* First, check if we need to linearize the skb (due to FW
3819 restrictions). No need to check fragmentation if page size > 8K
3820 (there will be no violation to FW restrictions) */
3821 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3822 /* Statistics of linearization */
3824 if (skb_linearize(skb) != 0) {
3825 DP(NETIF_MSG_TX_QUEUED,
3826 "SKB linearization failed - silently dropping this SKB\n");
3827 dev_kfree_skb_any(skb);
3828 return NETDEV_TX_OK;
3832 /* Map skb linear data for DMA */
3833 mapping = dma_map_single(&bp->pdev->dev, skb->data,
3834 skb_headlen(skb), DMA_TO_DEVICE);
3835 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3836 DP(NETIF_MSG_TX_QUEUED,
3837 "SKB mapping failed - silently dropping this SKB\n");
3838 dev_kfree_skb_any(skb);
3839 return NETDEV_TX_OK;
3842 Please read carefully. First we use one BD which we mark as start,
3843 then we have a parsing info BD (used for TSO or xsum),
3844 and only then we have the rest of the TSO BDs.
3845 (don't forget to mark the last one as last,
3846 and to unmap only AFTER you write to the BD ...)
3847 And above all, all pdb sizes are in words - NOT DWORDS!
3850 /* get current pkt produced now - advance it just before sending packet
3851 * since mapping of pages may fail and cause packet to be dropped
3853 pkt_prod = txdata->tx_pkt_prod;
3854 bd_prod = TX_BD(txdata->tx_bd_prod);
3856 /* get a tx_buf and first BD
3857 * tx_start_bd may be changed during SPLIT,
3858 * but first_bd will always stay first
3860 tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3861 tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3862 first_bd = tx_start_bd;
3864 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3866 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
3867 if (!(bp->flags & TX_TIMESTAMPING_EN)) {
3868 BNX2X_ERR("Tx timestamping was not enabled, this packet will not be timestamped\n");
3869 } else if (bp->ptp_tx_skb) {
3870 BNX2X_ERR("The device supports only a single outstanding packet to timestamp, this packet will not be timestamped\n");
3872 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3873 /* schedule check for Tx timestamp */
3874 bp->ptp_tx_skb = skb_get(skb);
3875 bp->ptp_tx_start = jiffies;
3876 schedule_work(&bp->ptp_task);
3880 /* header nbd: indirectly zero other flags! */
3881 tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3883 /* remember the first BD of the packet */
3884 tx_buf->first_bd = txdata->tx_bd_prod;
3888 DP(NETIF_MSG_TX_QUEUED,
3889 "sending pkt %u @%p next_idx %u bd %u @%p\n",
3890 pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3892 if (skb_vlan_tag_present(skb)) {
3893 tx_start_bd->vlan_or_ethertype =
3894 cpu_to_le16(skb_vlan_tag_get(skb));
3895 tx_start_bd->bd_flags.as_bitfield |=
3896 (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3898 /* when transmitting in a vf, start bd must hold the ethertype
3899 * for fw to enforce it
3901 #ifndef BNX2X_STOP_ON_ERROR
3904 tx_start_bd->vlan_or_ethertype =
3905 cpu_to_le16(ntohs(eth->h_proto));
3906 #ifndef BNX2X_STOP_ON_ERROR
3908 /* used by FW for packet accounting */
3909 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3913 nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3915 /* turn on parsing and get a BD */
3916 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3918 if (xmit_type & XMIT_CSUM)
3919 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3921 if (!CHIP_IS_E1x(bp)) {
3922 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3923 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3925 if (xmit_type & XMIT_CSUM_ENC) {
3926 u16 global_data = 0;
3928 /* Set PBD in enc checksum offload case */
3929 hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3930 &pbd_e2_parsing_data,
3933 /* turn on 2nd parsing and get a BD */
3934 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3936 pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3938 memset(pbd2, 0, sizeof(*pbd2));
3940 pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3941 (skb_inner_network_header(skb) -
3944 if (xmit_type & XMIT_GSO_ENC)
3945 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3949 pbd2->global_data = cpu_to_le16(global_data);
3951 /* add addition parse BD indication to start BD */
3952 SET_FLAG(tx_start_bd->general_data,
3953 ETH_TX_START_BD_PARSE_NBDS, 1);
3954 /* set encapsulation flag in start BD */
3955 SET_FLAG(tx_start_bd->general_data,
3956 ETH_TX_START_BD_TUNNEL_EXIST, 1);
3958 tx_buf->flags |= BNX2X_HAS_SECOND_PBD;
3961 } else if (xmit_type & XMIT_CSUM) {
3962 /* Set PBD in checksum offload case w/o encapsulation */
3963 hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3964 &pbd_e2_parsing_data,
3968 bnx2x_set_ipv6_ext_e2(skb, &pbd_e2_parsing_data, xmit_type);
3969 /* Add the macs to the parsing BD if this is a vf or if
3970 * Tx Switching is enabled.
3973 /* override GRE parameters in BD */
3974 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3975 &pbd_e2->data.mac_addr.src_mid,
3976 &pbd_e2->data.mac_addr.src_lo,
3979 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3980 &pbd_e2->data.mac_addr.dst_mid,
3981 &pbd_e2->data.mac_addr.dst_lo,
3984 if (bp->flags & TX_SWITCHING)
3985 bnx2x_set_fw_mac_addr(
3986 &pbd_e2->data.mac_addr.dst_hi,
3987 &pbd_e2->data.mac_addr.dst_mid,
3988 &pbd_e2->data.mac_addr.dst_lo,
3990 #ifdef BNX2X_STOP_ON_ERROR
3991 /* Enforce security is always set in Stop on Error -
3992 * source mac should be present in the parsing BD
3994 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3995 &pbd_e2->data.mac_addr.src_mid,
3996 &pbd_e2->data.mac_addr.src_lo,
4001 SET_FLAG(pbd_e2_parsing_data,
4002 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
4004 u16 global_data = 0;
4005 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
4006 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
4007 /* Set PBD in checksum offload case */
4008 if (xmit_type & XMIT_CSUM)
4009 hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
4011 SET_FLAG(global_data,
4012 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
4013 pbd_e1x->global_data |= cpu_to_le16(global_data);
4016 /* Setup the data pointer of the first BD of the packet */
4017 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4018 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4019 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
4020 pkt_size = tx_start_bd->nbytes;
4022 DP(NETIF_MSG_TX_QUEUED,
4023 "first bd @%p addr (%x:%x) nbytes %d flags %x vlan %x\n",
4024 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
4025 le16_to_cpu(tx_start_bd->nbytes),
4026 tx_start_bd->bd_flags.as_bitfield,
4027 le16_to_cpu(tx_start_bd->vlan_or_ethertype));
4029 if (xmit_type & XMIT_GSO) {
4031 DP(NETIF_MSG_TX_QUEUED,
4032 "TSO packet len %d hlen %d total len %d tso size %d\n",
4033 skb->len, hlen, skb_headlen(skb),
4034 skb_shinfo(skb)->gso_size);
4036 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
4038 if (unlikely(skb_headlen(skb) > hlen)) {
4040 bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
4044 if (!CHIP_IS_E1x(bp))
4045 pbd_e2_parsing_data |=
4046 (skb_shinfo(skb)->gso_size <<
4047 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
4048 ETH_TX_PARSE_BD_E2_LSO_MSS;
4050 bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
4053 /* Set the PBD's parsing_data field if not zero
4054 * (for the chips newer than 57711).
4056 if (pbd_e2_parsing_data)
4057 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
4059 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
4061 /* Handle fragmented skb */
4062 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4063 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4065 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
4066 skb_frag_size(frag), DMA_TO_DEVICE);
4067 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
4068 unsigned int pkts_compl = 0, bytes_compl = 0;
4070 DP(NETIF_MSG_TX_QUEUED,
4071 "Unable to map page - dropping packet...\n");
4073 /* we need unmap all buffers already mapped
4075 * first_bd->nbd need to be properly updated
4076 * before call to bnx2x_free_tx_pkt
4078 first_bd->nbd = cpu_to_le16(nbd);
4079 bnx2x_free_tx_pkt(bp, txdata,
4080 TX_BD(txdata->tx_pkt_prod),
4081 &pkts_compl, &bytes_compl);
4082 return NETDEV_TX_OK;
4085 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4086 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4087 if (total_pkt_bd == NULL)
4088 total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4090 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4091 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4092 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
4093 le16_add_cpu(&pkt_size, skb_frag_size(frag));
4096 DP(NETIF_MSG_TX_QUEUED,
4097 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
4098 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
4099 le16_to_cpu(tx_data_bd->nbytes));
4102 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
4104 /* update with actual num BDs */
4105 first_bd->nbd = cpu_to_le16(nbd);
4107 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4109 /* now send a tx doorbell, counting the next BD
4110 * if the packet contains or ends with it
4112 if (TX_BD_POFF(bd_prod) < nbd)
4115 /* total_pkt_bytes should be set on the first data BD if
4116 * it's not an LSO packet and there is more than one
4117 * data BD. In this case pkt_size is limited by an MTU value.
4118 * However we prefer to set it for an LSO packet (while we don't
4119 * have to) in order to save some CPU cycles in a none-LSO
4120 * case, when we much more care about them.
4122 if (total_pkt_bd != NULL)
4123 total_pkt_bd->total_pkt_bytes = pkt_size;
4126 DP(NETIF_MSG_TX_QUEUED,
4127 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u tcp_flags %x xsum %x seq %u hlen %u\n",
4128 pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
4129 pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
4130 pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
4131 le16_to_cpu(pbd_e1x->total_hlen_w));
4133 DP(NETIF_MSG_TX_QUEUED,
4134 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n",
4136 pbd_e2->data.mac_addr.dst_hi,
4137 pbd_e2->data.mac_addr.dst_mid,
4138 pbd_e2->data.mac_addr.dst_lo,
4139 pbd_e2->data.mac_addr.src_hi,
4140 pbd_e2->data.mac_addr.src_mid,
4141 pbd_e2->data.mac_addr.src_lo,
4142 pbd_e2->parsing_data);
4143 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
4145 netdev_tx_sent_queue(txq, skb->len);
4147 skb_tx_timestamp(skb);
4149 txdata->tx_pkt_prod++;
4151 * Make sure that the BD data is updated before updating the producer
4152 * since FW might read the BD right after the producer is updated.
4153 * This is only applicable for weak-ordered memory model archs such
4154 * as IA-64. The following barrier is also mandatory since FW will
4155 * assumes packets must have BDs.
4159 txdata->tx_db.data.prod += nbd;
4162 DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
4166 txdata->tx_bd_prod += nbd;
4168 if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4169 netif_tx_stop_queue(txq);
4171 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
4172 * ordering of set_bit() in netif_tx_stop_queue() and read of
4176 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4177 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4178 netif_tx_wake_queue(txq);
4182 return NETDEV_TX_OK;
4186 * bnx2x_setup_tc - routine to configure net_device for multi tc
4188 * @netdev: net device to configure
4189 * @tc: number of traffic classes to enable
4191 * callback connected to the ndo_setup_tc function pointer
4193 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4195 int cos, prio, count, offset;
4196 struct bnx2x *bp = netdev_priv(dev);
4198 /* setup tc must be called under rtnl lock */
4201 /* no traffic classes requested. Aborting */
4203 netdev_reset_tc(dev);
4207 /* requested to support too many traffic classes */
4208 if (num_tc > bp->max_cos) {
4209 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4210 num_tc, bp->max_cos);
4214 /* declare amount of supported traffic classes */
4215 if (netdev_set_num_tc(dev, num_tc)) {
4216 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4220 /* configure priority to traffic class mapping */
4221 for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4222 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
4223 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4224 "mapping priority %d to tc %d\n",
4225 prio, bp->prio_to_cos[prio]);
4228 /* Use this configuration to differentiate tc0 from other COSes
4229 This can be used for ets or pfc, and save the effort of setting
4230 up a multio class queue disc or negotiating DCBX with a switch
4231 netdev_set_prio_tc_map(dev, 0, 0);
4232 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4233 for (prio = 1; prio < 16; prio++) {
4234 netdev_set_prio_tc_map(dev, prio, 1);
4235 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4238 /* configure traffic class to transmission queue mapping */
4239 for (cos = 0; cos < bp->max_cos; cos++) {
4240 count = BNX2X_NUM_ETH_QUEUES(bp);
4241 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4242 netdev_set_tc_queue(dev, cos, count, offset);
4243 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4244 "mapping tc %d to offset %d count %d\n",
4245 cos, offset, count);
4251 /* called with rtnl_lock */
4252 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4254 struct sockaddr *addr = p;
4255 struct bnx2x *bp = netdev_priv(dev);
4258 if (!is_valid_ether_addr(addr->sa_data)) {
4259 BNX2X_ERR("Requested MAC address is not valid\n");
4263 if (IS_MF_STORAGE_ONLY(bp)) {
4264 BNX2X_ERR("Can't change address on STORAGE ONLY function\n");
4268 if (netif_running(dev)) {
4269 rc = bnx2x_set_eth_mac(bp, false);
4274 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4276 if (netif_running(dev))
4277 rc = bnx2x_set_eth_mac(bp, true);
4282 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4284 union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4285 struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4290 if (IS_FCOE_IDX(fp_index)) {
4291 memset(sb, 0, sizeof(union host_hc_status_block));
4292 fp->status_blk_mapping = 0;
4295 if (!CHIP_IS_E1x(bp))
4296 BNX2X_PCI_FREE(sb->e2_sb,
4297 bnx2x_fp(bp, fp_index,
4298 status_blk_mapping),
4299 sizeof(struct host_hc_status_block_e2));
4301 BNX2X_PCI_FREE(sb->e1x_sb,
4302 bnx2x_fp(bp, fp_index,
4303 status_blk_mapping),
4304 sizeof(struct host_hc_status_block_e1x));
4308 if (!skip_rx_queue(bp, fp_index)) {
4309 bnx2x_free_rx_bds(fp);
4311 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4312 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4313 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4314 bnx2x_fp(bp, fp_index, rx_desc_mapping),
4315 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4317 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4318 bnx2x_fp(bp, fp_index, rx_comp_mapping),
4319 sizeof(struct eth_fast_path_rx_cqe) *
4323 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4324 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4325 bnx2x_fp(bp, fp_index, rx_sge_mapping),
4326 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4330 if (!skip_tx_queue(bp, fp_index)) {
4331 /* fastpath tx rings: tx_buf tx_desc */
4332 for_each_cos_in_tx_queue(fp, cos) {
4333 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4335 DP(NETIF_MSG_IFDOWN,
4336 "freeing tx memory of fp %d cos %d cid %d\n",
4337 fp_index, cos, txdata->cid);
4339 BNX2X_FREE(txdata->tx_buf_ring);
4340 BNX2X_PCI_FREE(txdata->tx_desc_ring,
4341 txdata->tx_desc_mapping,
4342 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4345 /* end of fastpath */
4348 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4351 for_each_cnic_queue(bp, i)
4352 bnx2x_free_fp_mem_at(bp, i);
4355 void bnx2x_free_fp_mem(struct bnx2x *bp)
4358 for_each_eth_queue(bp, i)
4359 bnx2x_free_fp_mem_at(bp, i);
4362 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4364 union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4365 if (!CHIP_IS_E1x(bp)) {
4366 bnx2x_fp(bp, index, sb_index_values) =
4367 (__le16 *)status_blk.e2_sb->sb.index_values;
4368 bnx2x_fp(bp, index, sb_running_index) =
4369 (__le16 *)status_blk.e2_sb->sb.running_index;
4371 bnx2x_fp(bp, index, sb_index_values) =
4372 (__le16 *)status_blk.e1x_sb->sb.index_values;
4373 bnx2x_fp(bp, index, sb_running_index) =
4374 (__le16 *)status_blk.e1x_sb->sb.running_index;
4378 /* Returns the number of actually allocated BDs */
4379 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4382 struct bnx2x *bp = fp->bp;
4383 u16 ring_prod, cqe_ring_prod;
4384 int i, failure_cnt = 0;
4386 fp->rx_comp_cons = 0;
4387 cqe_ring_prod = ring_prod = 0;
4389 /* This routine is called only during fo init so
4390 * fp->eth_q_stats.rx_skb_alloc_failed = 0
4392 for (i = 0; i < rx_ring_size; i++) {
4393 if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4397 ring_prod = NEXT_RX_IDX(ring_prod);
4398 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4399 WARN_ON(ring_prod <= (i - failure_cnt));
4403 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4404 i - failure_cnt, fp->index);
4406 fp->rx_bd_prod = ring_prod;
4407 /* Limit the CQE producer by the CQE ring size */
4408 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4410 fp->rx_pkt = fp->rx_calls = 0;
4412 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4414 return i - failure_cnt;
4417 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4421 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4422 struct eth_rx_cqe_next_page *nextpg;
4424 nextpg = (struct eth_rx_cqe_next_page *)
4425 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4427 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4428 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4430 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4431 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4435 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4437 union host_hc_status_block *sb;
4438 struct bnx2x_fastpath *fp = &bp->fp[index];
4441 int rx_ring_size = 0;
4443 if (!bp->rx_ring_size && IS_MF_STORAGE_ONLY(bp)) {
4444 rx_ring_size = MIN_RX_SIZE_NONTPA;
4445 bp->rx_ring_size = rx_ring_size;
4446 } else if (!bp->rx_ring_size) {
4447 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4449 if (CHIP_IS_E3(bp)) {
4450 u32 cfg = SHMEM_RD(bp,
4451 dev_info.port_hw_config[BP_PORT(bp)].
4454 /* Decrease ring size for 1G functions */
4455 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4456 PORT_HW_CFG_NET_SERDES_IF_SGMII)
4460 /* allocate at least number of buffers required by FW */
4461 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4462 MIN_RX_SIZE_TPA, rx_ring_size);
4464 bp->rx_ring_size = rx_ring_size;
4465 } else /* if rx_ring_size specified - use it */
4466 rx_ring_size = bp->rx_ring_size;
4468 DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4471 sb = &bnx2x_fp(bp, index, status_blk);
4473 if (!IS_FCOE_IDX(index)) {
4475 if (!CHIP_IS_E1x(bp)) {
4476 sb->e2_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4477 sizeof(struct host_hc_status_block_e2));
4481 sb->e1x_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4482 sizeof(struct host_hc_status_block_e1x));
4488 /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4489 * set shortcuts for it.
4491 if (!IS_FCOE_IDX(index))
4492 set_sb_shortcuts(bp, index);
4495 if (!skip_tx_queue(bp, index)) {
4496 /* fastpath tx rings: tx_buf tx_desc */
4497 for_each_cos_in_tx_queue(fp, cos) {
4498 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4501 "allocating tx memory of fp %d cos %d\n",
4504 txdata->tx_buf_ring = kcalloc(NUM_TX_BD,
4505 sizeof(struct sw_tx_bd),
4507 if (!txdata->tx_buf_ring)
4509 txdata->tx_desc_ring = BNX2X_PCI_ALLOC(&txdata->tx_desc_mapping,
4510 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4511 if (!txdata->tx_desc_ring)
4517 if (!skip_rx_queue(bp, index)) {
4518 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4519 bnx2x_fp(bp, index, rx_buf_ring) =
4520 kcalloc(NUM_RX_BD, sizeof(struct sw_rx_bd), GFP_KERNEL);
4521 if (!bnx2x_fp(bp, index, rx_buf_ring))
4523 bnx2x_fp(bp, index, rx_desc_ring) =
4524 BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_desc_mapping),
4525 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4526 if (!bnx2x_fp(bp, index, rx_desc_ring))
4529 /* Seed all CQEs by 1s */
4530 bnx2x_fp(bp, index, rx_comp_ring) =
4531 BNX2X_PCI_FALLOC(&bnx2x_fp(bp, index, rx_comp_mapping),
4532 sizeof(struct eth_fast_path_rx_cqe) * NUM_RCQ_BD);
4533 if (!bnx2x_fp(bp, index, rx_comp_ring))
4537 bnx2x_fp(bp, index, rx_page_ring) =
4538 kcalloc(NUM_RX_SGE, sizeof(struct sw_rx_page),
4540 if (!bnx2x_fp(bp, index, rx_page_ring))
4542 bnx2x_fp(bp, index, rx_sge_ring) =
4543 BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_sge_mapping),
4544 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4545 if (!bnx2x_fp(bp, index, rx_sge_ring))
4548 bnx2x_set_next_page_rx_bd(fp);
4551 bnx2x_set_next_page_rx_cq(fp);
4554 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4555 if (ring_size < rx_ring_size)
4561 /* handles low memory cases */
4563 BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4565 /* FW will drop all packets if queue is not big enough,
4566 * In these cases we disable the queue
4567 * Min size is different for OOO, TPA and non-TPA queues
4569 if (ring_size < (fp->mode == TPA_MODE_DISABLED ?
4570 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4571 /* release memory allocated for this queue */
4572 bnx2x_free_fp_mem_at(bp, index);
4578 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4582 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4583 /* we will fail load process instead of mark
4591 static int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4595 /* 1. Allocate FP for leading - fatal if error
4596 * 2. Allocate RSS - fix number of queues if error
4600 if (bnx2x_alloc_fp_mem_at(bp, 0))
4604 for_each_nondefault_eth_queue(bp, i)
4605 if (bnx2x_alloc_fp_mem_at(bp, i))
4608 /* handle memory failures */
4609 if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4610 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4613 bnx2x_shrink_eth_fp(bp, delta);
4614 if (CNIC_SUPPORT(bp))
4615 /* move non eth FPs next to last eth FP
4616 * must be done in that order
4617 * FCOE_IDX < FWD_IDX < OOO_IDX
4620 /* move FCoE fp even NO_FCOE_FLAG is on */
4621 bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4622 bp->num_ethernet_queues -= delta;
4623 bp->num_queues = bp->num_ethernet_queues +
4624 bp->num_cnic_queues;
4625 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4626 bp->num_queues + delta, bp->num_queues);
4632 void bnx2x_free_mem_bp(struct bnx2x *bp)
4636 for (i = 0; i < bp->fp_array_size; i++)
4637 kfree(bp->fp[i].tpa_info);
4640 kfree(bp->fp_stats);
4641 kfree(bp->bnx2x_txq);
4642 kfree(bp->msix_table);
4646 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4648 struct bnx2x_fastpath *fp;
4649 struct msix_entry *tbl;
4650 struct bnx2x_ilt *ilt;
4651 int msix_table_size = 0;
4652 int fp_array_size, txq_array_size;
4656 * The biggest MSI-X table we might need is as a maximum number of fast
4657 * path IGU SBs plus default SB (for PF only).
4659 msix_table_size = bp->igu_sb_cnt;
4662 BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4664 /* fp array: RSS plus CNIC related L2 queues */
4665 fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4666 bp->fp_array_size = fp_array_size;
4667 BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4669 fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4672 for (i = 0; i < bp->fp_array_size; i++) {
4674 kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4675 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4676 if (!(fp[i].tpa_info))
4682 /* allocate sp objs */
4683 bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4688 /* allocate fp_stats */
4689 bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4694 /* Allocate memory for the transmission queues array */
4696 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4697 BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4699 bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4705 tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4708 bp->msix_table = tbl;
4711 ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4718 bnx2x_free_mem_bp(bp);
4722 int bnx2x_reload_if_running(struct net_device *dev)
4724 struct bnx2x *bp = netdev_priv(dev);
4726 if (unlikely(!netif_running(dev)))
4729 bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4730 return bnx2x_nic_load(bp, LOAD_NORMAL);
4733 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4735 u32 sel_phy_idx = 0;
4736 if (bp->link_params.num_phys <= 1)
4739 if (bp->link_vars.link_up) {
4740 sel_phy_idx = EXT_PHY1;
4741 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4742 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4743 (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4744 sel_phy_idx = EXT_PHY2;
4747 switch (bnx2x_phy_selection(&bp->link_params)) {
4748 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4749 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4750 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4751 sel_phy_idx = EXT_PHY1;
4753 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4754 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4755 sel_phy_idx = EXT_PHY2;
4762 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4764 u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4766 * The selected activated PHY is always after swapping (in case PHY
4767 * swapping is enabled). So when swapping is enabled, we need to reverse
4771 if (bp->link_params.multi_phy_config &
4772 PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4773 if (sel_phy_idx == EXT_PHY1)
4774 sel_phy_idx = EXT_PHY2;
4775 else if (sel_phy_idx == EXT_PHY2)
4776 sel_phy_idx = EXT_PHY1;
4778 return LINK_CONFIG_IDX(sel_phy_idx);
4781 #ifdef NETDEV_FCOE_WWNN
4782 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4784 struct bnx2x *bp = netdev_priv(dev);
4785 struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4788 case NETDEV_FCOE_WWNN:
4789 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4790 cp->fcoe_wwn_node_name_lo);
4792 case NETDEV_FCOE_WWPN:
4793 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4794 cp->fcoe_wwn_port_name_lo);
4797 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4805 /* called with rtnl_lock */
4806 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4808 struct bnx2x *bp = netdev_priv(dev);
4810 if (pci_num_vf(bp->pdev)) {
4811 DP(BNX2X_MSG_IOV, "VFs are enabled, can not change MTU\n");
4815 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4816 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4820 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4821 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4822 BNX2X_ERR("Can't support requested MTU size\n");
4826 /* This does not race with packet allocation
4827 * because the actual alloc size is
4828 * only updated as part of load
4832 return bnx2x_reload_if_running(dev);
4835 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4836 netdev_features_t features)
4838 struct bnx2x *bp = netdev_priv(dev);
4840 if (pci_num_vf(bp->pdev)) {
4841 netdev_features_t changed = dev->features ^ features;
4843 /* Revert the requested changes in features if they
4844 * would require internal reload of PF in bnx2x_set_features().
4846 if (!(features & NETIF_F_RXCSUM) && !bp->disable_tpa) {
4847 features &= ~NETIF_F_RXCSUM;
4848 features |= dev->features & NETIF_F_RXCSUM;
4851 if (changed & NETIF_F_LOOPBACK) {
4852 features &= ~NETIF_F_LOOPBACK;
4853 features |= dev->features & NETIF_F_LOOPBACK;
4857 /* TPA requires Rx CSUM offloading */
4858 if (!(features & NETIF_F_RXCSUM)) {
4859 features &= ~NETIF_F_LRO;
4860 features &= ~NETIF_F_GRO;
4866 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4868 struct bnx2x *bp = netdev_priv(dev);
4869 netdev_features_t changes = features ^ dev->features;
4870 bool bnx2x_reload = false;
4873 /* VFs or non SRIOV PFs should be able to change loopback feature */
4874 if (!pci_num_vf(bp->pdev)) {
4875 if (features & NETIF_F_LOOPBACK) {
4876 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4877 bp->link_params.loopback_mode = LOOPBACK_BMAC;
4878 bnx2x_reload = true;
4881 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4882 bp->link_params.loopback_mode = LOOPBACK_NONE;
4883 bnx2x_reload = true;
4888 /* if GRO is changed while LRO is enabled, don't force a reload */
4889 if ((changes & NETIF_F_GRO) && (features & NETIF_F_LRO))
4890 changes &= ~NETIF_F_GRO;
4892 /* if GRO is changed while HW TPA is off, don't force a reload */
4893 if ((changes & NETIF_F_GRO) && bp->disable_tpa)
4894 changes &= ~NETIF_F_GRO;
4897 bnx2x_reload = true;
4900 if (bp->recovery_state == BNX2X_RECOVERY_DONE) {
4901 dev->features = features;
4902 rc = bnx2x_reload_if_running(dev);
4905 /* else: bnx2x_nic_load() will be called at end of recovery */
4911 void bnx2x_tx_timeout(struct net_device *dev)
4913 struct bnx2x *bp = netdev_priv(dev);
4915 #ifdef BNX2X_STOP_ON_ERROR
4920 /* This allows the netif to be shutdown gracefully before resetting */
4921 bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_TX_TIMEOUT, 0);
4924 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4926 struct net_device *dev = pci_get_drvdata(pdev);
4930 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4933 bp = netdev_priv(dev);
4937 pci_save_state(pdev);
4939 if (!netif_running(dev)) {
4944 netif_device_detach(dev);
4946 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4948 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4955 int bnx2x_resume(struct pci_dev *pdev)
4957 struct net_device *dev = pci_get_drvdata(pdev);
4962 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4965 bp = netdev_priv(dev);
4967 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4968 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4974 pci_restore_state(pdev);
4976 if (!netif_running(dev)) {
4981 bnx2x_set_power_state(bp, PCI_D0);
4982 netif_device_attach(dev);
4984 rc = bnx2x_nic_load(bp, LOAD_OPEN);
4991 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4995 BNX2X_ERR("bad context pointer %p\n", cxt);
4999 /* ustorm cxt validation */
5000 cxt->ustorm_ag_context.cdu_usage =
5001 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
5002 CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
5003 /* xcontext validation */
5004 cxt->xstorm_ag_context.cdu_reserved =
5005 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
5006 CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
5009 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
5010 u8 fw_sb_id, u8 sb_index,
5013 u32 addr = BAR_CSTRORM_INTMEM +
5014 CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
5015 REG_WR8(bp, addr, ticks);
5017 "port %x fw_sb_id %d sb_index %d ticks %d\n",
5018 port, fw_sb_id, sb_index, ticks);
5021 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
5022 u16 fw_sb_id, u8 sb_index,
5025 u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
5026 u32 addr = BAR_CSTRORM_INTMEM +
5027 CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
5028 u8 flags = REG_RD8(bp, addr);
5030 flags &= ~HC_INDEX_DATA_HC_ENABLED;
5031 flags |= enable_flag;
5032 REG_WR8(bp, addr, flags);
5034 "port %x fw_sb_id %d sb_index %d disable %d\n",
5035 port, fw_sb_id, sb_index, disable);
5038 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
5039 u8 sb_index, u8 disable, u16 usec)
5041 int port = BP_PORT(bp);
5042 u8 ticks = usec / BNX2X_BTR;
5044 storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
5046 disable = disable ? 1 : (usec ? 0 : 1);
5047 storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
5050 void bnx2x_schedule_sp_rtnl(struct bnx2x *bp, enum sp_rtnl_flag flag,
5053 smp_mb__before_atomic();
5054 set_bit(flag, &bp->sp_rtnl_state);
5055 smp_mb__after_atomic();
5056 DP((BNX2X_MSG_SP | verbose), "Scheduling sp_rtnl task [Flag: %d]\n",
5058 schedule_delayed_work(&bp->sp_rtnl_task, 0);
5060 EXPORT_SYMBOL(bnx2x_schedule_sp_rtnl);