drivers/net/ethernet/wangxun/libwx/wx_lib.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. */
   3
   4 #include <linux/etherdevice.h>
   5 #include <net/ip6_checksum.h>
   6 #include <net/page_pool/helpers.h>
   7 #include <net/inet_ecn.h>
   8 #include <linux/iopoll.h>
   9 #include <linux/sctp.h>
  10 #include <linux/pci.h>
  11 #include <net/tcp.h>
  12 #include <net/ip.h>
  13
  14 #include "wx_type.h"
  15 #include "wx_lib.h"
  16 #include "wx_hw.h"
  17
  18 /* Lookup table mapping the HW PTYPE to the bit field for decoding */
  19 static struct wx_dec_ptype wx_ptype_lookup[256] = {
  20         /* L2: mac */
  21         [0x11] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
  22         [0x12] = WX_PTT(L2, NONE, NONE, NONE, TS,   PAY2),
  23         [0x13] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
  24         [0x14] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
  25         [0x15] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
  26         [0x16] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
  27         [0x17] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
  28
  29         /* L2: ethertype filter */
  30         [0x18 ... 0x1F] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
  31
  32         /* L3: ip non-tunnel */
  33         [0x21] = WX_PTT(IP, FGV4, NONE, NONE, NONE, PAY3),
  34         [0x22] = WX_PTT(IP, IPV4, NONE, NONE, NONE, PAY3),
  35         [0x23] = WX_PTT(IP, IPV4, NONE, NONE, UDP,  PAY4),
  36         [0x24] = WX_PTT(IP, IPV4, NONE, NONE, TCP,  PAY4),
  37         [0x25] = WX_PTT(IP, IPV4, NONE, NONE, SCTP, PAY4),
  38         [0x29] = WX_PTT(IP, FGV6, NONE, NONE, NONE, PAY3),
  39         [0x2A] = WX_PTT(IP, IPV6, NONE, NONE, NONE, PAY3),
  40         [0x2B] = WX_PTT(IP, IPV6, NONE, NONE, UDP,  PAY3),
  41         [0x2C] = WX_PTT(IP, IPV6, NONE, NONE, TCP,  PAY4),
  42         [0x2D] = WX_PTT(IP, IPV6, NONE, NONE, SCTP, PAY4),
  43
  44         /* L2: fcoe */
  45         [0x30 ... 0x34] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),
  46         [0x38 ... 0x3C] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),
  47
  48         /* IPv4 --> IPv4/IPv6 */
  49         [0x81] = WX_PTT(IP, IPV4, IPIP, FGV4, NONE, PAY3),
  50         [0x82] = WX_PTT(IP, IPV4, IPIP, IPV4, NONE, PAY3),
  51         [0x83] = WX_PTT(IP, IPV4, IPIP, IPV4, UDP,  PAY4),
  52         [0x84] = WX_PTT(IP, IPV4, IPIP, IPV4, TCP,  PAY4),
  53         [0x85] = WX_PTT(IP, IPV4, IPIP, IPV4, SCTP, PAY4),
  54         [0x89] = WX_PTT(IP, IPV4, IPIP, FGV6, NONE, PAY3),
  55         [0x8A] = WX_PTT(IP, IPV4, IPIP, IPV6, NONE, PAY3),
  56         [0x8B] = WX_PTT(IP, IPV4, IPIP, IPV6, UDP,  PAY4),
  57         [0x8C] = WX_PTT(IP, IPV4, IPIP, IPV6, TCP,  PAY4),
  58         [0x8D] = WX_PTT(IP, IPV4, IPIP, IPV6, SCTP, PAY4),
  59
  60         /* IPv4 --> GRE/NAT --> NONE/IPv4/IPv6 */
  61         [0x90] = WX_PTT(IP, IPV4, IG, NONE, NONE, PAY3),
  62         [0x91] = WX_PTT(IP, IPV4, IG, FGV4, NONE, PAY3),
  63         [0x92] = WX_PTT(IP, IPV4, IG, IPV4, NONE, PAY3),
  64         [0x93] = WX_PTT(IP, IPV4, IG, IPV4, UDP,  PAY4),
  65         [0x94] = WX_PTT(IP, IPV4, IG, IPV4, TCP,  PAY4),
  66         [0x95] = WX_PTT(IP, IPV4, IG, IPV4, SCTP, PAY4),
  67         [0x99] = WX_PTT(IP, IPV4, IG, FGV6, NONE, PAY3),
  68         [0x9A] = WX_PTT(IP, IPV4, IG, IPV6, NONE, PAY3),
  69         [0x9B] = WX_PTT(IP, IPV4, IG, IPV6, UDP,  PAY4),
  70         [0x9C] = WX_PTT(IP, IPV4, IG, IPV6, TCP,  PAY4),
  71         [0x9D] = WX_PTT(IP, IPV4, IG, IPV6, SCTP, PAY4),
  72
  73         /* IPv4 --> GRE/NAT --> MAC --> NONE/IPv4/IPv6 */
  74         [0xA0] = WX_PTT(IP, IPV4, IGM, NONE, NONE, PAY3),
  75         [0xA1] = WX_PTT(IP, IPV4, IGM, FGV4, NONE, PAY3),
  76         [0xA2] = WX_PTT(IP, IPV4, IGM, IPV4, NONE, PAY3),
  77         [0xA3] = WX_PTT(IP, IPV4, IGM, IPV4, UDP,  PAY4),
  78         [0xA4] = WX_PTT(IP, IPV4, IGM, IPV4, TCP,  PAY4),
  79         [0xA5] = WX_PTT(IP, IPV4, IGM, IPV4, SCTP, PAY4),
  80         [0xA9] = WX_PTT(IP, IPV4, IGM, FGV6, NONE, PAY3),
  81         [0xAA] = WX_PTT(IP, IPV4, IGM, IPV6, NONE, PAY3),
  82         [0xAB] = WX_PTT(IP, IPV4, IGM, IPV6, UDP,  PAY4),
  83         [0xAC] = WX_PTT(IP, IPV4, IGM, IPV6, TCP,  PAY4),
  84         [0xAD] = WX_PTT(IP, IPV4, IGM, IPV6, SCTP, PAY4),
  85
  86         /* IPv4 --> GRE/NAT --> MAC+VLAN --> NONE/IPv4/IPv6 */
  87         [0xB0] = WX_PTT(IP, IPV4, IGMV, NONE, NONE, PAY3),
  88         [0xB1] = WX_PTT(IP, IPV4, IGMV, FGV4, NONE, PAY3),
  89         [0xB2] = WX_PTT(IP, IPV4, IGMV, IPV4, NONE, PAY3),
  90         [0xB3] = WX_PTT(IP, IPV4, IGMV, IPV4, UDP,  PAY4),
  91         [0xB4] = WX_PTT(IP, IPV4, IGMV, IPV4, TCP,  PAY4),
  92         [0xB5] = WX_PTT(IP, IPV4, IGMV, IPV4, SCTP, PAY4),
  93         [0xB9] = WX_PTT(IP, IPV4, IGMV, FGV6, NONE, PAY3),
  94         [0xBA] = WX_PTT(IP, IPV4, IGMV, IPV6, NONE, PAY3),
  95         [0xBB] = WX_PTT(IP, IPV4, IGMV, IPV6, UDP,  PAY4),
  96         [0xBC] = WX_PTT(IP, IPV4, IGMV, IPV6, TCP,  PAY4),
  97         [0xBD] = WX_PTT(IP, IPV4, IGMV, IPV6, SCTP, PAY4),
  98
  99         /* IPv6 --> IPv4/IPv6 */
 100         [0xC1] = WX_PTT(IP, IPV6, IPIP, FGV4, NONE, PAY3),
 101         [0xC2] = WX_PTT(IP, IPV6, IPIP, IPV4, NONE, PAY3),
 102         [0xC3] = WX_PTT(IP, IPV6, IPIP, IPV4, UDP,  PAY4),
 103         [0xC4] = WX_PTT(IP, IPV6, IPIP, IPV4, TCP,  PAY4),
 104         [0xC5] = WX_PTT(IP, IPV6, IPIP, IPV4, SCTP, PAY4),
 105         [0xC9] = WX_PTT(IP, IPV6, IPIP, FGV6, NONE, PAY3),
 106         [0xCA] = WX_PTT(IP, IPV6, IPIP, IPV6, NONE, PAY3),
 107         [0xCB] = WX_PTT(IP, IPV6, IPIP, IPV6, UDP,  PAY4),
 108         [0xCC] = WX_PTT(IP, IPV6, IPIP, IPV6, TCP,  PAY4),
 109         [0xCD] = WX_PTT(IP, IPV6, IPIP, IPV6, SCTP, PAY4),
 110
 111         /* IPv6 --> GRE/NAT -> NONE/IPv4/IPv6 */
 112         [0xD0] = WX_PTT(IP, IPV6, IG, NONE, NONE, PAY3),
 113         [0xD1] = WX_PTT(IP, IPV6, IG, FGV4, NONE, PAY3),
 114         [0xD2] = WX_PTT(IP, IPV6, IG, IPV4, NONE, PAY3),
 115         [0xD3] = WX_PTT(IP, IPV6, IG, IPV4, UDP,  PAY4),
 116         [0xD4] = WX_PTT(IP, IPV6, IG, IPV4, TCP,  PAY4),
 117         [0xD5] = WX_PTT(IP, IPV6, IG, IPV4, SCTP, PAY4),
 118         [0xD9] = WX_PTT(IP, IPV6, IG, FGV6, NONE, PAY3),
 119         [0xDA] = WX_PTT(IP, IPV6, IG, IPV6, NONE, PAY3),
 120         [0xDB] = WX_PTT(IP, IPV6, IG, IPV6, UDP,  PAY4),
 121         [0xDC] = WX_PTT(IP, IPV6, IG, IPV6, TCP,  PAY4),
 122         [0xDD] = WX_PTT(IP, IPV6, IG, IPV6, SCTP, PAY4),
 123
 124         /* IPv6 --> GRE/NAT -> MAC -> NONE/IPv4/IPv6 */
 125         [0xE0] = WX_PTT(IP, IPV6, IGM, NONE, NONE, PAY3),
 126         [0xE1] = WX_PTT(IP, IPV6, IGM, FGV4, NONE, PAY3),
 127         [0xE2] = WX_PTT(IP, IPV6, IGM, IPV4, NONE, PAY3),
 128         [0xE3] = WX_PTT(IP, IPV6, IGM, IPV4, UDP,  PAY4),
 129         [0xE4] = WX_PTT(IP, IPV6, IGM, IPV4, TCP,  PAY4),
 130         [0xE5] = WX_PTT(IP, IPV6, IGM, IPV4, SCTP, PAY4),
 131         [0xE9] = WX_PTT(IP, IPV6, IGM, FGV6, NONE, PAY3),
 132         [0xEA] = WX_PTT(IP, IPV6, IGM, IPV6, NONE, PAY3),
 133         [0xEB] = WX_PTT(IP, IPV6, IGM, IPV6, UDP,  PAY4),
 134         [0xEC] = WX_PTT(IP, IPV6, IGM, IPV6, TCP,  PAY4),
 135         [0xED] = WX_PTT(IP, IPV6, IGM, IPV6, SCTP, PAY4),
 136
 137         /* IPv6 --> GRE/NAT -> MAC--> NONE/IPv */
 138         [0xF0] = WX_PTT(IP, IPV6, IGMV, NONE, NONE, PAY3),
 139         [0xF1] = WX_PTT(IP, IPV6, IGMV, FGV4, NONE, PAY3),
 140         [0xF2] = WX_PTT(IP, IPV6, IGMV, IPV4, NONE, PAY3),
 141         [0xF3] = WX_PTT(IP, IPV6, IGMV, IPV4, UDP,  PAY4),
 142         [0xF4] = WX_PTT(IP, IPV6, IGMV, IPV4, TCP,  PAY4),
 143         [0xF5] = WX_PTT(IP, IPV6, IGMV, IPV4, SCTP, PAY4),
 144         [0xF9] = WX_PTT(IP, IPV6, IGMV, FGV6, NONE, PAY3),
 145         [0xFA] = WX_PTT(IP, IPV6, IGMV, IPV6, NONE, PAY3),
 146         [0xFB] = WX_PTT(IP, IPV6, IGMV, IPV6, UDP,  PAY4),
 147         [0xFC] = WX_PTT(IP, IPV6, IGMV, IPV6, TCP,  PAY4),
 148         [0xFD] = WX_PTT(IP, IPV6, IGMV, IPV6, SCTP, PAY4),
 149 };
 150
 151 struct wx_dec_ptype wx_decode_ptype(const u8 ptype)
 152 {
 153         return wx_ptype_lookup[ptype];
 154 }
 155 EXPORT_SYMBOL(wx_decode_ptype);
 156
 157 /* wx_test_staterr - tests bits in Rx descriptor status and error fields */
 158 static __le32 wx_test_staterr(union wx_rx_desc *rx_desc,
 159                               const u32 stat_err_bits)
 160 {
 161         return rx_desc->wb.upper.status_error & cpu_to_le32(stat_err_bits);
 162 }
 163
 164 static void wx_dma_sync_frag(struct wx_ring *rx_ring,
 165                              struct wx_rx_buffer *rx_buffer)
 166 {
 167         struct sk_buff *skb = rx_buffer->skb;
 168         skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
 169
 170         dma_sync_single_range_for_cpu(rx_ring->dev,
 171                                       WX_CB(skb)->dma,
 172                                       skb_frag_off(frag),
 173                                       skb_frag_size(frag),
 174                                       DMA_FROM_DEVICE);
 175
 176         /* If the page was released, just unmap it. */
 177         if (unlikely(WX_CB(skb)->page_released))
 178                 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
 179 }
 180
 181 static struct wx_rx_buffer *wx_get_rx_buffer(struct wx_ring *rx_ring,
 182                                              union wx_rx_desc *rx_desc,
 183                                              struct sk_buff **skb,
 184                                              int *rx_buffer_pgcnt)
 185 {
 186         struct wx_rx_buffer *rx_buffer;
 187         unsigned int size;
 188
 189         rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
 190         size = le16_to_cpu(rx_desc->wb.upper.length);
 191
 192 #if (PAGE_SIZE < 8192)
 193         *rx_buffer_pgcnt = page_count(rx_buffer->page);
 194 #else
 195         *rx_buffer_pgcnt = 0;
 196 #endif
 197
 198         prefetchw(rx_buffer->page);
 199         *skb = rx_buffer->skb;
 200
 201         /* Delay unmapping of the first packet. It carries the header
 202          * information, HW may still access the header after the writeback.
 203          * Only unmap it when EOP is reached
 204          */
 205         if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)) {
 206                 if (!*skb)
 207                         goto skip_sync;
 208         } else {
 209                 if (*skb)
 210                         wx_dma_sync_frag(rx_ring, rx_buffer);
 211         }
 212
 213         /* we are reusing so sync this buffer for CPU use */
 214         dma_sync_single_range_for_cpu(rx_ring->dev,
 215                                       rx_buffer->dma,
 216                                       rx_buffer->page_offset,
 217                                       size,
 218                                       DMA_FROM_DEVICE);
 219 skip_sync:
 220         return rx_buffer;
 221 }
 222
 223 static void wx_put_rx_buffer(struct wx_ring *rx_ring,
 224                              struct wx_rx_buffer *rx_buffer,
 225                              struct sk_buff *skb,
 226                              int rx_buffer_pgcnt)
 227 {
 228         if (!IS_ERR(skb) && WX_CB(skb)->dma == rx_buffer->dma)
 229                 /* the page has been released from the ring */
 230                 WX_CB(skb)->page_released = true;
 231
 232         /* clear contents of rx_buffer */
 233         rx_buffer->page = NULL;
 234         rx_buffer->skb = NULL;
 235 }
 236
 237 static struct sk_buff *wx_build_skb(struct wx_ring *rx_ring,
 238                                     struct wx_rx_buffer *rx_buffer,
 239                                     union wx_rx_desc *rx_desc)
 240 {
 241         unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
 242 #if (PAGE_SIZE < 8192)
 243         unsigned int truesize = WX_RX_BUFSZ;
 244 #else
 245         unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
 246 #endif
 247         struct sk_buff *skb = rx_buffer->skb;
 248
 249         if (!skb) {
 250                 void *page_addr = page_address(rx_buffer->page) +
 251                                   rx_buffer->page_offset;
 252
 253                 /* prefetch first cache line of first page */
 254                 net_prefetch(page_addr);
 255
 256                 /* allocate a skb to store the frags */
 257                 skb = napi_alloc_skb(&rx_ring->q_vector->napi, WX_RXBUFFER_256);
 258                 if (unlikely(!skb))
 259                         return NULL;
 260
 261                 /* we will be copying header into skb->data in
 262                  * pskb_may_pull so it is in our interest to prefetch
 263                  * it now to avoid a possible cache miss
 264                  */
 265                 prefetchw(skb->data);
 266
 267                 if (size <= WX_RXBUFFER_256) {
 268                         memcpy(__skb_put(skb, size), page_addr,
 269                                ALIGN(size, sizeof(long)));
 270                         page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, true);
 271                         return skb;
 272                 }
 273
 274                 skb_mark_for_recycle(skb);
 275
 276                 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP))
 277                         WX_CB(skb)->dma = rx_buffer->dma;
 278
 279                 skb_add_rx_frag(skb, 0, rx_buffer->page,
 280                                 rx_buffer->page_offset,
 281                                 size, truesize);
 282                 goto out;
 283
 284         } else {
 285                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
 286                                 rx_buffer->page_offset, size, truesize);
 287         }
 288
 289 out:
 290 #if (PAGE_SIZE < 8192)
 291         /* flip page offset to other buffer */
 292         rx_buffer->page_offset ^= truesize;
 293 #else
 294         /* move offset up to the next cache line */
 295         rx_buffer->page_offset += truesize;
 296 #endif
 297
 298         return skb;
 299 }
 300
 301 static bool wx_alloc_mapped_page(struct wx_ring *rx_ring,
 302                                  struct wx_rx_buffer *bi)
 303 {
 304         struct page *page = bi->page;
 305         dma_addr_t dma;
 306
 307         /* since we are recycling buffers we should seldom need to alloc */
 308         if (likely(page))
 309                 return true;
 310
 311         page = page_pool_dev_alloc_pages(rx_ring->page_pool);
 312         WARN_ON(!page);
 313         dma = page_pool_get_dma_addr(page);
 314
 315         bi->page_dma = dma;
 316         bi->page = page;
 317         bi->page_offset = 0;
 318
 319         return true;
 320 }
 321
 322 /**
 323  * wx_alloc_rx_buffers - Replace used receive buffers
 324  * @rx_ring: ring to place buffers on
 325  * @cleaned_count: number of buffers to replace
 326  **/
 327 void wx_alloc_rx_buffers(struct wx_ring *rx_ring, u16 cleaned_count)
 328 {
 329         u16 i = rx_ring->next_to_use;
 330         union wx_rx_desc *rx_desc;
 331         struct wx_rx_buffer *bi;
 332
 333         /* nothing to do */
 334         if (!cleaned_count)
 335                 return;
 336
 337         rx_desc = WX_RX_DESC(rx_ring, i);
 338         bi = &rx_ring->rx_buffer_info[i];
 339         i -= rx_ring->count;
 340
 341         do {
 342                 if (!wx_alloc_mapped_page(rx_ring, bi))
 343                         break;
 344
 345                 /* sync the buffer for use by the device */
 346                 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
 347                                                  bi->page_offset,
 348                                                  WX_RX_BUFSZ,
 349                                                  DMA_FROM_DEVICE);
 350
 351                 rx_desc->read.pkt_addr =
 352                         cpu_to_le64(bi->page_dma + bi->page_offset);
 353
 354                 rx_desc++;
 355                 bi++;
 356                 i++;
 357                 if (unlikely(!i)) {
 358                         rx_desc = WX_RX_DESC(rx_ring, 0);
 359                         bi = rx_ring->rx_buffer_info;
 360                         i -= rx_ring->count;
 361                 }
 362
 363                 /* clear the status bits for the next_to_use descriptor */
 364                 rx_desc->wb.upper.status_error = 0;
 365
 366                 cleaned_count--;
 367         } while (cleaned_count);
 368
 369         i += rx_ring->count;
 370
 371         if (rx_ring->next_to_use != i) {
 372                 rx_ring->next_to_use = i;
 373                 /* update next to alloc since we have filled the ring */
 374                 rx_ring->next_to_alloc = i;
 375
 376                 /* Force memory writes to complete before letting h/w
 377                  * know there are new descriptors to fetch.  (Only
 378                  * applicable for weak-ordered memory model archs,
 379                  * such as IA-64).
 380                  */
 381                 wmb();
 382                 writel(i, rx_ring->tail);
 383         }
 384 }
 385
 386 u16 wx_desc_unused(struct wx_ring *ring)
 387 {
 388         u16 ntc = ring->next_to_clean;
 389         u16 ntu = ring->next_to_use;
 390
 391         return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1;
 392 }
 393
 394 /**
 395  * wx_is_non_eop - process handling of non-EOP buffers
 396  * @rx_ring: Rx ring being processed
 397  * @rx_desc: Rx descriptor for current buffer
 398  * @skb: Current socket buffer containing buffer in progress
 399  *
 400  * This function updates next to clean. If the buffer is an EOP buffer
 401  * this function exits returning false, otherwise it will place the
 402  * sk_buff in the next buffer to be chained and return true indicating
 403  * that this is in fact a non-EOP buffer.
 404  **/
 405 static bool wx_is_non_eop(struct wx_ring *rx_ring,
 406                           union wx_rx_desc *rx_desc,
 407                           struct sk_buff *skb)
 408 {
 409         u32 ntc = rx_ring->next_to_clean + 1;
 410
 411         /* fetch, update, and store next to clean */
 412         ntc = (ntc < rx_ring->count) ? ntc : 0;
 413         rx_ring->next_to_clean = ntc;
 414
 415         prefetch(WX_RX_DESC(rx_ring, ntc));
 416
 417         /* if we are the last buffer then there is nothing else to do */
 418         if (likely(wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)))
 419                 return false;
 420
 421         rx_ring->rx_buffer_info[ntc].skb = skb;
 422         rx_ring->rx_stats.non_eop_descs++;
 423
 424         return true;
 425 }
 426
 427 static void wx_pull_tail(struct sk_buff *skb)
 428 {
 429         skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
 430         unsigned int pull_len;
 431         unsigned char *va;
 432
 433         /* it is valid to use page_address instead of kmap since we are
 434          * working with pages allocated out of the lomem pool per
 435          * alloc_page(GFP_ATOMIC)
 436          */
 437         va = skb_frag_address(frag);
 438
 439         /* we need the header to contain the greater of either ETH_HLEN or
 440          * 60 bytes if the skb->len is less than 60 for skb_pad.
 441          */
 442         pull_len = eth_get_headlen(skb->dev, va, WX_RXBUFFER_256);
 443
 444         /* align pull length to size of long to optimize memcpy performance */
 445         skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
 446
 447         /* update all of the pointers */
 448         skb_frag_size_sub(frag, pull_len);
 449         skb_frag_off_add(frag, pull_len);
 450         skb->data_len -= pull_len;
 451         skb->tail += pull_len;
 452 }
 453
 454 /**
 455  * wx_cleanup_headers - Correct corrupted or empty headers
 456  * @rx_ring: rx descriptor ring packet is being transacted on
 457  * @rx_desc: pointer to the EOP Rx descriptor
 458  * @skb: pointer to current skb being fixed
 459  *
 460  * Check for corrupted packet headers caused by senders on the local L2
 461  * embedded NIC switch not setting up their Tx Descriptors right.  These
 462  * should be very rare.
 463  *
 464  * Also address the case where we are pulling data in on pages only
 465  * and as such no data is present in the skb header.
 466  *
 467  * In addition if skb is not at least 60 bytes we need to pad it so that
 468  * it is large enough to qualify as a valid Ethernet frame.
 469  *
 470  * Returns true if an error was encountered and skb was freed.
 471  **/
 472 static bool wx_cleanup_headers(struct wx_ring *rx_ring,
 473                                union wx_rx_desc *rx_desc,
 474                                struct sk_buff *skb)
 475 {
 476         struct net_device *netdev = rx_ring->netdev;
 477
 478         /* verify that the packet does not have any known errors */
 479         if (!netdev ||
 480             unlikely(wx_test_staterr(rx_desc, WX_RXD_ERR_RXE) &&
 481                      !(netdev->features & NETIF_F_RXALL))) {
 482                 dev_kfree_skb_any(skb);
 483                 return true;
 484         }
 485
 486         /* place header in linear portion of buffer */
 487         if (!skb_headlen(skb))
 488                 wx_pull_tail(skb);
 489
 490         /* if eth_skb_pad returns an error the skb was freed */
 491         if (eth_skb_pad(skb))
 492                 return true;
 493
 494         return false;
 495 }
 496
 497 static void wx_rx_hash(struct wx_ring *ring,
 498                        union wx_rx_desc *rx_desc,
 499                        struct sk_buff *skb)
 500 {
 501         u16 rss_type;
 502
 503         if (!(ring->netdev->features & NETIF_F_RXHASH))
 504                 return;
 505
 506         rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
 507                                WX_RXD_RSSTYPE_MASK;
 508
 509         if (!rss_type)
 510                 return;
 511
 512         skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
 513                      (WX_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
 514                      PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
 515 }
 516
 517 /**
 518  * wx_rx_checksum - indicate in skb if hw indicated a good cksum
 519  * @ring: structure containing ring specific data
 520  * @rx_desc: current Rx descriptor being processed
 521  * @skb: skb currently being received and modified
 522  **/
 523 static void wx_rx_checksum(struct wx_ring *ring,
 524                            union wx_rx_desc *rx_desc,
 525                            struct sk_buff *skb)
 526 {
 527         struct wx_dec_ptype dptype = wx_decode_ptype(WX_RXD_PKTTYPE(rx_desc));
 528
 529         skb_checksum_none_assert(skb);
 530         /* Rx csum disabled */
 531         if (!(ring->netdev->features & NETIF_F_RXCSUM))
 532                 return;
 533
 534         /* if IPv4 header checksum error */
 535         if ((wx_test_staterr(rx_desc, WX_RXD_STAT_IPCS) &&
 536              wx_test_staterr(rx_desc, WX_RXD_ERR_IPE)) ||
 537             (wx_test_staterr(rx_desc, WX_RXD_STAT_OUTERIPCS) &&
 538              wx_test_staterr(rx_desc, WX_RXD_ERR_OUTERIPER))) {
 539                 ring->rx_stats.csum_err++;
 540                 return;
 541         }
 542
 543         /* L4 checksum offload flag must set for the below code to work */
 544         if (!wx_test_staterr(rx_desc, WX_RXD_STAT_L4CS))
 545                 return;
 546
 547         /* Hardware can't guarantee csum if IPv6 Dest Header found */
 548         if (dptype.prot != WX_DEC_PTYPE_PROT_SCTP && WX_RXD_IPV6EX(rx_desc))
 549                 return;
 550
 551         /* if L4 checksum error */
 552         if (wx_test_staterr(rx_desc, WX_RXD_ERR_TCPE)) {
 553                 ring->rx_stats.csum_err++;
 554                 return;
 555         }
 556
 557         /* It must be a TCP or UDP or SCTP packet with a valid checksum */
 558         skb->ip_summed = CHECKSUM_UNNECESSARY;
 559
 560         /* If there is an outer header present that might contain a checksum
 561          * we need to bump the checksum level by 1 to reflect the fact that
 562          * we are indicating we validated the inner checksum.
 563          */
 564         if (dptype.etype >= WX_DEC_PTYPE_ETYPE_IG)
 565                 __skb_incr_checksum_unnecessary(skb);
 566         ring->rx_stats.csum_good_cnt++;
 567 }
 568
 569 static void wx_rx_vlan(struct wx_ring *ring, union wx_rx_desc *rx_desc,
 570                        struct sk_buff *skb)
 571 {
 572         u16 ethertype;
 573         u8 idx = 0;
 574
 575         if ((ring->netdev->features &
 576              (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) &&
 577             wx_test_staterr(rx_desc, WX_RXD_STAT_VP)) {
 578                 idx = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
 579                        0x1c0) >> 6;
 580                 ethertype = ring->q_vector->wx->tpid[idx];
 581                 __vlan_hwaccel_put_tag(skb, htons(ethertype),
 582                                        le16_to_cpu(rx_desc->wb.upper.vlan));
 583         }
 584 }
 585
 586 /**
 587  * wx_process_skb_fields - Populate skb header fields from Rx descriptor
 588  * @rx_ring: rx descriptor ring packet is being transacted on
 589  * @rx_desc: pointer to the EOP Rx descriptor
 590  * @skb: pointer to current skb being populated
 591  *
 592  * This function checks the ring, descriptor, and packet information in
 593  * order to populate the hash, checksum, protocol, and
 594  * other fields within the skb.
 595  **/
 596 static void wx_process_skb_fields(struct wx_ring *rx_ring,
 597                                   union wx_rx_desc *rx_desc,
 598                                   struct sk_buff *skb)
 599 {
 600         wx_rx_hash(rx_ring, rx_desc, skb);
 601         wx_rx_checksum(rx_ring, rx_desc, skb);
 602         wx_rx_vlan(rx_ring, rx_desc, skb);
 603         skb_record_rx_queue(skb, rx_ring->queue_index);
 604         skb->protocol = eth_type_trans(skb, rx_ring->netdev);
 605 }
 606
 607 /**
 608  * wx_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
 609  * @q_vector: structure containing interrupt and ring information
 610  * @rx_ring: rx descriptor ring to transact packets on
 611  * @budget: Total limit on number of packets to process
 612  *
 613  * This function provides a "bounce buffer" approach to Rx interrupt
 614  * processing.  The advantage to this is that on systems that have
 615  * expensive overhead for IOMMU access this provides a means of avoiding
 616  * it by maintaining the mapping of the page to the system.
 617  *
 618  * Returns amount of work completed.
 619  **/
 620 static int wx_clean_rx_irq(struct wx_q_vector *q_vector,
 621                            struct wx_ring *rx_ring,
 622                            int budget)
 623 {
 624         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 625         u16 cleaned_count = wx_desc_unused(rx_ring);
 626
 627         do {
 628                 struct wx_rx_buffer *rx_buffer;
 629                 union wx_rx_desc *rx_desc;
 630                 struct sk_buff *skb;
 631                 int rx_buffer_pgcnt;
 632
 633                 /* return some buffers to hardware, one at a time is too slow */
 634                 if (cleaned_count >= WX_RX_BUFFER_WRITE) {
 635                         wx_alloc_rx_buffers(rx_ring, cleaned_count);
 636                         cleaned_count = 0;
 637                 }
 638
 639                 rx_desc = WX_RX_DESC(rx_ring, rx_ring->next_to_clean);
 640                 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_DD))
 641                         break;
 642
 643                 /* This memory barrier is needed to keep us from reading
 644                  * any other fields out of the rx_desc until we know the
 645                  * descriptor has been written back
 646                  */
 647                 dma_rmb();
 648
 649                 rx_buffer = wx_get_rx_buffer(rx_ring, rx_desc, &skb, &rx_buffer_pgcnt);
 650
 651                 /* retrieve a buffer from the ring */
 652                 skb = wx_build_skb(rx_ring, rx_buffer, rx_desc);
 653
 654                 /* exit if we failed to retrieve a buffer */
 655                 if (!skb) {
 656                         rx_ring->rx_stats.alloc_rx_buff_failed++;
 657                         break;
 658                 }
 659
 660                 wx_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt);
 661                 cleaned_count++;
 662
 663                 /* place incomplete frames back on ring for completion */
 664                 if (wx_is_non_eop(rx_ring, rx_desc, skb))
 665                         continue;
 666
 667                 /* verify the packet layout is correct */
 668                 if (wx_cleanup_headers(rx_ring, rx_desc, skb))
 669                         continue;
 670
 671                 /* probably a little skewed due to removing CRC */
 672                 total_rx_bytes += skb->len;
 673
 674                 /* populate checksum, timestamp, VLAN, and protocol */
 675                 wx_process_skb_fields(rx_ring, rx_desc, skb);
 676                 napi_gro_receive(&q_vector->napi, skb);
 677
 678                 /* update budget accounting */
 679                 total_rx_packets++;
 680         } while (likely(total_rx_packets < budget));
 681
 682         u64_stats_update_begin(&rx_ring->syncp);
 683         rx_ring->stats.packets += total_rx_packets;
 684         rx_ring->stats.bytes += total_rx_bytes;
 685         u64_stats_update_end(&rx_ring->syncp);
 686         q_vector->rx.total_packets += total_rx_packets;
 687         q_vector->rx.total_bytes += total_rx_bytes;
 688
 689         return total_rx_packets;
 690 }
 691
 692 static struct netdev_queue *wx_txring_txq(const struct wx_ring *ring)
 693 {
 694         return netdev_get_tx_queue(ring->netdev, ring->queue_index);
 695 }
 696
 697 /**
 698  * wx_clean_tx_irq - Reclaim resources after transmit completes
 699  * @q_vector: structure containing interrupt and ring information
 700  * @tx_ring: tx ring to clean
 701  * @napi_budget: Used to determine if we are in netpoll
 702  **/
 703 static bool wx_clean_tx_irq(struct wx_q_vector *q_vector,
 704                             struct wx_ring *tx_ring, int napi_budget)
 705 {
 706         unsigned int budget = q_vector->wx->tx_work_limit;
 707         unsigned int total_bytes = 0, total_packets = 0;
 708         unsigned int i = tx_ring->next_to_clean;
 709         struct wx_tx_buffer *tx_buffer;
 710         union wx_tx_desc *tx_desc;
 711
 712         if (!netif_carrier_ok(tx_ring->netdev))
 713                 return true;
 714
 715         tx_buffer = &tx_ring->tx_buffer_info[i];
 716         tx_desc = WX_TX_DESC(tx_ring, i);
 717         i -= tx_ring->count;
 718
 719         do {
 720                 union wx_tx_desc *eop_desc = tx_buffer->next_to_watch;
 721
 722                 /* if next_to_watch is not set then there is no work pending */
 723                 if (!eop_desc)
 724                         break;
 725
 726                 /* prevent any other reads prior to eop_desc */
 727                 smp_rmb();
 728
 729                 /* if DD is not set pending work has not been completed */
 730                 if (!(eop_desc->wb.status & cpu_to_le32(WX_TXD_STAT_DD)))
 731                         break;
 732
 733                 /* clear next_to_watch to prevent false hangs */
 734                 tx_buffer->next_to_watch = NULL;
 735
 736                 /* update the statistics for this packet */
 737                 total_bytes += tx_buffer->bytecount;
 738                 total_packets += tx_buffer->gso_segs;
 739
 740                 /* free the skb */
 741                 napi_consume_skb(tx_buffer->skb, napi_budget);
 742
 743                 /* unmap skb header data */
 744                 dma_unmap_single(tx_ring->dev,
 745                                  dma_unmap_addr(tx_buffer, dma),
 746                                  dma_unmap_len(tx_buffer, len),
 747                                  DMA_TO_DEVICE);
 748
 749                 /* clear tx_buffer data */
 750                 dma_unmap_len_set(tx_buffer, len, 0);
 751
 752                 /* unmap remaining buffers */
 753                 while (tx_desc != eop_desc) {
 754                         tx_buffer++;
 755                         tx_desc++;
 756                         i++;
 757                         if (unlikely(!i)) {
 758                                 i -= tx_ring->count;
 759                                 tx_buffer = tx_ring->tx_buffer_info;
 760                                 tx_desc = WX_TX_DESC(tx_ring, 0);
 761                         }
 762
 763                         /* unmap any remaining paged data */
 764                         if (dma_unmap_len(tx_buffer, len)) {
 765                                 dma_unmap_page(tx_ring->dev,
 766                                                dma_unmap_addr(tx_buffer, dma),
 767                                                dma_unmap_len(tx_buffer, len),
 768                                                DMA_TO_DEVICE);
 769                                 dma_unmap_len_set(tx_buffer, len, 0);
 770                         }
 771                 }
 772
 773                 /* move us one more past the eop_desc for start of next pkt */
 774                 tx_buffer++;
 775                 tx_desc++;
 776                 i++;
 777                 if (unlikely(!i)) {
 778                         i -= tx_ring->count;
 779                         tx_buffer = tx_ring->tx_buffer_info;
 780                         tx_desc = WX_TX_DESC(tx_ring, 0);
 781                 }
 782
 783                 /* issue prefetch for next Tx descriptor */
 784                 prefetch(tx_desc);
 785
 786                 /* update budget accounting */
 787                 budget--;
 788         } while (likely(budget));
 789
 790         i += tx_ring->count;
 791         tx_ring->next_to_clean = i;
 792         u64_stats_update_begin(&tx_ring->syncp);
 793         tx_ring->stats.bytes += total_bytes;
 794         tx_ring->stats.packets += total_packets;
 795         u64_stats_update_end(&tx_ring->syncp);
 796         q_vector->tx.total_bytes += total_bytes;
 797         q_vector->tx.total_packets += total_packets;
 798
 799         netdev_tx_completed_queue(wx_txring_txq(tx_ring),
 800                                   total_packets, total_bytes);
 801
 802 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
 803         if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
 804                      (wx_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
 805                 /* Make sure that anybody stopping the queue after this
 806                  * sees the new next_to_clean.
 807                  */
 808                 smp_mb();
 809
 810                 if (__netif_subqueue_stopped(tx_ring->netdev,
 811                                              tx_ring->queue_index) &&
 812                     netif_running(tx_ring->netdev)) {
 813                         netif_wake_subqueue(tx_ring->netdev,
 814                                             tx_ring->queue_index);
 815                         ++tx_ring->tx_stats.restart_queue;
 816                 }
 817         }
 818
 819         return !!budget;
 820 }
 821
 822 /**
 823  * wx_poll - NAPI polling RX/TX cleanup routine
 824  * @napi: napi struct with our devices info in it
 825  * @budget: amount of work driver is allowed to do this pass, in packets
 826  *
 827  * This function will clean all queues associated with a q_vector.
 828  **/
 829 static int wx_poll(struct napi_struct *napi, int budget)
 830 {
 831         struct wx_q_vector *q_vector = container_of(napi, struct wx_q_vector, napi);
 832         int per_ring_budget, work_done = 0;
 833         struct wx *wx = q_vector->wx;
 834         bool clean_complete = true;
 835         struct wx_ring *ring;
 836
 837         wx_for_each_ring(ring, q_vector->tx) {
 838                 if (!wx_clean_tx_irq(q_vector, ring, budget))
 839                         clean_complete = false;
 840         }
 841
 842         /* Exit if we are called by netpoll */
 843         if (budget <= 0)
 844                 return budget;
 845
 846         /* attempt to distribute budget to each queue fairly, but don't allow
 847          * the budget to go below 1 because we'll exit polling
 848          */
 849         if (q_vector->rx.count > 1)
 850                 per_ring_budget = max(budget / q_vector->rx.count, 1);
 851         else
 852                 per_ring_budget = budget;
 853
 854         wx_for_each_ring(ring, q_vector->rx) {
 855                 int cleaned = wx_clean_rx_irq(q_vector, ring, per_ring_budget);
 856
 857                 work_done += cleaned;
 858                 if (cleaned >= per_ring_budget)
 859                         clean_complete = false;
 860         }
 861
 862         /* If all work not completed, return budget and keep polling */
 863         if (!clean_complete)
 864                 return budget;
 865
 866         /* all work done, exit the polling mode */
 867         if (likely(napi_complete_done(napi, work_done))) {
 868                 if (netif_running(wx->netdev))
 869                         wx_intr_enable(wx, WX_INTR_Q(q_vector->v_idx));
 870         }
 871
 872         return min(work_done, budget - 1);
 873 }
 874
 875 static int wx_maybe_stop_tx(struct wx_ring *tx_ring, u16 size)
 876 {
 877         if (likely(wx_desc_unused(tx_ring) >= size))
 878                 return 0;
 879
 880         netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
 881
 882         /* For the next check */
 883         smp_mb();
 884
 885         /* We need to check again in a case another CPU has just
 886          * made room available.
 887          */
 888         if (likely(wx_desc_unused(tx_ring) < size))
 889                 return -EBUSY;
 890
 891         /* A reprieve! - use start_queue because it doesn't call schedule */
 892         netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
 893         ++tx_ring->tx_stats.restart_queue;
 894
 895         return 0;
 896 }
 897
 898 static u32 wx_tx_cmd_type(u32 tx_flags)
 899 {
 900         /* set type for advanced descriptor with frame checksum insertion */
 901         u32 cmd_type = WX_TXD_DTYP_DATA | WX_TXD_IFCS;
 902
 903         /* set HW vlan bit if vlan is present */
 904         cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_HW_VLAN, WX_TXD_VLE);
 905         /* set segmentation enable bits for TSO/FSO */
 906         cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSO, WX_TXD_TSE);
 907         /* set timestamp bit if present */
 908         cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSTAMP, WX_TXD_MAC_TSTAMP);
 909         cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_LINKSEC, WX_TXD_LINKSEC);
 910
 911         return cmd_type;
 912 }
 913
 914 static void wx_tx_olinfo_status(union wx_tx_desc *tx_desc,
 915                                 u32 tx_flags, unsigned int paylen)
 916 {
 917         u32 olinfo_status = paylen << WX_TXD_PAYLEN_SHIFT;
 918
 919         /* enable L4 checksum for TSO and TX checksum offload */
 920         olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CSUM, WX_TXD_L4CS);
 921         /* enable IPv4 checksum for TSO */
 922         olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPV4, WX_TXD_IIPCS);
 923         /* enable outer IPv4 checksum for TSO */
 924         olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_OUTER_IPV4,
 925                                      WX_TXD_EIPCS);
 926         /* Check Context must be set if Tx switch is enabled, which it
 927          * always is for case where virtual functions are running
 928          */
 929         olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CC, WX_TXD_CC);
 930         olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPSEC,
 931                                      WX_TXD_IPSEC);
 932         tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
 933 }
 934
 935 static void wx_tx_map(struct wx_ring *tx_ring,
 936                       struct wx_tx_buffer *first,
 937                       const u8 hdr_len)
 938 {
 939         struct sk_buff *skb = first->skb;
 940         struct wx_tx_buffer *tx_buffer;
 941         u32 tx_flags = first->tx_flags;
 942         u16 i = tx_ring->next_to_use;
 943         unsigned int data_len, size;
 944         union wx_tx_desc *tx_desc;
 945         skb_frag_t *frag;
 946         dma_addr_t dma;
 947         u32 cmd_type;
 948
 949         cmd_type = wx_tx_cmd_type(tx_flags);
 950         tx_desc = WX_TX_DESC(tx_ring, i);
 951         wx_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);
 952
 953         size = skb_headlen(skb);
 954         data_len = skb->data_len;
 955         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
 956
 957         tx_buffer = first;
 958
 959         for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
 960                 if (dma_mapping_error(tx_ring->dev, dma))
 961                         goto dma_error;
 962
 963                 /* record length, and DMA address */
 964                 dma_unmap_len_set(tx_buffer, len, size);
 965                 dma_unmap_addr_set(tx_buffer, dma, dma);
 966
 967                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
 968
 969                 while (unlikely(size > WX_MAX_DATA_PER_TXD)) {
 970                         tx_desc->read.cmd_type_len =
 971                                 cpu_to_le32(cmd_type ^ WX_MAX_DATA_PER_TXD);
 972
 973                         i++;
 974                         tx_desc++;
 975                         if (i == tx_ring->count) {
 976                                 tx_desc = WX_TX_DESC(tx_ring, 0);
 977                                 i = 0;
 978                         }
 979                         tx_desc->read.olinfo_status = 0;
 980
 981                         dma += WX_MAX_DATA_PER_TXD;
 982                         size -= WX_MAX_DATA_PER_TXD;
 983
 984                         tx_desc->read.buffer_addr = cpu_to_le64(dma);
 985                 }
 986
 987                 if (likely(!data_len))
 988                         break;
 989
 990                 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
 991
 992                 i++;
 993                 tx_desc++;
 994                 if (i == tx_ring->count) {
 995                         tx_desc = WX_TX_DESC(tx_ring, 0);
 996                         i = 0;
 997                 }
 998                 tx_desc->read.olinfo_status = 0;
 999
1000                 size = skb_frag_size(frag);
1001
1002                 data_len -= size;
1003
1004                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
1005                                        DMA_TO_DEVICE);
1006
1007                 tx_buffer = &tx_ring->tx_buffer_info[i];
1008         }
1009
1010         /* write last descriptor with RS and EOP bits */
1011         cmd_type |= size | WX_TXD_EOP | WX_TXD_RS;
1012         tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1013
1014         netdev_tx_sent_queue(wx_txring_txq(tx_ring), first->bytecount);
1015
1016         skb_tx_timestamp(skb);
1017
1018         /* Force memory writes to complete before letting h/w know there
1019          * are new descriptors to fetch.  (Only applicable for weak-ordered
1020          * memory model archs, such as IA-64).
1021          *
1022          * We also need this memory barrier to make certain all of the
1023          * status bits have been updated before next_to_watch is written.
1024          */
1025         wmb();
1026
1027         /* set next_to_watch value indicating a packet is present */
1028         first->next_to_watch = tx_desc;
1029
1030         i++;
1031         if (i == tx_ring->count)
1032                 i = 0;
1033
1034         tx_ring->next_to_use = i;
1035
1036         wx_maybe_stop_tx(tx_ring, DESC_NEEDED);
1037
1038         if (netif_xmit_stopped(wx_txring_txq(tx_ring)) || !netdev_xmit_more())
1039                 writel(i, tx_ring->tail);
1040
1041         return;
1042 dma_error:
1043         dev_err(tx_ring->dev, "TX DMA map failed\n");
1044
1045         /* clear dma mappings for failed tx_buffer_info map */
1046         for (;;) {
1047                 tx_buffer = &tx_ring->tx_buffer_info[i];
1048                 if (dma_unmap_len(tx_buffer, len))
1049                         dma_unmap_page(tx_ring->dev,
1050                                        dma_unmap_addr(tx_buffer, dma),
1051                                        dma_unmap_len(tx_buffer, len),
1052                                        DMA_TO_DEVICE);
1053                 dma_unmap_len_set(tx_buffer, len, 0);
1054                 if (tx_buffer == first)
1055                         break;
1056                 if (i == 0)
1057                         i += tx_ring->count;
1058                 i--;
1059         }
1060
1061         dev_kfree_skb_any(first->skb);
1062         first->skb = NULL;
1063
1064         tx_ring->next_to_use = i;
1065 }
1066
1067 static void wx_tx_ctxtdesc(struct wx_ring *tx_ring, u32 vlan_macip_lens,
1068                            u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx)
1069 {
1070         struct wx_tx_context_desc *context_desc;
1071         u16 i = tx_ring->next_to_use;
1072
1073         context_desc = WX_TX_CTXTDESC(tx_ring, i);
1074         i++;
1075         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1076
1077         /* set bits to identify this as an advanced context descriptor */
1078         type_tucmd |= WX_TXD_DTYP_CTXT;
1079         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
1080         context_desc->seqnum_seed       = cpu_to_le32(fcoe_sof_eof);
1081         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
1082         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
1083 }
1084
1085 static void wx_get_ipv6_proto(struct sk_buff *skb, int offset, u8 *nexthdr)
1086 {
1087         struct ipv6hdr *hdr = (struct ipv6hdr *)(skb->data + offset);
1088
1089         *nexthdr = hdr->nexthdr;
1090         offset += sizeof(struct ipv6hdr);
1091         while (ipv6_ext_hdr(*nexthdr)) {
1092                 struct ipv6_opt_hdr _hdr, *hp;
1093
1094                 if (*nexthdr == NEXTHDR_NONE)
1095                         return;
1096                 hp = skb_header_pointer(skb, offset, sizeof(_hdr), &_hdr);
1097                 if (!hp)
1098                         return;
1099                 if (*nexthdr == NEXTHDR_FRAGMENT)
1100                         break;
1101                 *nexthdr = hp->nexthdr;
1102         }
1103 }
1104
1105 union network_header {
1106         struct iphdr *ipv4;
1107         struct ipv6hdr *ipv6;
1108         void *raw;
1109 };
1110
1111 static u8 wx_encode_tx_desc_ptype(const struct wx_tx_buffer *first)
1112 {
1113         u8 tun_prot = 0, l4_prot = 0, ptype = 0;
1114         struct sk_buff *skb = first->skb;
1115
1116         if (skb->encapsulation) {
1117                 union network_header hdr;
1118
1119                 switch (first->protocol) {
1120                 case htons(ETH_P_IP):
1121                         tun_prot = ip_hdr(skb)->protocol;
1122                         ptype = WX_PTYPE_TUN_IPV4;
1123                         break;
1124                 case htons(ETH_P_IPV6):
1125                         wx_get_ipv6_proto(skb, skb_network_offset(skb), &tun_prot);
1126                         ptype = WX_PTYPE_TUN_IPV6;
1127                         break;
1128                 default:
1129                         return ptype;
1130                 }
1131
1132                 if (tun_prot == IPPROTO_IPIP) {
1133                         hdr.raw = (void *)inner_ip_hdr(skb);
1134                         ptype |= WX_PTYPE_PKT_IPIP;
1135                 } else if (tun_prot == IPPROTO_UDP) {
1136                         hdr.raw = (void *)inner_ip_hdr(skb);
1137                         if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
1138                             skb->inner_protocol != htons(ETH_P_TEB)) {
1139                                 ptype |= WX_PTYPE_PKT_IG;
1140                         } else {
1141                                 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto
1142                                      == htons(ETH_P_8021Q))
1143                                         ptype |= WX_PTYPE_PKT_IGMV;
1144                                 else
1145                                         ptype |= WX_PTYPE_PKT_IGM;
1146                         }
1147
1148                 } else if (tun_prot == IPPROTO_GRE) {
1149                         hdr.raw = (void *)inner_ip_hdr(skb);
1150                         if (skb->inner_protocol ==  htons(ETH_P_IP) ||
1151                             skb->inner_protocol ==  htons(ETH_P_IPV6)) {
1152                                 ptype |= WX_PTYPE_PKT_IG;
1153                         } else {
1154                                 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto
1155                                     == htons(ETH_P_8021Q))
1156                                         ptype |= WX_PTYPE_PKT_IGMV;
1157                                 else
1158                                         ptype |= WX_PTYPE_PKT_IGM;
1159                         }
1160                 } else {
1161                         return ptype;
1162                 }
1163
1164                 switch (hdr.ipv4->version) {
1165                 case IPVERSION:
1166                         l4_prot = hdr.ipv4->protocol;
1167                         break;
1168                 case 6:
1169                         wx_get_ipv6_proto(skb, skb_inner_network_offset(skb), &l4_prot);
1170                         ptype |= WX_PTYPE_PKT_IPV6;
1171                         break;
1172                 default:
1173                         return ptype;
1174                 }
1175         } else {
1176                 switch (first->protocol) {
1177                 case htons(ETH_P_IP):
1178                         l4_prot = ip_hdr(skb)->protocol;
1179                         ptype = WX_PTYPE_PKT_IP;
1180                         break;
1181                 case htons(ETH_P_IPV6):
1182                         wx_get_ipv6_proto(skb, skb_network_offset(skb), &l4_prot);
1183                         ptype = WX_PTYPE_PKT_IP | WX_PTYPE_PKT_IPV6;
1184                         break;
1185                 default:
1186                         return WX_PTYPE_PKT_MAC | WX_PTYPE_TYP_MAC;
1187                 }
1188         }
1189         switch (l4_prot) {
1190         case IPPROTO_TCP:
1191                 ptype |= WX_PTYPE_TYP_TCP;
1192                 break;
1193         case IPPROTO_UDP:
1194                 ptype |= WX_PTYPE_TYP_UDP;
1195                 break;
1196         case IPPROTO_SCTP:
1197                 ptype |= WX_PTYPE_TYP_SCTP;
1198                 break;
1199         default:
1200                 ptype |= WX_PTYPE_TYP_IP;
1201                 break;
1202         }
1203
1204         return ptype;
1205 }
1206
1207 static int wx_tso(struct wx_ring *tx_ring, struct wx_tx_buffer *first,
1208                   u8 *hdr_len, u8 ptype)
1209 {
1210         u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1211         struct net_device *netdev = tx_ring->netdev;
1212         u32 l4len, tunhdr_eiplen_tunlen = 0;
1213         struct sk_buff *skb = first->skb;
1214         bool enc = skb->encapsulation;
1215         struct ipv6hdr *ipv6h;
1216         struct tcphdr *tcph;
1217         struct iphdr *iph;
1218         u8 tun_prot = 0;
1219         int err;
1220
1221         if (skb->ip_summed != CHECKSUM_PARTIAL)
1222                 return 0;
1223
1224         if (!skb_is_gso(skb))
1225                 return 0;
1226
1227         err = skb_cow_head(skb, 0);
1228         if (err < 0)
1229                 return err;
1230
1231         /* indicates the inner headers in the skbuff are valid. */
1232         iph = enc ? inner_ip_hdr(skb) : ip_hdr(skb);
1233         if (iph->version == 4) {
1234                 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1235                 iph->tot_len = 0;
1236                 iph->check = 0;
1237                 tcph->check = ~csum_tcpudp_magic(iph->saddr,
1238                                                  iph->daddr, 0,
1239                                                  IPPROTO_TCP, 0);
1240                 first->tx_flags |= WX_TX_FLAGS_TSO |
1241                                    WX_TX_FLAGS_CSUM |
1242                                    WX_TX_FLAGS_IPV4 |
1243                                    WX_TX_FLAGS_CC;
1244         } else if (iph->version == 6 && skb_is_gso_v6(skb)) {
1245                 ipv6h = enc ? inner_ipv6_hdr(skb) : ipv6_hdr(skb);
1246                 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1247                 ipv6h->payload_len = 0;
1248                 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr,
1249                                                &ipv6h->daddr, 0,
1250                                                IPPROTO_TCP, 0);
1251                 first->tx_flags |= WX_TX_FLAGS_TSO |
1252                                    WX_TX_FLAGS_CSUM |
1253                                    WX_TX_FLAGS_CC;
1254         }
1255
1256         /* compute header lengths */
1257         l4len = enc ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1258         *hdr_len = enc ? skb_inner_transport_offset(skb) :
1259                          skb_transport_offset(skb);
1260         *hdr_len += l4len;
1261
1262         /* update gso size and bytecount with header size */
1263         first->gso_segs = skb_shinfo(skb)->gso_segs;
1264         first->bytecount += (first->gso_segs - 1) * *hdr_len;
1265
1266         /* mss_l4len_id: use 0 as index for TSO */
1267         mss_l4len_idx = l4len << WX_TXD_L4LEN_SHIFT;
1268         mss_l4len_idx |= skb_shinfo(skb)->gso_size << WX_TXD_MSS_SHIFT;
1269
1270         /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
1271         if (enc) {
1272                 switch (first->protocol) {
1273                 case htons(ETH_P_IP):
1274                         tun_prot = ip_hdr(skb)->protocol;
1275                         first->tx_flags |= WX_TX_FLAGS_OUTER_IPV4;
1276                         break;
1277                 case htons(ETH_P_IPV6):
1278                         tun_prot = ipv6_hdr(skb)->nexthdr;
1279                         break;
1280                 default:
1281                         break;
1282                 }
1283                 switch (tun_prot) {
1284                 case IPPROTO_UDP:
1285                         tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;
1286                         tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1287                                                  WX_TXD_OUTER_IPLEN_SHIFT) |
1288                                                 (((skb_inner_mac_header(skb) -
1289                                                 skb_transport_header(skb)) >> 1) <<
1290                                                 WX_TXD_TUNNEL_LEN_SHIFT);
1291                         break;
1292                 case IPPROTO_GRE:
1293                         tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;
1294                         tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1295                                                  WX_TXD_OUTER_IPLEN_SHIFT) |
1296                                                 (((skb_inner_mac_header(skb) -
1297                                                 skb_transport_header(skb)) >> 1) <<
1298                                                 WX_TXD_TUNNEL_LEN_SHIFT);
1299                         break;
1300                 case IPPROTO_IPIP:
1301                         tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -
1302                                                 (char *)ip_hdr(skb)) >> 2) <<
1303                                                 WX_TXD_OUTER_IPLEN_SHIFT;
1304                         break;
1305                 default:
1306                         break;
1307                 }
1308                 vlan_macip_lens = skb_inner_network_header_len(skb) >> 1;
1309         } else {
1310                 vlan_macip_lens = skb_network_header_len(skb) >> 1;
1311         }
1312
1313         vlan_macip_lens |= skb_network_offset(skb) << WX_TXD_MACLEN_SHIFT;
1314         vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;
1315
1316         type_tucmd = ptype << 24;
1317         if (skb->vlan_proto == htons(ETH_P_8021AD) &&
1318             netdev->features & NETIF_F_HW_VLAN_STAG_TX)
1319                 type_tucmd |= WX_SET_FLAG(first->tx_flags,
1320                                           WX_TX_FLAGS_HW_VLAN,
1321                                           0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);
1322         wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,
1323                        type_tucmd, mss_l4len_idx);
1324
1325         return 1;
1326 }
1327
1328 static void wx_tx_csum(struct wx_ring *tx_ring, struct wx_tx_buffer *first,
1329                        u8 ptype)
1330 {
1331         u32 tunhdr_eiplen_tunlen = 0, vlan_macip_lens = 0;
1332         struct net_device *netdev = tx_ring->netdev;
1333         u32 mss_l4len_idx = 0, type_tucmd;
1334         struct sk_buff *skb = first->skb;
1335         u8 tun_prot = 0;
1336
1337         if (skb->ip_summed != CHECKSUM_PARTIAL) {
1338                 if (!(first->tx_flags & WX_TX_FLAGS_HW_VLAN) &&
1339                     !(first->tx_flags & WX_TX_FLAGS_CC))
1340                         return;
1341                 vlan_macip_lens = skb_network_offset(skb) <<
1342                                   WX_TXD_MACLEN_SHIFT;
1343         } else {
1344                 u8 l4_prot = 0;
1345                 union {
1346                         struct iphdr *ipv4;
1347                         struct ipv6hdr *ipv6;
1348                         u8 *raw;
1349                 } network_hdr;
1350                 union {
1351                         struct tcphdr *tcphdr;
1352                         u8 *raw;
1353                 } transport_hdr;
1354
1355                 if (skb->encapsulation) {
1356                         network_hdr.raw = skb_inner_network_header(skb);
1357                         transport_hdr.raw = skb_inner_transport_header(skb);
1358                         vlan_macip_lens = skb_network_offset(skb) <<
1359                                           WX_TXD_MACLEN_SHIFT;
1360                         switch (first->protocol) {
1361                         case htons(ETH_P_IP):
1362                                 tun_prot = ip_hdr(skb)->protocol;
1363                                 break;
1364                         case htons(ETH_P_IPV6):
1365                                 tun_prot = ipv6_hdr(skb)->nexthdr;
1366                                 break;
1367                         default:
1368                                 return;
1369                         }
1370                         switch (tun_prot) {
1371                         case IPPROTO_UDP:
1372                                 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;
1373                                 tunhdr_eiplen_tunlen |=
1374                                         ((skb_network_header_len(skb) >> 2) <<
1375                                         WX_TXD_OUTER_IPLEN_SHIFT) |
1376                                         (((skb_inner_mac_header(skb) -
1377                                         skb_transport_header(skb)) >> 1) <<
1378                                         WX_TXD_TUNNEL_LEN_SHIFT);
1379                                 break;
1380                         case IPPROTO_GRE:
1381                                 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;
1382                                 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1383                                                          WX_TXD_OUTER_IPLEN_SHIFT) |
1384                                                          (((skb_inner_mac_header(skb) -
1385                                                             skb_transport_header(skb)) >> 1) <<
1386                                                           WX_TXD_TUNNEL_LEN_SHIFT);
1387                                 break;
1388                         case IPPROTO_IPIP:
1389                                 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -
1390                                                         (char *)ip_hdr(skb)) >> 2) <<
1391                                                         WX_TXD_OUTER_IPLEN_SHIFT;
1392                                 break;
1393                         default:
1394                                 break;
1395                         }
1396
1397                 } else {
1398                         network_hdr.raw = skb_network_header(skb);
1399                         transport_hdr.raw = skb_transport_header(skb);
1400                         vlan_macip_lens = skb_network_offset(skb) <<
1401                                           WX_TXD_MACLEN_SHIFT;
1402                 }
1403
1404                 switch (network_hdr.ipv4->version) {
1405                 case IPVERSION:
1406                         vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;
1407                         l4_prot = network_hdr.ipv4->protocol;
1408                         break;
1409                 case 6:
1410                         vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;
1411                         l4_prot = network_hdr.ipv6->nexthdr;
1412                         break;
1413                 default:
1414                         break;
1415                 }
1416
1417                 switch (l4_prot) {
1418                 case IPPROTO_TCP:
1419                 mss_l4len_idx = (transport_hdr.tcphdr->doff * 4) <<
1420                                 WX_TXD_L4LEN_SHIFT;
1421                         break;
1422                 case IPPROTO_SCTP:
1423                         mss_l4len_idx = sizeof(struct sctphdr) <<
1424                                         WX_TXD_L4LEN_SHIFT;
1425                         break;
1426                 case IPPROTO_UDP:
1427                         mss_l4len_idx = sizeof(struct udphdr) <<
1428                                         WX_TXD_L4LEN_SHIFT;
1429                         break;
1430                 default:
1431                         break;
1432                 }
1433
1434                 /* update TX checksum flag */
1435                 first->tx_flags |= WX_TX_FLAGS_CSUM;
1436         }
1437         first->tx_flags |= WX_TX_FLAGS_CC;
1438         /* vlan_macip_lens: MACLEN, VLAN tag */
1439         vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;
1440
1441         type_tucmd = ptype << 24;
1442         if (skb->vlan_proto == htons(ETH_P_8021AD) &&
1443             netdev->features & NETIF_F_HW_VLAN_STAG_TX)
1444                 type_tucmd |= WX_SET_FLAG(first->tx_flags,
1445                                           WX_TX_FLAGS_HW_VLAN,
1446                                           0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);
1447         wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,
1448                        type_tucmd, mss_l4len_idx);
1449 }
1450
1451 static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb,
1452                                       struct wx_ring *tx_ring)
1453 {
1454         struct wx *wx = netdev_priv(tx_ring->netdev);
1455         u16 count = TXD_USE_COUNT(skb_headlen(skb));
1456         struct wx_tx_buffer *first;
1457         u8 hdr_len = 0, ptype;
1458         unsigned short f;
1459         u32 tx_flags = 0;
1460         int tso;
1461
1462         /* need: 1 descriptor per page * PAGE_SIZE/WX_MAX_DATA_PER_TXD,
1463          *       + 1 desc for skb_headlen/WX_MAX_DATA_PER_TXD,
1464          *       + 2 desc gap to keep tail from touching head,
1465          *       + 1 desc for context descriptor,
1466          * otherwise try next time
1467          */
1468         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1469                 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->
1470                                                      frags[f]));
1471
1472         if (wx_maybe_stop_tx(tx_ring, count + 3)) {
1473                 tx_ring->tx_stats.tx_busy++;
1474                 return NETDEV_TX_BUSY;
1475         }
1476
1477         /* record the location of the first descriptor for this packet */
1478         first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1479         first->skb = skb;
1480         first->bytecount = skb->len;
1481         first->gso_segs = 1;
1482
1483         /* if we have a HW VLAN tag being added default to the HW one */
1484         if (skb_vlan_tag_present(skb)) {
1485                 tx_flags |= skb_vlan_tag_get(skb) << WX_TX_FLAGS_VLAN_SHIFT;
1486                 tx_flags |= WX_TX_FLAGS_HW_VLAN;
1487         }
1488
1489         /* record initial flags and protocol */
1490         first->tx_flags = tx_flags;
1491         first->protocol = vlan_get_protocol(skb);
1492
1493         ptype = wx_encode_tx_desc_ptype(first);
1494
1495         tso = wx_tso(tx_ring, first, &hdr_len, ptype);
1496         if (tso < 0)
1497                 goto out_drop;
1498         else if (!tso)
1499                 wx_tx_csum(tx_ring, first, ptype);
1500
1501         if (test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags) && tx_ring->atr_sample_rate)
1502                 wx->atr(tx_ring, first, ptype);
1503
1504         wx_tx_map(tx_ring, first, hdr_len);
1505
1506         return NETDEV_TX_OK;
1507 out_drop:
1508         dev_kfree_skb_any(first->skb);
1509         first->skb = NULL;
1510
1511         return NETDEV_TX_OK;
1512 }
1513
1514 netdev_tx_t wx_xmit_frame(struct sk_buff *skb,
1515                           struct net_device *netdev)
1516 {
1517         unsigned int r_idx = skb->queue_mapping;
1518         struct wx *wx = netdev_priv(netdev);
1519         struct wx_ring *tx_ring;
1520
1521         if (!netif_carrier_ok(netdev)) {
1522                 dev_kfree_skb_any(skb);
1523                 return NETDEV_TX_OK;
1524         }
1525
1526         /* The minimum packet size for olinfo paylen is 17 so pad the skb
1527          * in order to meet this minimum size requirement.
1528          */
1529         if (skb_put_padto(skb, 17))
1530                 return NETDEV_TX_OK;
1531
1532         if (r_idx >= wx->num_tx_queues)
1533                 r_idx = r_idx % wx->num_tx_queues;
1534         tx_ring = wx->tx_ring[r_idx];
1535
1536         return wx_xmit_frame_ring(skb, tx_ring);
1537 }
1538 EXPORT_SYMBOL(wx_xmit_frame);
1539
1540 void wx_napi_enable_all(struct wx *wx)
1541 {
1542         struct wx_q_vector *q_vector;
1543         int q_idx;
1544
1545         for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {
1546                 q_vector = wx->q_vector[q_idx];
1547                 napi_enable(&q_vector->napi);
1548         }
1549 }
1550 EXPORT_SYMBOL(wx_napi_enable_all);
1551
1552 void wx_napi_disable_all(struct wx *wx)
1553 {
1554         struct wx_q_vector *q_vector;
1555         int q_idx;
1556
1557         for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {
1558                 q_vector = wx->q_vector[q_idx];
1559                 napi_disable(&q_vector->napi);
1560         }
1561 }
1562 EXPORT_SYMBOL(wx_napi_disable_all);
1563
1564 /**
1565  * wx_set_rss_queues: Allocate queues for RSS
1566  * @wx: board private structure to initialize
1567  *
1568  * This is our "base" multiqueue mode.  RSS (Receive Side Scaling) will try
1569  * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
1570  *
1571  **/
1572 static void wx_set_rss_queues(struct wx *wx)
1573 {
1574         struct wx_ring_feature *f;
1575
1576         /* set mask for 16 queue limit of RSS */
1577         f = &wx->ring_feature[RING_F_RSS];
1578         f->indices = f->limit;
1579
1580         if (!(test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags)))
1581                 goto out;
1582
1583         clear_bit(WX_FLAG_FDIR_HASH, wx->flags);
1584
1585         /* Use Flow Director in addition to RSS to ensure the best
1586          * distribution of flows across cores, even when an FDIR flow
1587          * isn't matched.
1588          */
1589         if (f->indices > 1) {
1590                 f = &wx->ring_feature[RING_F_FDIR];
1591
1592                 f->indices = f->limit;
1593
1594                 if (!(test_bit(WX_FLAG_FDIR_PERFECT, wx->flags)))
1595                         set_bit(WX_FLAG_FDIR_HASH, wx->flags);
1596         }
1597
1598 out:
1599         wx->num_rx_queues = f->indices;
1600         wx->num_tx_queues = f->indices;
1601 }
1602
1603 static void wx_set_num_queues(struct wx *wx)
1604 {
1605         /* Start with base case */
1606         wx->num_rx_queues = 1;
1607         wx->num_tx_queues = 1;
1608         wx->queues_per_pool = 1;
1609
1610         wx_set_rss_queues(wx);
1611 }
1612
1613 /**
1614  * wx_acquire_msix_vectors - acquire MSI-X vectors
1615  * @wx: board private structure
1616  *
1617  * Attempts to acquire a suitable range of MSI-X vector interrupts. Will
1618  * return a negative error code if unable to acquire MSI-X vectors for any
1619  * reason.
1620  */
1621 static int wx_acquire_msix_vectors(struct wx *wx)
1622 {
1623         struct irq_affinity affd = { .pre_vectors = 1 };
1624         int nvecs, i;
1625
1626         /* We start by asking for one vector per queue pair */
1627         nvecs = max(wx->num_rx_queues, wx->num_tx_queues);
1628         nvecs = min_t(int, nvecs, num_online_cpus());
1629         nvecs = min_t(int, nvecs, wx->mac.max_msix_vectors);
1630
1631         wx->msix_q_entries = kcalloc(nvecs, sizeof(struct msix_entry),
1632                                      GFP_KERNEL);
1633         if (!wx->msix_q_entries)
1634                 return -ENOMEM;
1635
1636         /* One for non-queue interrupts */
1637         nvecs += 1;
1638
1639         wx->msix_entry = kcalloc(1, sizeof(struct msix_entry),
1640                                  GFP_KERNEL);
1641         if (!wx->msix_entry) {
1642                 kfree(wx->msix_q_entries);
1643                 wx->msix_q_entries = NULL;
1644                 return -ENOMEM;
1645         }
1646
1647         nvecs = pci_alloc_irq_vectors_affinity(wx->pdev, nvecs,
1648                                                nvecs,
1649                                                PCI_IRQ_MSIX | PCI_IRQ_AFFINITY,
1650                                                &affd);
1651         if (nvecs < 0) {
1652                 wx_err(wx, "Failed to allocate MSI-X interrupts. Err: %d\n", nvecs);
1653                 kfree(wx->msix_q_entries);
1654                 wx->msix_q_entries = NULL;
1655                 kfree(wx->msix_entry);
1656                 wx->msix_entry = NULL;
1657                 return nvecs;
1658         }
1659
1660         wx->msix_entry->entry = 0;
1661         wx->msix_entry->vector = pci_irq_vector(wx->pdev, 0);
1662         nvecs -= 1;
1663         for (i = 0; i < nvecs; i++) {
1664                 wx->msix_q_entries[i].entry = i;
1665                 wx->msix_q_entries[i].vector = pci_irq_vector(wx->pdev, i + 1);
1666         }
1667
1668         wx->num_q_vectors = nvecs;
1669
1670         return 0;
1671 }
1672
1673 /**
1674  * wx_set_interrupt_capability - set MSI-X or MSI if supported
1675  * @wx: board private structure to initialize
1676  *
1677  * Attempt to configure the interrupts using the best available
1678  * capabilities of the hardware and the kernel.
1679  **/
1680 static int wx_set_interrupt_capability(struct wx *wx)
1681 {
1682         struct pci_dev *pdev = wx->pdev;
1683         int nvecs, ret;
1684
1685         /* We will try to get MSI-X interrupts first */
1686         ret = wx_acquire_msix_vectors(wx);
1687         if (ret == 0 || (ret == -ENOMEM))
1688                 return ret;
1689
1690         /* Disable RSS */
1691         dev_warn(&wx->pdev->dev, "Disabling RSS support\n");
1692         wx->ring_feature[RING_F_RSS].limit = 1;
1693
1694         wx_set_num_queues(wx);
1695
1696         /* minmum one for queue, one for misc*/
1697         nvecs = 1;
1698         nvecs = pci_alloc_irq_vectors(pdev, nvecs,
1699                                       nvecs, PCI_IRQ_MSI | PCI_IRQ_INTX);
1700         if (nvecs == 1) {
1701                 if (pdev->msi_enabled)
1702                         wx_err(wx, "Fallback to MSI.\n");
1703                 else
1704                         wx_err(wx, "Fallback to INTx.\n");
1705         } else {
1706                 wx_err(wx, "Failed to allocate MSI/INTx interrupts. Error: %d\n", nvecs);
1707                 return nvecs;
1708         }
1709
1710         pdev->irq = pci_irq_vector(pdev, 0);
1711         wx->num_q_vectors = 1;
1712
1713         return 0;
1714 }
1715
1716 /**
1717  * wx_cache_ring_rss - Descriptor ring to register mapping for RSS
1718  * @wx: board private structure to initialize
1719  *
1720  * Cache the descriptor ring offsets for RSS, ATR, FCoE, and SR-IOV.
1721  *
1722  **/
1723 static void wx_cache_ring_rss(struct wx *wx)
1724 {
1725         u16 i;
1726
1727         for (i = 0; i < wx->num_rx_queues; i++)
1728                 wx->rx_ring[i]->reg_idx = i;
1729
1730         for (i = 0; i < wx->num_tx_queues; i++)
1731                 wx->tx_ring[i]->reg_idx = i;
1732 }
1733
1734 static void wx_add_ring(struct wx_ring *ring, struct wx_ring_container *head)
1735 {
1736         ring->next = head->ring;
1737         head->ring = ring;
1738         head->count++;
1739 }
1740
1741 /**
1742  * wx_alloc_q_vector - Allocate memory for a single interrupt vector
1743  * @wx: board private structure to initialize
1744  * @v_count: q_vectors allocated on wx, used for ring interleaving
1745  * @v_idx: index of vector in wx struct
1746  * @txr_count: total number of Tx rings to allocate
1747  * @txr_idx: index of first Tx ring to allocate
1748  * @rxr_count: total number of Rx rings to allocate
1749  * @rxr_idx: index of first Rx ring to allocate
1750  *
1751  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
1752  **/
1753 static int wx_alloc_q_vector(struct wx *wx,
1754                              unsigned int v_count, unsigned int v_idx,
1755                              unsigned int txr_count, unsigned int txr_idx,
1756                              unsigned int rxr_count, unsigned int rxr_idx)
1757 {
1758         struct wx_q_vector *q_vector;
1759         int ring_count, default_itr;
1760         struct wx_ring *ring;
1761
1762         /* note this will allocate space for the ring structure as well! */
1763         ring_count = txr_count + rxr_count;
1764
1765         q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
1766                            GFP_KERNEL);
1767         if (!q_vector)
1768                 return -ENOMEM;
1769
1770         /* initialize NAPI */
1771         netif_napi_add(wx->netdev, &q_vector->napi,
1772                        wx_poll);
1773
1774         /* tie q_vector and wx together */
1775         wx->q_vector[v_idx] = q_vector;
1776         q_vector->wx = wx;
1777         q_vector->v_idx = v_idx;
1778         if (cpu_online(v_idx))
1779                 q_vector->numa_node = cpu_to_node(v_idx);
1780
1781         /* initialize pointer to rings */
1782         ring = q_vector->ring;
1783
1784         if (wx->mac.type == wx_mac_sp)
1785                 default_itr = WX_12K_ITR;
1786         else
1787                 default_itr = WX_7K_ITR;
1788         /* initialize ITR */
1789         if (txr_count && !rxr_count)
1790                 /* tx only vector */
1791                 q_vector->itr = wx->tx_itr_setting ?
1792                                 default_itr : wx->tx_itr_setting;
1793         else
1794                 /* rx or rx/tx vector */
1795                 q_vector->itr = wx->rx_itr_setting ?
1796                                 default_itr : wx->rx_itr_setting;
1797
1798         while (txr_count) {
1799                 /* assign generic ring traits */
1800                 ring->dev = &wx->pdev->dev;
1801                 ring->netdev = wx->netdev;
1802
1803                 /* configure backlink on ring */
1804                 ring->q_vector = q_vector;
1805
1806                 /* update q_vector Tx values */
1807                 wx_add_ring(ring, &q_vector->tx);
1808
1809                 /* apply Tx specific ring traits */
1810                 ring->count = wx->tx_ring_count;
1811
1812                 ring->queue_index = txr_idx;
1813
1814                 /* assign ring to wx */
1815                 wx->tx_ring[txr_idx] = ring;
1816
1817                 /* update count and index */
1818                 txr_count--;
1819                 txr_idx += v_count;
1820
1821                 /* push pointer to next ring */
1822                 ring++;
1823         }
1824
1825         while (rxr_count) {
1826                 /* assign generic ring traits */
1827                 ring->dev = &wx->pdev->dev;
1828                 ring->netdev = wx->netdev;
1829
1830                 /* configure backlink on ring */
1831                 ring->q_vector = q_vector;
1832
1833                 /* update q_vector Rx values */
1834                 wx_add_ring(ring, &q_vector->rx);
1835
1836                 /* apply Rx specific ring traits */
1837                 ring->count = wx->rx_ring_count;
1838                 ring->queue_index = rxr_idx;
1839
1840                 /* assign ring to wx */
1841                 wx->rx_ring[rxr_idx] = ring;
1842
1843                 /* update count and index */
1844                 rxr_count--;
1845                 rxr_idx += v_count;
1846
1847                 /* push pointer to next ring */
1848                 ring++;
1849         }
1850
1851         return 0;
1852 }
1853
1854 /**
1855  * wx_free_q_vector - Free memory allocated for specific interrupt vector
1856  * @wx: board private structure to initialize
1857  * @v_idx: Index of vector to be freed
1858  *
1859  * This function frees the memory allocated to the q_vector.  In addition if
1860  * NAPI is enabled it will delete any references to the NAPI struct prior
1861  * to freeing the q_vector.
1862  **/
1863 static void wx_free_q_vector(struct wx *wx, int v_idx)
1864 {
1865         struct wx_q_vector *q_vector = wx->q_vector[v_idx];
1866         struct wx_ring *ring;
1867
1868         wx_for_each_ring(ring, q_vector->tx)
1869                 wx->tx_ring[ring->queue_index] = NULL;
1870
1871         wx_for_each_ring(ring, q_vector->rx)
1872                 wx->rx_ring[ring->queue_index] = NULL;
1873
1874         wx->q_vector[v_idx] = NULL;
1875         netif_napi_del(&q_vector->napi);
1876         kfree_rcu(q_vector, rcu);
1877 }
1878
1879 /**
1880  * wx_alloc_q_vectors - Allocate memory for interrupt vectors
1881  * @wx: board private structure to initialize
1882  *
1883  * We allocate one q_vector per queue interrupt.  If allocation fails we
1884  * return -ENOMEM.
1885  **/
1886 static int wx_alloc_q_vectors(struct wx *wx)
1887 {
1888         unsigned int rxr_idx = 0, txr_idx = 0, v_idx = 0;
1889         unsigned int rxr_remaining = wx->num_rx_queues;
1890         unsigned int txr_remaining = wx->num_tx_queues;
1891         unsigned int q_vectors = wx->num_q_vectors;
1892         int rqpv, tqpv;
1893         int err;
1894
1895         for (; v_idx < q_vectors; v_idx++) {
1896                 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
1897                 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1898                 err = wx_alloc_q_vector(wx, q_vectors, v_idx,
1899                                         tqpv, txr_idx,
1900                                         rqpv, rxr_idx);
1901
1902                 if (err)
1903                         goto err_out;
1904
1905                 /* update counts and index */
1906                 rxr_remaining -= rqpv;
1907                 txr_remaining -= tqpv;
1908                 rxr_idx++;
1909                 txr_idx++;
1910         }
1911
1912         return 0;
1913
1914 err_out:
1915         wx->num_tx_queues = 0;
1916         wx->num_rx_queues = 0;
1917         wx->num_q_vectors = 0;
1918
1919         while (v_idx--)
1920                 wx_free_q_vector(wx, v_idx);
1921
1922         return -ENOMEM;
1923 }
1924
1925 /**
1926  * wx_free_q_vectors - Free memory allocated for interrupt vectors
1927  * @wx: board private structure to initialize
1928  *
1929  * This function frees the memory allocated to the q_vectors.  In addition if
1930  * NAPI is enabled it will delete any references to the NAPI struct prior
1931  * to freeing the q_vector.
1932  **/
1933 static void wx_free_q_vectors(struct wx *wx)
1934 {
1935         int v_idx = wx->num_q_vectors;
1936
1937         wx->num_tx_queues = 0;
1938         wx->num_rx_queues = 0;
1939         wx->num_q_vectors = 0;
1940
1941         while (v_idx--)
1942                 wx_free_q_vector(wx, v_idx);
1943 }
1944
1945 void wx_reset_interrupt_capability(struct wx *wx)
1946 {
1947         struct pci_dev *pdev = wx->pdev;
1948
1949         if (!pdev->msi_enabled && !pdev->msix_enabled)
1950                 return;
1951
1952         if (pdev->msix_enabled) {
1953                 kfree(wx->msix_q_entries);
1954                 wx->msix_q_entries = NULL;
1955                 kfree(wx->msix_entry);
1956                 wx->msix_entry = NULL;
1957         }
1958         pci_free_irq_vectors(wx->pdev);
1959 }
1960 EXPORT_SYMBOL(wx_reset_interrupt_capability);
1961
1962 /**
1963  * wx_clear_interrupt_scheme - Clear the current interrupt scheme settings
1964  * @wx: board private structure to clear interrupt scheme on
1965  *
1966  * We go through and clear interrupt specific resources and reset the structure
1967  * to pre-load conditions
1968  **/
1969 void wx_clear_interrupt_scheme(struct wx *wx)
1970 {
1971         wx_free_q_vectors(wx);
1972         wx_reset_interrupt_capability(wx);
1973 }
1974 EXPORT_SYMBOL(wx_clear_interrupt_scheme);
1975
1976 int wx_init_interrupt_scheme(struct wx *wx)
1977 {
1978         int ret;
1979
1980         /* Number of supported queues */
1981         wx_set_num_queues(wx);
1982
1983         /* Set interrupt mode */
1984         ret = wx_set_interrupt_capability(wx);
1985         if (ret) {
1986                 wx_err(wx, "Allocate irq vectors for failed.\n");
1987                 return ret;
1988         }
1989
1990         /* Allocate memory for queues */
1991         ret = wx_alloc_q_vectors(wx);
1992         if (ret) {
1993                 wx_err(wx, "Unable to allocate memory for queue vectors.\n");
1994                 wx_reset_interrupt_capability(wx);
1995                 return ret;
1996         }
1997
1998         wx_cache_ring_rss(wx);
1999
2000         return 0;
2001 }
2002 EXPORT_SYMBOL(wx_init_interrupt_scheme);
2003
2004 irqreturn_t wx_msix_clean_rings(int __always_unused irq, void *data)
2005 {
2006         struct wx_q_vector *q_vector = data;
2007
2008         /* EIAM disabled interrupts (on this vector) for us */
2009         if (q_vector->rx.ring || q_vector->tx.ring)
2010                 napi_schedule_irqoff(&q_vector->napi);
2011
2012         return IRQ_HANDLED;
2013 }
2014 EXPORT_SYMBOL(wx_msix_clean_rings);
2015
2016 void wx_free_irq(struct wx *wx)
2017 {
2018         struct pci_dev *pdev = wx->pdev;
2019         int vector;
2020
2021         if (!(pdev->msix_enabled)) {
2022                 if (!wx->misc_irq_domain)
2023                         free_irq(pdev->irq, wx);
2024                 return;
2025         }
2026
2027         for (vector = 0; vector < wx->num_q_vectors; vector++) {
2028                 struct wx_q_vector *q_vector = wx->q_vector[vector];
2029                 struct msix_entry *entry = &wx->msix_q_entries[vector];
2030
2031                 /* free only the irqs that were actually requested */
2032                 if (!q_vector->rx.ring && !q_vector->tx.ring)
2033                         continue;
2034
2035                 free_irq(entry->vector, q_vector);
2036         }
2037
2038         if (!wx->misc_irq_domain)
2039                 free_irq(wx->msix_entry->vector, wx);
2040 }
2041 EXPORT_SYMBOL(wx_free_irq);
2042
2043 /**
2044  * wx_setup_isb_resources - allocate interrupt status resources
2045  * @wx: board private structure
2046  *
2047  * Return 0 on success, negative on failure
2048  **/
2049 int wx_setup_isb_resources(struct wx *wx)
2050 {
2051         struct pci_dev *pdev = wx->pdev;
2052
2053         if (wx->isb_mem)
2054                 return 0;
2055
2056         wx->isb_mem = dma_alloc_coherent(&pdev->dev,
2057                                          sizeof(u32) * 4,
2058                                          &wx->isb_dma,
2059                                          GFP_KERNEL);
2060         if (!wx->isb_mem) {
2061                 wx_err(wx, "Alloc isb_mem failed\n");
2062                 return -ENOMEM;
2063         }
2064
2065         return 0;
2066 }
2067 EXPORT_SYMBOL(wx_setup_isb_resources);
2068
2069 /**
2070  * wx_free_isb_resources - allocate all queues Rx resources
2071  * @wx: board private structure
2072  *
2073  * Return 0 on success, negative on failure
2074  **/
2075 void wx_free_isb_resources(struct wx *wx)
2076 {
2077         struct pci_dev *pdev = wx->pdev;
2078
2079         dma_free_coherent(&pdev->dev, sizeof(u32) * 4,
2080                           wx->isb_mem, wx->isb_dma);
2081         wx->isb_mem = NULL;
2082 }
2083 EXPORT_SYMBOL(wx_free_isb_resources);
2084
2085 u32 wx_misc_isb(struct wx *wx, enum wx_isb_idx idx)
2086 {
2087         u32 cur_tag = 0;
2088
2089         cur_tag = wx->isb_mem[WX_ISB_HEADER];
2090         wx->isb_tag[idx] = cur_tag;
2091
2092         return (__force u32)cpu_to_le32(wx->isb_mem[idx]);
2093 }
2094 EXPORT_SYMBOL(wx_misc_isb);
2095
2096 /**
2097  * wx_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
2098  * @wx: pointer to wx struct
2099  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
2100  * @queue: queue to map the corresponding interrupt to
2101  * @msix_vector: the vector to map to the corresponding queue
2102  *
2103  **/
2104 static void wx_set_ivar(struct wx *wx, s8 direction,
2105                         u16 queue, u16 msix_vector)
2106 {
2107         u32 ivar, index;
2108
2109         if (direction == -1) {
2110                 /* other causes */
2111                 msix_vector |= WX_PX_IVAR_ALLOC_VAL;
2112                 index = 0;
2113                 ivar = rd32(wx, WX_PX_MISC_IVAR);
2114                 ivar &= ~(0xFF << index);
2115                 ivar |= (msix_vector << index);
2116                 wr32(wx, WX_PX_MISC_IVAR, ivar);
2117         } else {
2118                 /* tx or rx causes */
2119                 msix_vector += 1; /* offset for queue vectors */
2120                 msix_vector |= WX_PX_IVAR_ALLOC_VAL;
2121                 index = ((16 * (queue & 1)) + (8 * direction));
2122                 ivar = rd32(wx, WX_PX_IVAR(queue >> 1));
2123                 ivar &= ~(0xFF << index);
2124                 ivar |= (msix_vector << index);
2125                 wr32(wx, WX_PX_IVAR(queue >> 1), ivar);
2126         }
2127 }
2128
2129 /**
2130  * wx_write_eitr - write EITR register in hardware specific way
2131  * @q_vector: structure containing interrupt and ring information
2132  *
2133  * This function is made to be called by ethtool and by the driver
2134  * when it needs to update EITR registers at runtime.  Hardware
2135  * specific quirks/differences are taken care of here.
2136  */
2137 void wx_write_eitr(struct wx_q_vector *q_vector)
2138 {
2139         struct wx *wx = q_vector->wx;
2140         int v_idx = q_vector->v_idx;
2141         u32 itr_reg;
2142
2143         if (wx->mac.type == wx_mac_sp)
2144                 itr_reg = q_vector->itr & WX_SP_MAX_EITR;
2145         else
2146                 itr_reg = q_vector->itr & WX_EM_MAX_EITR;
2147
2148         itr_reg |= WX_PX_ITR_CNT_WDIS;
2149
2150         wr32(wx, WX_PX_ITR(v_idx + 1), itr_reg);
2151 }
2152
2153 /**
2154  * wx_configure_vectors - Configure vectors for hardware
2155  * @wx: board private structure
2156  *
2157  * wx_configure_vectors sets up the hardware to properly generate MSI-X/MSI/INTx
2158  * interrupts.
2159  **/
2160 void wx_configure_vectors(struct wx *wx)
2161 {
2162         struct pci_dev *pdev = wx->pdev;
2163         u32 eitrsel = 0;
2164         u16 v_idx;
2165
2166         if (pdev->msix_enabled) {
2167                 /* Populate MSIX to EITR Select */
2168                 wr32(wx, WX_PX_ITRSEL, eitrsel);
2169                 /* use EIAM to auto-mask when MSI-X interrupt is asserted
2170                  * this saves a register write for every interrupt
2171                  */
2172                 wr32(wx, WX_PX_GPIE, WX_PX_GPIE_MODEL);
2173         } else {
2174                 /* legacy interrupts, use EIAM to auto-mask when reading EICR,
2175                  * specifically only auto mask tx and rx interrupts.
2176                  */
2177                 wr32(wx, WX_PX_GPIE, 0);
2178         }
2179
2180         /* Populate the IVAR table and set the ITR values to the
2181          * corresponding register.
2182          */
2183         for (v_idx = 0; v_idx < wx->num_q_vectors; v_idx++) {
2184                 struct wx_q_vector *q_vector = wx->q_vector[v_idx];
2185                 struct wx_ring *ring;
2186
2187                 wx_for_each_ring(ring, q_vector->rx)
2188                         wx_set_ivar(wx, 0, ring->reg_idx, v_idx);
2189
2190                 wx_for_each_ring(ring, q_vector->tx)
2191                         wx_set_ivar(wx, 1, ring->reg_idx, v_idx);
2192
2193                 wx_write_eitr(q_vector);
2194         }
2195
2196         wx_set_ivar(wx, -1, 0, 0);
2197         if (pdev->msix_enabled)
2198                 wr32(wx, WX_PX_ITR(0), 1950);
2199 }
2200 EXPORT_SYMBOL(wx_configure_vectors);
2201
2202 /**
2203  * wx_clean_rx_ring - Free Rx Buffers per Queue
2204  * @rx_ring: ring to free buffers from
2205  **/
2206 static void wx_clean_rx_ring(struct wx_ring *rx_ring)
2207 {
2208         struct wx_rx_buffer *rx_buffer;
2209         u16 i = rx_ring->next_to_clean;
2210
2211         rx_buffer = &rx_ring->rx_buffer_info[i];
2212
2213         /* Free all the Rx ring sk_buffs */
2214         while (i != rx_ring->next_to_alloc) {
2215                 if (rx_buffer->skb) {
2216                         struct sk_buff *skb = rx_buffer->skb;
2217
2218                         if (WX_CB(skb)->page_released)
2219                                 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
2220
2221                         dev_kfree_skb(skb);
2222                 }
2223
2224                 /* Invalidate cache lines that may have been written to by
2225                  * device so that we avoid corrupting memory.
2226                  */
2227                 dma_sync_single_range_for_cpu(rx_ring->dev,
2228                                               rx_buffer->dma,
2229                                               rx_buffer->page_offset,
2230                                               WX_RX_BUFSZ,
2231                                               DMA_FROM_DEVICE);
2232
2233                 /* free resources associated with mapping */
2234                 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
2235
2236                 i++;
2237                 rx_buffer++;
2238                 if (i == rx_ring->count) {
2239                         i = 0;
2240                         rx_buffer = rx_ring->rx_buffer_info;
2241                 }
2242         }
2243
2244         rx_ring->next_to_alloc = 0;
2245         rx_ring->next_to_clean = 0;
2246         rx_ring->next_to_use = 0;
2247 }
2248
2249 /**
2250  * wx_clean_all_rx_rings - Free Rx Buffers for all queues
2251  * @wx: board private structure
2252  **/
2253 void wx_clean_all_rx_rings(struct wx *wx)
2254 {
2255         int i;
2256
2257         for (i = 0; i < wx->num_rx_queues; i++)
2258                 wx_clean_rx_ring(wx->rx_ring[i]);
2259 }
2260 EXPORT_SYMBOL(wx_clean_all_rx_rings);
2261
2262 /**
2263  * wx_free_rx_resources - Free Rx Resources
2264  * @rx_ring: ring to clean the resources from
2265  *
2266  * Free all receive software resources
2267  **/
2268 static void wx_free_rx_resources(struct wx_ring *rx_ring)
2269 {
2270         wx_clean_rx_ring(rx_ring);
2271         kvfree(rx_ring->rx_buffer_info);
2272         rx_ring->rx_buffer_info = NULL;
2273
2274         /* if not set, then don't free */
2275         if (!rx_ring->desc)
2276                 return;
2277
2278         dma_free_coherent(rx_ring->dev, rx_ring->size,
2279                           rx_ring->desc, rx_ring->dma);
2280
2281         rx_ring->desc = NULL;
2282
2283         if (rx_ring->page_pool) {
2284                 page_pool_destroy(rx_ring->page_pool);
2285                 rx_ring->page_pool = NULL;
2286         }
2287 }
2288
2289 /**
2290  * wx_free_all_rx_resources - Free Rx Resources for All Queues
2291  * @wx: pointer to hardware structure
2292  *
2293  * Free all receive software resources
2294  **/
2295 static void wx_free_all_rx_resources(struct wx *wx)
2296 {
2297         int i;
2298
2299         for (i = 0; i < wx->num_rx_queues; i++)
2300                 wx_free_rx_resources(wx->rx_ring[i]);
2301 }
2302
2303 /**
2304  * wx_clean_tx_ring - Free Tx Buffers
2305  * @tx_ring: ring to be cleaned
2306  **/
2307 static void wx_clean_tx_ring(struct wx_ring *tx_ring)
2308 {
2309         struct wx_tx_buffer *tx_buffer;
2310         u16 i = tx_ring->next_to_clean;
2311
2312         tx_buffer = &tx_ring->tx_buffer_info[i];
2313
2314         while (i != tx_ring->next_to_use) {
2315                 union wx_tx_desc *eop_desc, *tx_desc;
2316
2317                 /* Free all the Tx ring sk_buffs */
2318                 dev_kfree_skb_any(tx_buffer->skb);
2319
2320                 /* unmap skb header data */
2321                 dma_unmap_single(tx_ring->dev,
2322                                  dma_unmap_addr(tx_buffer, dma),
2323                                  dma_unmap_len(tx_buffer, len),
2324                                  DMA_TO_DEVICE);
2325
2326                 /* check for eop_desc to determine the end of the packet */
2327                 eop_desc = tx_buffer->next_to_watch;
2328                 tx_desc = WX_TX_DESC(tx_ring, i);
2329
2330                 /* unmap remaining buffers */
2331                 while (tx_desc != eop_desc) {
2332                         tx_buffer++;
2333                         tx_desc++;
2334                         i++;
2335                         if (unlikely(i == tx_ring->count)) {
2336                                 i = 0;
2337                                 tx_buffer = tx_ring->tx_buffer_info;
2338                                 tx_desc = WX_TX_DESC(tx_ring, 0);
2339                         }
2340
2341                         /* unmap any remaining paged data */
2342                         if (dma_unmap_len(tx_buffer, len))
2343                                 dma_unmap_page(tx_ring->dev,
2344                                                dma_unmap_addr(tx_buffer, dma),
2345                                                dma_unmap_len(tx_buffer, len),
2346                                                DMA_TO_DEVICE);
2347                 }
2348
2349                 /* move us one more past the eop_desc for start of next pkt */
2350                 tx_buffer++;
2351                 i++;
2352                 if (unlikely(i == tx_ring->count)) {
2353                         i = 0;
2354                         tx_buffer = tx_ring->tx_buffer_info;
2355                 }
2356         }
2357
2358         netdev_tx_reset_queue(wx_txring_txq(tx_ring));
2359
2360         /* reset next_to_use and next_to_clean */
2361         tx_ring->next_to_use = 0;
2362         tx_ring->next_to_clean = 0;
2363 }
2364
2365 /**
2366  * wx_clean_all_tx_rings - Free Tx Buffers for all queues
2367  * @wx: board private structure
2368  **/
2369 void wx_clean_all_tx_rings(struct wx *wx)
2370 {
2371         int i;
2372
2373         for (i = 0; i < wx->num_tx_queues; i++)
2374                 wx_clean_tx_ring(wx->tx_ring[i]);
2375 }
2376 EXPORT_SYMBOL(wx_clean_all_tx_rings);
2377
2378 /**
2379  * wx_free_tx_resources - Free Tx Resources per Queue
2380  * @tx_ring: Tx descriptor ring for a specific queue
2381  *
2382  * Free all transmit software resources
2383  **/
2384 static void wx_free_tx_resources(struct wx_ring *tx_ring)
2385 {
2386         wx_clean_tx_ring(tx_ring);
2387         kvfree(tx_ring->tx_buffer_info);
2388         tx_ring->tx_buffer_info = NULL;
2389
2390         /* if not set, then don't free */
2391         if (!tx_ring->desc)
2392                 return;
2393
2394         dma_free_coherent(tx_ring->dev, tx_ring->size,
2395                           tx_ring->desc, tx_ring->dma);
2396         tx_ring->desc = NULL;
2397 }
2398
2399 /**
2400  * wx_free_all_tx_resources - Free Tx Resources for All Queues
2401  * @wx: pointer to hardware structure
2402  *
2403  * Free all transmit software resources
2404  **/
2405 static void wx_free_all_tx_resources(struct wx *wx)
2406 {
2407         int i;
2408
2409         for (i = 0; i < wx->num_tx_queues; i++)
2410                 wx_free_tx_resources(wx->tx_ring[i]);
2411 }
2412
2413 void wx_free_resources(struct wx *wx)
2414 {
2415         wx_free_all_rx_resources(wx);
2416         wx_free_all_tx_resources(wx);
2417 }
2418 EXPORT_SYMBOL(wx_free_resources);
2419
2420 static int wx_alloc_page_pool(struct wx_ring *rx_ring)
2421 {
2422         int ret = 0;
2423
2424         struct page_pool_params pp_params = {
2425                 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
2426                 .order = 0,
2427                 .pool_size = rx_ring->size,
2428                 .nid = dev_to_node(rx_ring->dev),
2429                 .dev = rx_ring->dev,
2430                 .dma_dir = DMA_FROM_DEVICE,
2431                 .offset = 0,
2432                 .max_len = PAGE_SIZE,
2433         };
2434
2435         rx_ring->page_pool = page_pool_create(&pp_params);
2436         if (IS_ERR(rx_ring->page_pool)) {
2437                 ret = PTR_ERR(rx_ring->page_pool);
2438                 rx_ring->page_pool = NULL;
2439         }
2440
2441         return ret;
2442 }
2443
2444 /**
2445  * wx_setup_rx_resources - allocate Rx resources (Descriptors)
2446  * @rx_ring: rx descriptor ring (for a specific queue) to setup
2447  *
2448  * Returns 0 on success, negative on failure
2449  **/
2450 static int wx_setup_rx_resources(struct wx_ring *rx_ring)
2451 {
2452         struct device *dev = rx_ring->dev;
2453         int orig_node = dev_to_node(dev);
2454         int numa_node = NUMA_NO_NODE;
2455         int size, ret;
2456
2457         size = sizeof(struct wx_rx_buffer) * rx_ring->count;
2458
2459         if (rx_ring->q_vector)
2460                 numa_node = rx_ring->q_vector->numa_node;
2461
2462         rx_ring->rx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);
2463         if (!rx_ring->rx_buffer_info)
2464                 rx_ring->rx_buffer_info = kvmalloc(size, GFP_KERNEL);
2465         if (!rx_ring->rx_buffer_info)
2466                 goto err;
2467
2468         /* Round up to nearest 4K */
2469         rx_ring->size = rx_ring->count * sizeof(union wx_rx_desc);
2470         rx_ring->size = ALIGN(rx_ring->size, 4096);
2471
2472         set_dev_node(dev, numa_node);
2473         rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
2474                                            &rx_ring->dma, GFP_KERNEL);
2475         if (!rx_ring->desc) {
2476                 set_dev_node(dev, orig_node);
2477                 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
2478                                                    &rx_ring->dma, GFP_KERNEL);
2479         }
2480
2481         if (!rx_ring->desc)
2482                 goto err;
2483
2484         rx_ring->next_to_clean = 0;
2485         rx_ring->next_to_use = 0;
2486
2487         ret = wx_alloc_page_pool(rx_ring);
2488         if (ret < 0) {
2489                 dev_err(rx_ring->dev, "Page pool creation failed: %d\n", ret);
2490                 goto err_desc;
2491         }
2492
2493         return 0;
2494
2495 err_desc:
2496         dma_free_coherent(dev, rx_ring->size, rx_ring->desc, rx_ring->dma);
2497 err:
2498         kvfree(rx_ring->rx_buffer_info);
2499         rx_ring->rx_buffer_info = NULL;
2500         dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
2501         return -ENOMEM;
2502 }
2503
2504 /**
2505  * wx_setup_all_rx_resources - allocate all queues Rx resources
2506  * @wx: pointer to hardware structure
2507  *
2508  * If this function returns with an error, then it's possible one or
2509  * more of the rings is populated (while the rest are not).  It is the
2510  * callers duty to clean those orphaned rings.
2511  *
2512  * Return 0 on success, negative on failure
2513  **/
2514 static int wx_setup_all_rx_resources(struct wx *wx)
2515 {
2516         int i, err = 0;
2517
2518         for (i = 0; i < wx->num_rx_queues; i++) {
2519                 err = wx_setup_rx_resources(wx->rx_ring[i]);
2520                 if (!err)
2521                         continue;
2522
2523                 wx_err(wx, "Allocation for Rx Queue %u failed\n", i);
2524                 goto err_setup_rx;
2525         }
2526
2527         return 0;
2528 err_setup_rx:
2529         /* rewind the index freeing the rings as we go */
2530         while (i--)
2531                 wx_free_rx_resources(wx->rx_ring[i]);
2532         return err;
2533 }
2534
2535 /**
2536  * wx_setup_tx_resources - allocate Tx resources (Descriptors)
2537  * @tx_ring: tx descriptor ring (for a specific queue) to setup
2538  *
2539  * Return 0 on success, negative on failure
2540  **/
2541 static int wx_setup_tx_resources(struct wx_ring *tx_ring)
2542 {
2543         struct device *dev = tx_ring->dev;
2544         int orig_node = dev_to_node(dev);
2545         int numa_node = NUMA_NO_NODE;
2546         int size;
2547
2548         size = sizeof(struct wx_tx_buffer) * tx_ring->count;
2549
2550         if (tx_ring->q_vector)
2551                 numa_node = tx_ring->q_vector->numa_node;
2552
2553         tx_ring->tx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);
2554         if (!tx_ring->tx_buffer_info)
2555                 tx_ring->tx_buffer_info = kvmalloc(size, GFP_KERNEL);
2556         if (!tx_ring->tx_buffer_info)
2557                 goto err;
2558
2559         /* round up to nearest 4K */
2560         tx_ring->size = tx_ring->count * sizeof(union wx_tx_desc);
2561         tx_ring->size = ALIGN(tx_ring->size, 4096);
2562
2563         set_dev_node(dev, numa_node);
2564         tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
2565                                            &tx_ring->dma, GFP_KERNEL);
2566         if (!tx_ring->desc) {
2567                 set_dev_node(dev, orig_node);
2568                 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
2569                                                    &tx_ring->dma, GFP_KERNEL);
2570         }
2571
2572         if (!tx_ring->desc)
2573                 goto err;
2574
2575         tx_ring->next_to_use = 0;
2576         tx_ring->next_to_clean = 0;
2577
2578         return 0;
2579
2580 err:
2581         kvfree(tx_ring->tx_buffer_info);
2582         tx_ring->tx_buffer_info = NULL;
2583         dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
2584         return -ENOMEM;
2585 }
2586
2587 /**
2588  * wx_setup_all_tx_resources - allocate all queues Tx resources
2589  * @wx: pointer to private structure
2590  *
2591  * If this function returns with an error, then it's possible one or
2592  * more of the rings is populated (while the rest are not).  It is the
2593  * callers duty to clean those orphaned rings.
2594  *
2595  * Return 0 on success, negative on failure
2596  **/
2597 static int wx_setup_all_tx_resources(struct wx *wx)
2598 {
2599         int i, err = 0;
2600
2601         for (i = 0; i < wx->num_tx_queues; i++) {
2602                 err = wx_setup_tx_resources(wx->tx_ring[i]);
2603                 if (!err)
2604                         continue;
2605
2606                 wx_err(wx, "Allocation for Tx Queue %u failed\n", i);
2607                 goto err_setup_tx;
2608         }
2609
2610         return 0;
2611 err_setup_tx:
2612         /* rewind the index freeing the rings as we go */
2613         while (i--)
2614                 wx_free_tx_resources(wx->tx_ring[i]);
2615         return err;
2616 }
2617
2618 int wx_setup_resources(struct wx *wx)
2619 {
2620         int err;
2621
2622         /* allocate transmit descriptors */
2623         err = wx_setup_all_tx_resources(wx);
2624         if (err)
2625                 return err;
2626
2627         /* allocate receive descriptors */
2628         err = wx_setup_all_rx_resources(wx);
2629         if (err)
2630                 goto err_free_tx;
2631
2632         err = wx_setup_isb_resources(wx);
2633         if (err)
2634                 goto err_free_rx;
2635
2636         return 0;
2637
2638 err_free_rx:
2639         wx_free_all_rx_resources(wx);
2640 err_free_tx:
2641         wx_free_all_tx_resources(wx);
2642
2643         return err;
2644 }
2645 EXPORT_SYMBOL(wx_setup_resources);
2646
2647 /**
2648  * wx_get_stats64 - Get System Network Statistics
2649  * @netdev: network interface device structure
2650  * @stats: storage space for 64bit statistics
2651  */
2652 void wx_get_stats64(struct net_device *netdev,
2653                     struct rtnl_link_stats64 *stats)
2654 {
2655         struct wx *wx = netdev_priv(netdev);
2656         struct wx_hw_stats *hwstats;
2657         int i;
2658
2659         wx_update_stats(wx);
2660
2661         rcu_read_lock();
2662         for (i = 0; i < wx->num_rx_queues; i++) {
2663                 struct wx_ring *ring = READ_ONCE(wx->rx_ring[i]);
2664                 u64 bytes, packets;
2665                 unsigned int start;
2666
2667                 if (ring) {
2668                         do {
2669                                 start = u64_stats_fetch_begin(&ring->syncp);
2670                                 packets = ring->stats.packets;
2671                                 bytes   = ring->stats.bytes;
2672                         } while (u64_stats_fetch_retry(&ring->syncp, start));
2673                         stats->rx_packets += packets;
2674                         stats->rx_bytes   += bytes;
2675                 }
2676         }
2677
2678         for (i = 0; i < wx->num_tx_queues; i++) {
2679                 struct wx_ring *ring = READ_ONCE(wx->tx_ring[i]);
2680                 u64 bytes, packets;
2681                 unsigned int start;
2682
2683                 if (ring) {
2684                         do {
2685                                 start = u64_stats_fetch_begin(&ring->syncp);
2686                                 packets = ring->stats.packets;
2687                                 bytes   = ring->stats.bytes;
2688                         } while (u64_stats_fetch_retry(&ring->syncp,
2689                                                            start));
2690                         stats->tx_packets += packets;
2691                         stats->tx_bytes   += bytes;
2692                 }
2693         }
2694
2695         rcu_read_unlock();
2696
2697         hwstats = &wx->stats;
2698         stats->rx_errors = hwstats->crcerrs + hwstats->rlec;
2699         stats->multicast = hwstats->qmprc;
2700         stats->rx_length_errors = hwstats->rlec;
2701         stats->rx_crc_errors = hwstats->crcerrs;
2702 }
2703 EXPORT_SYMBOL(wx_get_stats64);
2704
2705 int wx_set_features(struct net_device *netdev, netdev_features_t features)
2706 {
2707         netdev_features_t changed = netdev->features ^ features;
2708         struct wx *wx = netdev_priv(netdev);
2709         bool need_reset = false;
2710
2711         if (features & NETIF_F_RXHASH) {
2712                 wr32m(wx, WX_RDB_RA_CTL, WX_RDB_RA_CTL_RSS_EN,
2713                       WX_RDB_RA_CTL_RSS_EN);
2714                 wx->rss_enabled = true;
2715         } else {
2716                 wr32m(wx, WX_RDB_RA_CTL, WX_RDB_RA_CTL_RSS_EN, 0);
2717                 wx->rss_enabled = false;
2718         }
2719
2720         netdev->features = features;
2721
2722         if (wx->mac.type == wx_mac_sp && changed & NETIF_F_HW_VLAN_CTAG_RX)
2723                 wx->do_reset(netdev);
2724         else if (changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER))
2725                 wx_set_rx_mode(netdev);
2726
2727         if (!(test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags)))
2728                 return 0;
2729
2730         /* Check if Flow Director n-tuple support was enabled or disabled.  If
2731          * the state changed, we need to reset.
2732          */
2733         switch (features & NETIF_F_NTUPLE) {
2734         case NETIF_F_NTUPLE:
2735                 /* turn off ATR, enable perfect filters and reset */
2736                 if (!(test_and_set_bit(WX_FLAG_FDIR_PERFECT, wx->flags)))
2737                         need_reset = true;
2738
2739                 clear_bit(WX_FLAG_FDIR_HASH, wx->flags);
2740                 break;
2741         default:
2742                 /* turn off perfect filters, enable ATR and reset */
2743                 if (test_and_clear_bit(WX_FLAG_FDIR_PERFECT, wx->flags))
2744                         need_reset = true;
2745
2746                 /* We cannot enable ATR if RSS is disabled */
2747                 if (wx->ring_feature[RING_F_RSS].limit <= 1)
2748                         break;
2749
2750                 set_bit(WX_FLAG_FDIR_HASH, wx->flags);
2751                 break;
2752         }
2753
2754         if (need_reset)
2755                 wx->do_reset(netdev);
2756
2757         return 0;
2758 }
2759 EXPORT_SYMBOL(wx_set_features);
2760
2761 #define NETIF_VLAN_STRIPPING_FEATURES   (NETIF_F_HW_VLAN_CTAG_RX | \
2762                                          NETIF_F_HW_VLAN_STAG_RX)
2763
2764 #define NETIF_VLAN_INSERTION_FEATURES   (NETIF_F_HW_VLAN_CTAG_TX | \
2765                                          NETIF_F_HW_VLAN_STAG_TX)
2766
2767 #define NETIF_VLAN_FILTERING_FEATURES   (NETIF_F_HW_VLAN_CTAG_FILTER | \
2768                                          NETIF_F_HW_VLAN_STAG_FILTER)
2769
2770 netdev_features_t wx_fix_features(struct net_device *netdev,
2771                                   netdev_features_t features)
2772 {
2773         netdev_features_t changed = netdev->features ^ features;
2774         struct wx *wx = netdev_priv(netdev);
2775
2776         if (changed & NETIF_VLAN_STRIPPING_FEATURES) {
2777                 if ((features & NETIF_VLAN_STRIPPING_FEATURES) != NETIF_VLAN_STRIPPING_FEATURES &&
2778                     (features & NETIF_VLAN_STRIPPING_FEATURES) != 0) {
2779                         features &= ~NETIF_VLAN_STRIPPING_FEATURES;
2780                         features |= netdev->features & NETIF_VLAN_STRIPPING_FEATURES;
2781                         wx_err(wx, "802.1Q and 802.1ad VLAN stripping must be either both on or both off.");
2782                 }
2783         }
2784
2785         if (changed & NETIF_VLAN_INSERTION_FEATURES) {
2786                 if ((features & NETIF_VLAN_INSERTION_FEATURES) != NETIF_VLAN_INSERTION_FEATURES &&
2787                     (features & NETIF_VLAN_INSERTION_FEATURES) != 0) {
2788                         features &= ~NETIF_VLAN_INSERTION_FEATURES;
2789                         features |= netdev->features & NETIF_VLAN_INSERTION_FEATURES;
2790                         wx_err(wx, "802.1Q and 802.1ad VLAN insertion must be either both on or both off.");
2791                 }
2792         }
2793
2794         if (changed & NETIF_VLAN_FILTERING_FEATURES) {
2795                 if ((features & NETIF_VLAN_FILTERING_FEATURES) != NETIF_VLAN_FILTERING_FEATURES &&
2796                     (features & NETIF_VLAN_FILTERING_FEATURES) != 0) {
2797                         features &= ~NETIF_VLAN_FILTERING_FEATURES;
2798                         features |= netdev->features & NETIF_VLAN_FILTERING_FEATURES;
2799                         wx_err(wx, "802.1Q and 802.1ad VLAN filtering must be either both on or both off.");
2800                 }
2801         }
2802
2803         return features;
2804 }
2805 EXPORT_SYMBOL(wx_fix_features);
2806
2807 void wx_set_ring(struct wx *wx, u32 new_tx_count,
2808                  u32 new_rx_count, struct wx_ring *temp_ring)
2809 {
2810         int i, err = 0;
2811
2812         /* Setup new Tx resources and free the old Tx resources in that order.
2813          * We can then assign the new resources to the rings via a memcpy.
2814          * The advantage to this approach is that we are guaranteed to still
2815          * have resources even in the case of an allocation failure.
2816          */
2817         if (new_tx_count != wx->tx_ring_count) {
2818                 for (i = 0; i < wx->num_tx_queues; i++) {
2819                         memcpy(&temp_ring[i], wx->tx_ring[i],
2820                                sizeof(struct wx_ring));
2821
2822                         temp_ring[i].count = new_tx_count;
2823                         err = wx_setup_tx_resources(&temp_ring[i]);
2824                         if (err) {
2825                                 wx_err(wx, "setup new tx resources failed, keep using the old config\n");
2826                                 while (i) {
2827                                         i--;
2828                                         wx_free_tx_resources(&temp_ring[i]);
2829                                 }
2830                                 return;
2831                         }
2832                 }
2833
2834                 for (i = 0; i < wx->num_tx_queues; i++) {
2835                         wx_free_tx_resources(wx->tx_ring[i]);
2836
2837                         memcpy(wx->tx_ring[i], &temp_ring[i],
2838                                sizeof(struct wx_ring));
2839                 }
2840
2841                 wx->tx_ring_count = new_tx_count;
2842         }
2843
2844         /* Repeat the process for the Rx rings if needed */
2845         if (new_rx_count != wx->rx_ring_count) {
2846                 for (i = 0; i < wx->num_rx_queues; i++) {
2847                         memcpy(&temp_ring[i], wx->rx_ring[i],
2848                                sizeof(struct wx_ring));
2849
2850                         temp_ring[i].count = new_rx_count;
2851                         err = wx_setup_rx_resources(&temp_ring[i]);
2852                         if (err) {
2853                                 wx_err(wx, "setup new rx resources failed, keep using the old config\n");
2854                                 while (i) {
2855                                         i--;
2856                                         wx_free_rx_resources(&temp_ring[i]);
2857                                 }
2858                                 return;
2859                         }
2860                 }
2861
2862                 for (i = 0; i < wx->num_rx_queues; i++) {
2863                         wx_free_rx_resources(wx->rx_ring[i]);
2864                         memcpy(wx->rx_ring[i], &temp_ring[i],
2865                                sizeof(struct wx_ring));
2866                 }
2867
2868                 wx->rx_ring_count = new_rx_count;
2869         }
2870 }
2871 EXPORT_SYMBOL(wx_set_ring);
2872
2873 MODULE_DESCRIPTION("Common library for Wangxun(R) Ethernet drivers.");
2874 MODULE_LICENSE("GPL");