drivers/net/ethernet/freescale/enetc/enetc.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2 /* Copyright 2017-2019 NXP */
   3
   4 #include "enetc.h"
   5 #include <linux/tcp.h>
   6 #include <linux/udp.h>
   7 #include <linux/vmalloc.h>
   8
   9 /* ENETC overhead: optional extension BD + 1 BD gap */
  10 #define ENETC_TXBDS_NEEDED(val) ((val) + 2)
  11 /* max # of chained Tx BDs is 15, including head and extension BD */
  12 #define ENETC_MAX_SKB_FRAGS     13
  13 #define ENETC_TXBDS_MAX_NEEDED  ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1)
  14
  15 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
  16                               int active_offloads);
  17
  18 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
  19 {
  20         struct enetc_ndev_priv *priv = netdev_priv(ndev);
  21         struct enetc_bdr *tx_ring;
  22         int count;
  23
  24         tx_ring = priv->tx_ring[skb->queue_mapping];
  25
  26         if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
  27                 if (unlikely(skb_linearize(skb)))
  28                         goto drop_packet_err;
  29
  30         count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
  31         if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
  32                 netif_stop_subqueue(ndev, tx_ring->index);
  33                 return NETDEV_TX_BUSY;
  34         }
  35
  36         count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads);
  37         if (unlikely(!count))
  38                 goto drop_packet_err;
  39
  40         if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
  41                 netif_stop_subqueue(ndev, tx_ring->index);
  42
  43         return NETDEV_TX_OK;
  44
  45 drop_packet_err:
  46         dev_kfree_skb_any(skb);
  47         return NETDEV_TX_OK;
  48 }
  49
  50 static bool enetc_tx_csum(struct sk_buff *skb, union enetc_tx_bd *txbd)
  51 {
  52         int l3_start, l3_hsize;
  53         u16 l3_flags, l4_flags;
  54
  55         if (skb->ip_summed != CHECKSUM_PARTIAL)
  56                 return false;
  57
  58         switch (skb->csum_offset) {
  59         case offsetof(struct tcphdr, check):
  60                 l4_flags = ENETC_TXBD_L4_TCP;
  61                 break;
  62         case offsetof(struct udphdr, check):
  63                 l4_flags = ENETC_TXBD_L4_UDP;
  64                 break;
  65         default:
  66                 skb_checksum_help(skb);
  67                 return false;
  68         }
  69
  70         l3_start = skb_network_offset(skb);
  71         l3_hsize = skb_network_header_len(skb);
  72
  73         l3_flags = 0;
  74         if (skb->protocol == htons(ETH_P_IPV6))
  75                 l3_flags = ENETC_TXBD_L3_IPV6;
  76
  77         /* write BD fields */
  78         txbd->l3_csoff = enetc_txbd_l3_csoff(l3_start, l3_hsize, l3_flags);
  79         txbd->l4_csoff = l4_flags;
  80
  81         return true;
  82 }
  83
  84 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
  85                                 struct enetc_tx_swbd *tx_swbd)
  86 {
  87         if (tx_swbd->is_dma_page)
  88                 dma_unmap_page(tx_ring->dev, tx_swbd->dma,
  89                                tx_swbd->len, DMA_TO_DEVICE);
  90         else
  91                 dma_unmap_single(tx_ring->dev, tx_swbd->dma,
  92                                  tx_swbd->len, DMA_TO_DEVICE);
  93         tx_swbd->dma = 0;
  94 }
  95
  96 static void enetc_free_tx_skb(struct enetc_bdr *tx_ring,
  97                               struct enetc_tx_swbd *tx_swbd)
  98 {
  99         if (tx_swbd->dma)
 100                 enetc_unmap_tx_buff(tx_ring, tx_swbd);
 101
 102         if (tx_swbd->skb) {
 103                 dev_kfree_skb_any(tx_swbd->skb);
 104                 tx_swbd->skb = NULL;
 105         }
 106 }
 107
 108 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
 109                               int active_offloads)
 110 {
 111         struct enetc_tx_swbd *tx_swbd;
 112         skb_frag_t *frag;
 113         int len = skb_headlen(skb);
 114         union enetc_tx_bd temp_bd;
 115         union enetc_tx_bd *txbd;
 116         bool do_vlan, do_tstamp;
 117         int i, count = 0;
 118         unsigned int f;
 119         dma_addr_t dma;
 120         u8 flags = 0;
 121
 122         i = tx_ring->next_to_use;
 123         txbd = ENETC_TXBD(*tx_ring, i);
 124         prefetchw(txbd);
 125
 126         dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
 127         if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
 128                 goto dma_err;
 129
 130         temp_bd.addr = cpu_to_le64(dma);
 131         temp_bd.buf_len = cpu_to_le16(len);
 132         temp_bd.lstatus = 0;
 133
 134         tx_swbd = &tx_ring->tx_swbd[i];
 135         tx_swbd->dma = dma;
 136         tx_swbd->len = len;
 137         tx_swbd->is_dma_page = 0;
 138         count++;
 139
 140         do_vlan = skb_vlan_tag_present(skb);
 141         do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) &&
 142                     (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP);
 143         tx_swbd->do_tstamp = do_tstamp;
 144         tx_swbd->check_wb = tx_swbd->do_tstamp;
 145
 146         if (do_vlan || do_tstamp)
 147                 flags |= ENETC_TXBD_FLAGS_EX;
 148
 149         if (enetc_tx_csum(skb, &temp_bd))
 150                 flags |= ENETC_TXBD_FLAGS_CSUM | ENETC_TXBD_FLAGS_L4CS;
 151         else if (tx_ring->tsd_enable)
 152                 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
 153
 154         /* first BD needs frm_len and offload flags set */
 155         temp_bd.frm_len = cpu_to_le16(skb->len);
 156         temp_bd.flags = flags;
 157
 158         if (flags & ENETC_TXBD_FLAGS_TSE) {
 159                 u32 temp;
 160
 161                 temp = (skb->skb_mstamp_ns >> 5 & ENETC_TXBD_TXSTART_MASK)
 162                         | (flags << ENETC_TXBD_FLAGS_OFFSET);
 163                 temp_bd.txstart = cpu_to_le32(temp);
 164         }
 165
 166         if (flags & ENETC_TXBD_FLAGS_EX) {
 167                 u8 e_flags = 0;
 168                 *txbd = temp_bd;
 169                 enetc_clear_tx_bd(&temp_bd);
 170
 171                 /* add extension BD for VLAN and/or timestamping */
 172                 flags = 0;
 173                 tx_swbd++;
 174                 txbd++;
 175                 i++;
 176                 if (unlikely(i == tx_ring->bd_count)) {
 177                         i = 0;
 178                         tx_swbd = tx_ring->tx_swbd;
 179                         txbd = ENETC_TXBD(*tx_ring, 0);
 180                 }
 181                 prefetchw(txbd);
 182
 183                 if (do_vlan) {
 184                         temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
 185                         temp_bd.ext.tpid = 0; /* < C-TAG */
 186                         e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
 187                 }
 188
 189                 if (do_tstamp) {
 190                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 191                         e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
 192                 }
 193
 194                 temp_bd.ext.e_flags = e_flags;
 195                 count++;
 196         }
 197
 198         frag = &skb_shinfo(skb)->frags[0];
 199         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
 200                 len = skb_frag_size(frag);
 201                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
 202                                        DMA_TO_DEVICE);
 203                 if (dma_mapping_error(tx_ring->dev, dma))
 204                         goto dma_err;
 205
 206                 *txbd = temp_bd;
 207                 enetc_clear_tx_bd(&temp_bd);
 208
 209                 flags = 0;
 210                 tx_swbd++;
 211                 txbd++;
 212                 i++;
 213                 if (unlikely(i == tx_ring->bd_count)) {
 214                         i = 0;
 215                         tx_swbd = tx_ring->tx_swbd;
 216                         txbd = ENETC_TXBD(*tx_ring, 0);
 217                 }
 218                 prefetchw(txbd);
 219
 220                 temp_bd.addr = cpu_to_le64(dma);
 221                 temp_bd.buf_len = cpu_to_le16(len);
 222
 223                 tx_swbd->dma = dma;
 224                 tx_swbd->len = len;
 225                 tx_swbd->is_dma_page = 1;
 226                 count++;
 227         }
 228
 229         /* last BD needs 'F' bit set */
 230         flags |= ENETC_TXBD_FLAGS_F;
 231         temp_bd.flags = flags;
 232         *txbd = temp_bd;
 233
 234         tx_ring->tx_swbd[i].skb = skb;
 235
 236         enetc_bdr_idx_inc(tx_ring, &i);
 237         tx_ring->next_to_use = i;
 238
 239         skb_tx_timestamp(skb);
 240
 241         /* let H/W know BD ring has been updated */
 242         enetc_wr_reg(tx_ring->tpir, i); /* includes wmb() */
 243
 244         return count;
 245
 246 dma_err:
 247         dev_err(tx_ring->dev, "DMA map error");
 248
 249         do {
 250                 tx_swbd = &tx_ring->tx_swbd[i];
 251                 enetc_free_tx_skb(tx_ring, tx_swbd);
 252                 if (i == 0)
 253                         i = tx_ring->bd_count;
 254                 i--;
 255         } while (count--);
 256
 257         return 0;
 258 }
 259
 260 static irqreturn_t enetc_msix(int irq, void *data)
 261 {
 262         struct enetc_int_vector *v = data;
 263         int i;
 264
 265         /* disable interrupts */
 266         enetc_wr_reg(v->rbier, 0);
 267         enetc_wr_reg(v->ricr1, v->rx_ictt);
 268
 269         for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
 270                 enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 0);
 271
 272         napi_schedule(&v->napi);
 273
 274         return IRQ_HANDLED;
 275 }
 276
 277 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget);
 278 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
 279                                struct napi_struct *napi, int work_limit);
 280
 281 static void enetc_rx_dim_work(struct work_struct *w)
 282 {
 283         struct dim *dim = container_of(w, struct dim, work);
 284         struct dim_cq_moder moder =
 285                 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
 286         struct enetc_int_vector *v =
 287                 container_of(dim, struct enetc_int_vector, rx_dim);
 288
 289         v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
 290         dim->state = DIM_START_MEASURE;
 291 }
 292
 293 static void enetc_rx_net_dim(struct enetc_int_vector *v)
 294 {
 295         struct dim_sample dim_sample;
 296
 297         v->comp_cnt++;
 298
 299         if (!v->rx_napi_work)
 300                 return;
 301
 302         dim_update_sample(v->comp_cnt,
 303                           v->rx_ring.stats.packets,
 304                           v->rx_ring.stats.bytes,
 305                           &dim_sample);
 306         net_dim(&v->rx_dim, dim_sample);
 307 }
 308
 309 static int enetc_poll(struct napi_struct *napi, int budget)
 310 {
 311         struct enetc_int_vector
 312                 *v = container_of(napi, struct enetc_int_vector, napi);
 313         bool complete = true;
 314         int work_done;
 315         int i;
 316
 317         for (i = 0; i < v->count_tx_rings; i++)
 318                 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
 319                         complete = false;
 320
 321         work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget);
 322         if (work_done == budget)
 323                 complete = false;
 324         if (work_done)
 325                 v->rx_napi_work = true;
 326
 327         if (!complete)
 328                 return budget;
 329
 330         napi_complete_done(napi, work_done);
 331
 332         if (likely(v->rx_dim_en))
 333                 enetc_rx_net_dim(v);
 334
 335         v->rx_napi_work = false;
 336
 337         /* enable interrupts */
 338         enetc_wr_reg(v->rbier, ENETC_RBIER_RXTIE);
 339
 340         for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
 341                 enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i),
 342                              ENETC_TBIER_TXTIE);
 343
 344         return work_done;
 345 }
 346
 347 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
 348 {
 349         int pi = enetc_rd_reg(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
 350
 351         return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
 352 }
 353
 354 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
 355                                 u64 *tstamp)
 356 {
 357         u32 lo, hi, tstamp_lo;
 358
 359         lo = enetc_rd(hw, ENETC_SICTR0);
 360         hi = enetc_rd(hw, ENETC_SICTR1);
 361         tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
 362         if (lo <= tstamp_lo)
 363                 hi -= 1;
 364         *tstamp = (u64)hi << 32 | tstamp_lo;
 365 }
 366
 367 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
 368 {
 369         struct skb_shared_hwtstamps shhwtstamps;
 370
 371         if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
 372                 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
 373                 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
 374                 skb_tstamp_tx(skb, &shhwtstamps);
 375         }
 376 }
 377
 378 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
 379 {
 380         struct net_device *ndev = tx_ring->ndev;
 381         int tx_frm_cnt = 0, tx_byte_cnt = 0;
 382         struct enetc_tx_swbd *tx_swbd;
 383         int i, bds_to_clean;
 384         bool do_tstamp;
 385         u64 tstamp = 0;
 386
 387         i = tx_ring->next_to_clean;
 388         tx_swbd = &tx_ring->tx_swbd[i];
 389         bds_to_clean = enetc_bd_ready_count(tx_ring, i);
 390
 391         do_tstamp = false;
 392
 393         while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
 394                 bool is_eof = !!tx_swbd->skb;
 395
 396                 if (unlikely(tx_swbd->check_wb)) {
 397                         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 398                         union enetc_tx_bd *txbd;
 399
 400                         txbd = ENETC_TXBD(*tx_ring, i);
 401
 402                         if (txbd->flags & ENETC_TXBD_FLAGS_W &&
 403                             tx_swbd->do_tstamp) {
 404                                 enetc_get_tx_tstamp(&priv->si->hw, txbd,
 405                                                     &tstamp);
 406                                 do_tstamp = true;
 407                         }
 408                 }
 409
 410                 if (likely(tx_swbd->dma))
 411                         enetc_unmap_tx_buff(tx_ring, tx_swbd);
 412
 413                 if (is_eof) {
 414                         if (unlikely(do_tstamp)) {
 415                                 enetc_tstamp_tx(tx_swbd->skb, tstamp);
 416                                 do_tstamp = false;
 417                         }
 418                         napi_consume_skb(tx_swbd->skb, napi_budget);
 419                         tx_swbd->skb = NULL;
 420                 }
 421
 422                 tx_byte_cnt += tx_swbd->len;
 423
 424                 bds_to_clean--;
 425                 tx_swbd++;
 426                 i++;
 427                 if (unlikely(i == tx_ring->bd_count)) {
 428                         i = 0;
 429                         tx_swbd = tx_ring->tx_swbd;
 430                 }
 431
 432                 /* BD iteration loop end */
 433                 if (is_eof) {
 434                         tx_frm_cnt++;
 435                         /* re-arm interrupt source */
 436                         enetc_wr_reg(tx_ring->idr, BIT(tx_ring->index) |
 437                                      BIT(16 + tx_ring->index));
 438                 }
 439
 440                 if (unlikely(!bds_to_clean))
 441                         bds_to_clean = enetc_bd_ready_count(tx_ring, i);
 442         }
 443
 444         tx_ring->next_to_clean = i;
 445         tx_ring->stats.packets += tx_frm_cnt;
 446         tx_ring->stats.bytes += tx_byte_cnt;
 447
 448         if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
 449                      __netif_subqueue_stopped(ndev, tx_ring->index) &&
 450                      (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
 451                 netif_wake_subqueue(ndev, tx_ring->index);
 452         }
 453
 454         return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
 455 }
 456
 457 static bool enetc_new_page(struct enetc_bdr *rx_ring,
 458                            struct enetc_rx_swbd *rx_swbd)
 459 {
 460         struct page *page;
 461         dma_addr_t addr;
 462
 463         page = dev_alloc_page();
 464         if (unlikely(!page))
 465                 return false;
 466
 467         addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
 468         if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
 469                 __free_page(page);
 470
 471                 return false;
 472         }
 473
 474         rx_swbd->dma = addr;
 475         rx_swbd->page = page;
 476         rx_swbd->page_offset = ENETC_RXB_PAD;
 477
 478         return true;
 479 }
 480
 481 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
 482 {
 483         struct enetc_rx_swbd *rx_swbd;
 484         union enetc_rx_bd *rxbd;
 485         int i, j;
 486
 487         i = rx_ring->next_to_use;
 488         rx_swbd = &rx_ring->rx_swbd[i];
 489         rxbd = enetc_rxbd(rx_ring, i);
 490
 491         for (j = 0; j < buff_cnt; j++) {
 492                 /* try reuse page */
 493                 if (unlikely(!rx_swbd->page)) {
 494                         if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
 495                                 rx_ring->stats.rx_alloc_errs++;
 496                                 break;
 497                         }
 498                 }
 499
 500                 /* update RxBD */
 501                 rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
 502                                            rx_swbd->page_offset);
 503                 /* clear 'R" as well */
 504                 rxbd->r.lstatus = 0;
 505
 506                 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
 507                 rx_swbd++;
 508                 i++;
 509                 if (unlikely(i == rx_ring->bd_count)) {
 510                         i = 0;
 511                         rx_swbd = rx_ring->rx_swbd;
 512                 }
 513         }
 514
 515         if (likely(j)) {
 516                 rx_ring->next_to_alloc = i; /* keep track from page reuse */
 517                 rx_ring->next_to_use = i;
 518                 /* update ENETC's consumer index */
 519                 enetc_wr_reg(rx_ring->rcir, i);
 520         }
 521
 522         return j;
 523 }
 524
 525 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
 526 static void enetc_get_rx_tstamp(struct net_device *ndev,
 527                                 union enetc_rx_bd *rxbd,
 528                                 struct sk_buff *skb)
 529 {
 530         struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
 531         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 532         struct enetc_hw *hw = &priv->si->hw;
 533         u32 lo, hi, tstamp_lo;
 534         u64 tstamp;
 535
 536         if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
 537                 lo = enetc_rd(hw, ENETC_SICTR0);
 538                 hi = enetc_rd(hw, ENETC_SICTR1);
 539                 rxbd = enetc_rxbd_ext(rxbd);
 540                 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
 541                 if (lo <= tstamp_lo)
 542                         hi -= 1;
 543
 544                 tstamp = (u64)hi << 32 | tstamp_lo;
 545                 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
 546                 shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
 547         }
 548 }
 549 #endif
 550
 551 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
 552                                union enetc_rx_bd *rxbd, struct sk_buff *skb)
 553 {
 554 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
 555         struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
 556 #endif
 557         /* TODO: hashing */
 558         if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
 559                 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
 560
 561                 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
 562                 skb->ip_summed = CHECKSUM_COMPLETE;
 563         }
 564
 565         /* copy VLAN to skb, if one is extracted, for now we assume it's a
 566          * standard TPID, but HW also supports custom values
 567          */
 568         if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN)
 569                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 570                                        le16_to_cpu(rxbd->r.vlan_opt));
 571 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
 572         if (priv->active_offloads & ENETC_F_RX_TSTAMP)
 573                 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
 574 #endif
 575 }
 576
 577 static void enetc_process_skb(struct enetc_bdr *rx_ring,
 578                               struct sk_buff *skb)
 579 {
 580         skb_record_rx_queue(skb, rx_ring->index);
 581         skb->protocol = eth_type_trans(skb, rx_ring->ndev);
 582 }
 583
 584 static bool enetc_page_reusable(struct page *page)
 585 {
 586         return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
 587 }
 588
 589 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
 590                              struct enetc_rx_swbd *old)
 591 {
 592         struct enetc_rx_swbd *new;
 593
 594         new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
 595
 596         /* next buf that may reuse a page */
 597         enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
 598
 599         /* copy page reference */
 600         *new = *old;
 601 }
 602
 603 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
 604                                                int i, u16 size)
 605 {
 606         struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
 607
 608         dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
 609                                       rx_swbd->page_offset,
 610                                       size, DMA_FROM_DEVICE);
 611         return rx_swbd;
 612 }
 613
 614 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
 615                               struct enetc_rx_swbd *rx_swbd)
 616 {
 617         if (likely(enetc_page_reusable(rx_swbd->page))) {
 618                 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
 619                 page_ref_inc(rx_swbd->page);
 620
 621                 enetc_reuse_page(rx_ring, rx_swbd);
 622
 623                 /* sync for use by the device */
 624                 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
 625                                                  rx_swbd->page_offset,
 626                                                  ENETC_RXB_DMA_SIZE,
 627                                                  DMA_FROM_DEVICE);
 628         } else {
 629                 dma_unmap_page(rx_ring->dev, rx_swbd->dma,
 630                                PAGE_SIZE, DMA_FROM_DEVICE);
 631         }
 632
 633         rx_swbd->page = NULL;
 634 }
 635
 636 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
 637                                                 int i, u16 size)
 638 {
 639         struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
 640         struct sk_buff *skb;
 641         void *ba;
 642
 643         ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
 644         skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE);
 645         if (unlikely(!skb)) {
 646                 rx_ring->stats.rx_alloc_errs++;
 647                 return NULL;
 648         }
 649
 650         skb_reserve(skb, ENETC_RXB_PAD);
 651         __skb_put(skb, size);
 652
 653         enetc_put_rx_buff(rx_ring, rx_swbd);
 654
 655         return skb;
 656 }
 657
 658 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
 659                                      u16 size, struct sk_buff *skb)
 660 {
 661         struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
 662
 663         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
 664                         rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
 665
 666         enetc_put_rx_buff(rx_ring, rx_swbd);
 667 }
 668
 669 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
 670
 671 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
 672                                struct napi_struct *napi, int work_limit)
 673 {
 674         int rx_frm_cnt = 0, rx_byte_cnt = 0;
 675         int cleaned_cnt, i;
 676
 677         cleaned_cnt = enetc_bd_unused(rx_ring);
 678         /* next descriptor to process */
 679         i = rx_ring->next_to_clean;
 680
 681         while (likely(rx_frm_cnt < work_limit)) {
 682                 union enetc_rx_bd *rxbd;
 683                 struct sk_buff *skb;
 684                 u32 bd_status;
 685                 u16 size;
 686
 687                 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) {
 688                         int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt);
 689
 690                         cleaned_cnt -= count;
 691                 }
 692
 693                 rxbd = enetc_rxbd(rx_ring, i);
 694                 bd_status = le32_to_cpu(rxbd->r.lstatus);
 695                 if (!bd_status)
 696                         break;
 697
 698                 enetc_wr_reg(rx_ring->idr, BIT(rx_ring->index));
 699                 dma_rmb(); /* for reading other rxbd fields */
 700                 size = le16_to_cpu(rxbd->r.buf_len);
 701                 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size);
 702                 if (!skb)
 703                         break;
 704
 705                 enetc_get_offloads(rx_ring, rxbd, skb);
 706
 707                 cleaned_cnt++;
 708
 709                 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
 710                 if (unlikely(++i == rx_ring->bd_count))
 711                         i = 0;
 712
 713                 if (unlikely(bd_status &
 714                              ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) {
 715                         dev_kfree_skb(skb);
 716                         while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
 717                                 dma_rmb();
 718                                 bd_status = le32_to_cpu(rxbd->r.lstatus);
 719
 720                                 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
 721                                 if (unlikely(++i == rx_ring->bd_count))
 722                                         i = 0;
 723                         }
 724
 725                         rx_ring->ndev->stats.rx_dropped++;
 726                         rx_ring->ndev->stats.rx_errors++;
 727
 728                         break;
 729                 }
 730
 731                 /* not last BD in frame? */
 732                 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
 733                         bd_status = le32_to_cpu(rxbd->r.lstatus);
 734                         size = ENETC_RXB_DMA_SIZE;
 735
 736                         if (bd_status & ENETC_RXBD_LSTATUS_F) {
 737                                 dma_rmb();
 738                                 size = le16_to_cpu(rxbd->r.buf_len);
 739                         }
 740
 741                         enetc_add_rx_buff_to_skb(rx_ring, i, size, skb);
 742
 743                         cleaned_cnt++;
 744
 745                         rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
 746                         if (unlikely(++i == rx_ring->bd_count))
 747                                 i = 0;
 748                 }
 749
 750                 rx_byte_cnt += skb->len;
 751
 752                 enetc_process_skb(rx_ring, skb);
 753
 754                 napi_gro_receive(napi, skb);
 755
 756                 rx_frm_cnt++;
 757         }
 758
 759         rx_ring->next_to_clean = i;
 760
 761         rx_ring->stats.packets += rx_frm_cnt;
 762         rx_ring->stats.bytes += rx_byte_cnt;
 763
 764         return rx_frm_cnt;
 765 }
 766
 767 /* Probing and Init */
 768 #define ENETC_MAX_RFS_SIZE 64
 769 void enetc_get_si_caps(struct enetc_si *si)
 770 {
 771         struct enetc_hw *hw = &si->hw;
 772         u32 val;
 773
 774         /* find out how many of various resources we have to work with */
 775         val = enetc_rd(hw, ENETC_SICAPR0);
 776         si->num_rx_rings = (val >> 16) & 0xff;
 777         si->num_tx_rings = val & 0xff;
 778
 779         val = enetc_rd(hw, ENETC_SIRFSCAPR);
 780         si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
 781         si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
 782
 783         si->num_rss = 0;
 784         val = enetc_rd(hw, ENETC_SIPCAPR0);
 785         if (val & ENETC_SIPCAPR0_RSS) {
 786                 u32 rss;
 787
 788                 rss = enetc_rd(hw, ENETC_SIRSSCAPR);
 789                 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
 790         }
 791
 792         if (val & ENETC_SIPCAPR0_QBV)
 793                 si->hw_features |= ENETC_SI_F_QBV;
 794
 795         if (val & ENETC_SIPCAPR0_PSFP)
 796                 si->hw_features |= ENETC_SI_F_PSFP;
 797 }
 798
 799 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size)
 800 {
 801         r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size,
 802                                         &r->bd_dma_base, GFP_KERNEL);
 803         if (!r->bd_base)
 804                 return -ENOMEM;
 805
 806         /* h/w requires 128B alignment */
 807         if (!IS_ALIGNED(r->bd_dma_base, 128)) {
 808                 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base,
 809                                   r->bd_dma_base);
 810                 return -EINVAL;
 811         }
 812
 813         return 0;
 814 }
 815
 816 static int enetc_alloc_txbdr(struct enetc_bdr *txr)
 817 {
 818         int err;
 819
 820         txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd));
 821         if (!txr->tx_swbd)
 822                 return -ENOMEM;
 823
 824         err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd));
 825         if (err) {
 826                 vfree(txr->tx_swbd);
 827                 return err;
 828         }
 829
 830         txr->next_to_clean = 0;
 831         txr->next_to_use = 0;
 832
 833         return 0;
 834 }
 835
 836 static void enetc_free_txbdr(struct enetc_bdr *txr)
 837 {
 838         int size, i;
 839
 840         for (i = 0; i < txr->bd_count; i++)
 841                 enetc_free_tx_skb(txr, &txr->tx_swbd[i]);
 842
 843         size = txr->bd_count * sizeof(union enetc_tx_bd);
 844
 845         dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base);
 846         txr->bd_base = NULL;
 847
 848         vfree(txr->tx_swbd);
 849         txr->tx_swbd = NULL;
 850 }
 851
 852 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
 853 {
 854         int i, err;
 855
 856         for (i = 0; i < priv->num_tx_rings; i++) {
 857                 err = enetc_alloc_txbdr(priv->tx_ring[i]);
 858
 859                 if (err)
 860                         goto fail;
 861         }
 862
 863         return 0;
 864
 865 fail:
 866         while (i-- > 0)
 867                 enetc_free_txbdr(priv->tx_ring[i]);
 868
 869         return err;
 870 }
 871
 872 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv)
 873 {
 874         int i;
 875
 876         for (i = 0; i < priv->num_tx_rings; i++)
 877                 enetc_free_txbdr(priv->tx_ring[i]);
 878 }
 879
 880 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended)
 881 {
 882         size_t size = sizeof(union enetc_rx_bd);
 883         int err;
 884
 885         rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd));
 886         if (!rxr->rx_swbd)
 887                 return -ENOMEM;
 888
 889         if (extended)
 890                 size *= 2;
 891
 892         err = enetc_dma_alloc_bdr(rxr, size);
 893         if (err) {
 894                 vfree(rxr->rx_swbd);
 895                 return err;
 896         }
 897
 898         rxr->next_to_clean = 0;
 899         rxr->next_to_use = 0;
 900         rxr->next_to_alloc = 0;
 901         rxr->ext_en = extended;
 902
 903         return 0;
 904 }
 905
 906 static void enetc_free_rxbdr(struct enetc_bdr *rxr)
 907 {
 908         int size;
 909
 910         size = rxr->bd_count * sizeof(union enetc_rx_bd);
 911
 912         dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base);
 913         rxr->bd_base = NULL;
 914
 915         vfree(rxr->rx_swbd);
 916         rxr->rx_swbd = NULL;
 917 }
 918
 919 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv)
 920 {
 921         bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
 922         int i, err;
 923
 924         for (i = 0; i < priv->num_rx_rings; i++) {
 925                 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended);
 926
 927                 if (err)
 928                         goto fail;
 929         }
 930
 931         return 0;
 932
 933 fail:
 934         while (i-- > 0)
 935                 enetc_free_rxbdr(priv->rx_ring[i]);
 936
 937         return err;
 938 }
 939
 940 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv)
 941 {
 942         int i;
 943
 944         for (i = 0; i < priv->num_rx_rings; i++)
 945                 enetc_free_rxbdr(priv->rx_ring[i]);
 946 }
 947
 948 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
 949 {
 950         int i;
 951
 952         if (!tx_ring->tx_swbd)
 953                 return;
 954
 955         for (i = 0; i < tx_ring->bd_count; i++) {
 956                 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
 957
 958                 enetc_free_tx_skb(tx_ring, tx_swbd);
 959         }
 960
 961         tx_ring->next_to_clean = 0;
 962         tx_ring->next_to_use = 0;
 963 }
 964
 965 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
 966 {
 967         int i;
 968
 969         if (!rx_ring->rx_swbd)
 970                 return;
 971
 972         for (i = 0; i < rx_ring->bd_count; i++) {
 973                 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
 974
 975                 if (!rx_swbd->page)
 976                         continue;
 977
 978                 dma_unmap_page(rx_ring->dev, rx_swbd->dma,
 979                                PAGE_SIZE, DMA_FROM_DEVICE);
 980                 __free_page(rx_swbd->page);
 981                 rx_swbd->page = NULL;
 982         }
 983
 984         rx_ring->next_to_clean = 0;
 985         rx_ring->next_to_use = 0;
 986         rx_ring->next_to_alloc = 0;
 987 }
 988
 989 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
 990 {
 991         int i;
 992
 993         for (i = 0; i < priv->num_rx_rings; i++)
 994                 enetc_free_rx_ring(priv->rx_ring[i]);
 995
 996         for (i = 0; i < priv->num_tx_rings; i++)
 997                 enetc_free_tx_ring(priv->tx_ring[i]);
 998 }
 999
1000 static int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
1001 {
1002         int size = cbdr->bd_count * sizeof(struct enetc_cbd);
1003
1004         cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base,
1005                                            GFP_KERNEL);
1006         if (!cbdr->bd_base)
1007                 return -ENOMEM;
1008
1009         /* h/w requires 128B alignment */
1010         if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) {
1011                 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
1012                 return -EINVAL;
1013         }
1014
1015         cbdr->next_to_clean = 0;
1016         cbdr->next_to_use = 0;
1017
1018         return 0;
1019 }
1020
1021 static void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
1022 {
1023         int size = cbdr->bd_count * sizeof(struct enetc_cbd);
1024
1025         dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
1026         cbdr->bd_base = NULL;
1027 }
1028
1029 static void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr)
1030 {
1031         /* set CBDR cache attributes */
1032         enetc_wr(hw, ENETC_SICAR2,
1033                  ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1034
1035         enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base));
1036         enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base));
1037         enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count));
1038
1039         enetc_wr(hw, ENETC_SICBDRPIR, 0);
1040         enetc_wr(hw, ENETC_SICBDRCIR, 0);
1041
1042         /* enable ring */
1043         enetc_wr(hw, ENETC_SICBDRMR, BIT(31));
1044
1045         cbdr->pir = hw->reg + ENETC_SICBDRPIR;
1046         cbdr->cir = hw->reg + ENETC_SICBDRCIR;
1047 }
1048
1049 static void enetc_clear_cbdr(struct enetc_hw *hw)
1050 {
1051         enetc_wr(hw, ENETC_SICBDRMR, 0);
1052 }
1053
1054 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
1055 {
1056         int *rss_table;
1057         int i;
1058
1059         rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
1060         if (!rss_table)
1061                 return -ENOMEM;
1062
1063         /* Set up RSS table defaults */
1064         for (i = 0; i < si->num_rss; i++)
1065                 rss_table[i] = i % num_groups;
1066
1067         enetc_set_rss_table(si, rss_table, si->num_rss);
1068
1069         kfree(rss_table);
1070
1071         return 0;
1072 }
1073
1074 static int enetc_configure_si(struct enetc_ndev_priv *priv)
1075 {
1076         struct enetc_si *si = priv->si;
1077         struct enetc_hw *hw = &si->hw;
1078         int err;
1079
1080         enetc_setup_cbdr(hw, &si->cbd_ring);
1081         /* set SI cache attributes */
1082         enetc_wr(hw, ENETC_SICAR0,
1083                  ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1084         enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
1085         /* enable SI */
1086         enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
1087
1088         if (si->num_rss) {
1089                 err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
1090                 if (err)
1091                         return err;
1092         }
1093
1094         return 0;
1095 }
1096
1097 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
1098 {
1099         struct enetc_si *si = priv->si;
1100         int cpus = num_online_cpus();
1101
1102         priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
1103         priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
1104
1105         /* Enable all available TX rings in order to configure as many
1106          * priorities as possible, when needed.
1107          * TODO: Make # of TX rings run-time configurable
1108          */
1109         priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
1110         priv->num_tx_rings = si->num_tx_rings;
1111         priv->bdr_int_num = cpus;
1112         priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
1113         priv->tx_ictt = ENETC_TXIC_TIMETHR;
1114
1115         /* SI specific */
1116         si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE;
1117 }
1118
1119 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
1120 {
1121         struct enetc_si *si = priv->si;
1122         int err;
1123
1124         err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring);
1125         if (err)
1126                 return err;
1127
1128         priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
1129                                   GFP_KERNEL);
1130         if (!priv->cls_rules) {
1131                 err = -ENOMEM;
1132                 goto err_alloc_cls;
1133         }
1134
1135         err = enetc_configure_si(priv);
1136         if (err)
1137                 goto err_config_si;
1138
1139         return 0;
1140
1141 err_config_si:
1142         kfree(priv->cls_rules);
1143 err_alloc_cls:
1144         enetc_clear_cbdr(&si->hw);
1145         enetc_free_cbdr(priv->dev, &si->cbd_ring);
1146
1147         return err;
1148 }
1149
1150 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
1151 {
1152         struct enetc_si *si = priv->si;
1153
1154         enetc_clear_cbdr(&si->hw);
1155         enetc_free_cbdr(priv->dev, &si->cbd_ring);
1156
1157         kfree(priv->cls_rules);
1158 }
1159
1160 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1161 {
1162         int idx = tx_ring->index;
1163         u32 tbmr;
1164
1165         enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
1166                        lower_32_bits(tx_ring->bd_dma_base));
1167
1168         enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
1169                        upper_32_bits(tx_ring->bd_dma_base));
1170
1171         WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
1172         enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
1173                        ENETC_RTBLENR_LEN(tx_ring->bd_count));
1174
1175         /* clearing PI/CI registers for Tx not supported, adjust sw indexes */
1176         tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
1177         tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
1178
1179         /* enable Tx ints by setting pkt thr to 1 */
1180         enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
1181
1182         tbmr = ENETC_TBMR_EN;
1183         if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
1184                 tbmr |= ENETC_TBMR_VIH;
1185
1186         /* enable ring */
1187         enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
1188
1189         tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
1190         tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
1191         tx_ring->idr = hw->reg + ENETC_SITXIDR;
1192 }
1193
1194 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1195 {
1196         int idx = rx_ring->index;
1197         u32 rbmr;
1198
1199         enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
1200                        lower_32_bits(rx_ring->bd_dma_base));
1201
1202         enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
1203                        upper_32_bits(rx_ring->bd_dma_base));
1204
1205         WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
1206         enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
1207                        ENETC_RTBLENR_LEN(rx_ring->bd_count));
1208
1209         enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
1210
1211         enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
1212
1213         /* enable Rx ints by setting pkt thr to 1 */
1214         enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
1215
1216         rbmr = ENETC_RBMR_EN;
1217
1218         if (rx_ring->ext_en)
1219                 rbmr |= ENETC_RBMR_BDS;
1220
1221         if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
1222                 rbmr |= ENETC_RBMR_VTE;
1223
1224         rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
1225         rx_ring->idr = hw->reg + ENETC_SIRXIDR;
1226
1227         enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
1228
1229         /* enable ring */
1230         enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
1231 }
1232
1233 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv)
1234 {
1235         int i;
1236
1237         for (i = 0; i < priv->num_tx_rings; i++)
1238                 enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]);
1239
1240         for (i = 0; i < priv->num_rx_rings; i++)
1241                 enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]);
1242 }
1243
1244 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1245 {
1246         int idx = rx_ring->index;
1247
1248         /* disable EN bit on ring */
1249         enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
1250 }
1251
1252 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1253 {
1254         int delay = 8, timeout = 100;
1255         int idx = tx_ring->index;
1256
1257         /* disable EN bit on ring */
1258         enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
1259
1260         /* wait for busy to clear */
1261         while (delay < timeout &&
1262                enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
1263                 msleep(delay);
1264                 delay *= 2;
1265         }
1266
1267         if (delay >= timeout)
1268                 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
1269                             idx);
1270 }
1271
1272 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
1273 {
1274         int i;
1275
1276         for (i = 0; i < priv->num_tx_rings; i++)
1277                 enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]);
1278
1279         for (i = 0; i < priv->num_rx_rings; i++)
1280                 enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]);
1281
1282         udelay(1);
1283 }
1284
1285 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
1286 {
1287         struct pci_dev *pdev = priv->si->pdev;
1288         cpumask_t cpu_mask;
1289         int i, j, err;
1290
1291         for (i = 0; i < priv->bdr_int_num; i++) {
1292                 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1293                 struct enetc_int_vector *v = priv->int_vector[i];
1294                 int entry = ENETC_BDR_INT_BASE_IDX + i;
1295                 struct enetc_hw *hw = &priv->si->hw;
1296
1297                 snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
1298                          priv->ndev->name, i);
1299                 err = request_irq(irq, enetc_msix, 0, v->name, v);
1300                 if (err) {
1301                         dev_err(priv->dev, "request_irq() failed!\n");
1302                         goto irq_err;
1303                 }
1304                 disable_irq(irq);
1305
1306                 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
1307                 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
1308                 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
1309
1310                 enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
1311
1312                 for (j = 0; j < v->count_tx_rings; j++) {
1313                         int idx = v->tx_ring[j].index;
1314
1315                         enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
1316                 }
1317                 cpumask_clear(&cpu_mask);
1318                 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask);
1319                 irq_set_affinity_hint(irq, &cpu_mask);
1320         }
1321
1322         return 0;
1323
1324 irq_err:
1325         while (i--) {
1326                 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1327
1328                 irq_set_affinity_hint(irq, NULL);
1329                 free_irq(irq, priv->int_vector[i]);
1330         }
1331
1332         return err;
1333 }
1334
1335 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
1336 {
1337         struct pci_dev *pdev = priv->si->pdev;
1338         int i;
1339
1340         for (i = 0; i < priv->bdr_int_num; i++) {
1341                 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1342
1343                 irq_set_affinity_hint(irq, NULL);
1344                 free_irq(irq, priv->int_vector[i]);
1345         }
1346 }
1347
1348 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
1349 {
1350         struct enetc_hw *hw = &priv->si->hw;
1351         u32 icpt, ictt;
1352         int i;
1353
1354         /* enable Tx & Rx event indication */
1355         if (priv->ic_mode &
1356             (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
1357                 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
1358                 /* init to non-0 minimum, will be adjusted later */
1359                 ictt = 0x1;
1360         } else {
1361                 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
1362                 ictt = 0;
1363         }
1364
1365         for (i = 0; i < priv->num_rx_rings; i++) {
1366                 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
1367                 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
1368                 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
1369         }
1370
1371         if (priv->ic_mode & ENETC_IC_TX_MANUAL)
1372                 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
1373         else
1374                 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
1375
1376         for (i = 0; i < priv->num_tx_rings; i++) {
1377                 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
1378                 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
1379                 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
1380         }
1381 }
1382
1383 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
1384 {
1385         int i;
1386
1387         for (i = 0; i < priv->num_tx_rings; i++)
1388                 enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0);
1389
1390         for (i = 0; i < priv->num_rx_rings; i++)
1391                 enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0);
1392 }
1393
1394 static int enetc_phylink_connect(struct net_device *ndev)
1395 {
1396         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1397         struct ethtool_eee edata;
1398         int err;
1399
1400         if (!priv->phylink)
1401                 return 0; /* phy-less mode */
1402
1403         err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
1404         if (err) {
1405                 dev_err(&ndev->dev, "could not attach to PHY\n");
1406                 return err;
1407         }
1408
1409         /* disable EEE autoneg, until ENETC driver supports it */
1410         memset(&edata, 0, sizeof(struct ethtool_eee));
1411         phylink_ethtool_set_eee(priv->phylink, &edata);
1412
1413         return 0;
1414 }
1415
1416 void enetc_start(struct net_device *ndev)
1417 {
1418         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1419         int i;
1420
1421         enetc_setup_interrupts(priv);
1422
1423         for (i = 0; i < priv->bdr_int_num; i++) {
1424                 int irq = pci_irq_vector(priv->si->pdev,
1425                                          ENETC_BDR_INT_BASE_IDX + i);
1426
1427                 napi_enable(&priv->int_vector[i]->napi);
1428                 enable_irq(irq);
1429         }
1430
1431         if (priv->phylink)
1432                 phylink_start(priv->phylink);
1433         else
1434                 netif_carrier_on(ndev);
1435
1436         netif_tx_start_all_queues(ndev);
1437 }
1438
1439 int enetc_open(struct net_device *ndev)
1440 {
1441         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1442         int err;
1443
1444         err = enetc_setup_irqs(priv);
1445         if (err)
1446                 return err;
1447
1448         err = enetc_phylink_connect(ndev);
1449         if (err)
1450                 goto err_phy_connect;
1451
1452         err = enetc_alloc_tx_resources(priv);
1453         if (err)
1454                 goto err_alloc_tx;
1455
1456         err = enetc_alloc_rx_resources(priv);
1457         if (err)
1458                 goto err_alloc_rx;
1459
1460         err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1461         if (err)
1462                 goto err_set_queues;
1463
1464         err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings);
1465         if (err)
1466                 goto err_set_queues;
1467
1468         enetc_setup_bdrs(priv);
1469         enetc_start(ndev);
1470
1471         return 0;
1472
1473 err_set_queues:
1474         enetc_free_rx_resources(priv);
1475 err_alloc_rx:
1476         enetc_free_tx_resources(priv);
1477 err_alloc_tx:
1478         if (priv->phylink)
1479                 phylink_disconnect_phy(priv->phylink);
1480 err_phy_connect:
1481         enetc_free_irqs(priv);
1482
1483         return err;
1484 }
1485
1486 void enetc_stop(struct net_device *ndev)
1487 {
1488         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1489         int i;
1490
1491         netif_tx_stop_all_queues(ndev);
1492
1493         for (i = 0; i < priv->bdr_int_num; i++) {
1494                 int irq = pci_irq_vector(priv->si->pdev,
1495                                          ENETC_BDR_INT_BASE_IDX + i);
1496
1497                 disable_irq(irq);
1498                 napi_synchronize(&priv->int_vector[i]->napi);
1499                 napi_disable(&priv->int_vector[i]->napi);
1500         }
1501
1502         if (priv->phylink)
1503                 phylink_stop(priv->phylink);
1504         else
1505                 netif_carrier_off(ndev);
1506
1507         enetc_clear_interrupts(priv);
1508 }
1509
1510 int enetc_close(struct net_device *ndev)
1511 {
1512         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1513
1514         enetc_stop(ndev);
1515         enetc_clear_bdrs(priv);
1516
1517         if (priv->phylink)
1518                 phylink_disconnect_phy(priv->phylink);
1519         enetc_free_rxtx_rings(priv);
1520         enetc_free_rx_resources(priv);
1521         enetc_free_tx_resources(priv);
1522         enetc_free_irqs(priv);
1523
1524         return 0;
1525 }
1526
1527 static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
1528 {
1529         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1530         struct tc_mqprio_qopt *mqprio = type_data;
1531         struct enetc_bdr *tx_ring;
1532         u8 num_tc;
1533         int i;
1534
1535         mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
1536         num_tc = mqprio->num_tc;
1537
1538         if (!num_tc) {
1539                 netdev_reset_tc(ndev);
1540                 netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1541
1542                 /* Reset all ring priorities to 0 */
1543                 for (i = 0; i < priv->num_tx_rings; i++) {
1544                         tx_ring = priv->tx_ring[i];
1545                         enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0);
1546                 }
1547
1548                 return 0;
1549         }
1550
1551         /* Check if we have enough BD rings available to accommodate all TCs */
1552         if (num_tc > priv->num_tx_rings) {
1553                 netdev_err(ndev, "Max %d traffic classes supported\n",
1554                            priv->num_tx_rings);
1555                 return -EINVAL;
1556         }
1557
1558         /* For the moment, we use only one BD ring per TC.
1559          *
1560          * Configure num_tc BD rings with increasing priorities.
1561          */
1562         for (i = 0; i < num_tc; i++) {
1563                 tx_ring = priv->tx_ring[i];
1564                 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i);
1565         }
1566
1567         /* Reset the number of netdev queues based on the TC count */
1568         netif_set_real_num_tx_queues(ndev, num_tc);
1569
1570         netdev_set_num_tc(ndev, num_tc);
1571
1572         /* Each TC is associated with one netdev queue */
1573         for (i = 0; i < num_tc; i++)
1574                 netdev_set_tc_queue(ndev, i, 1, i);
1575
1576         return 0;
1577 }
1578
1579 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type,
1580                    void *type_data)
1581 {
1582         switch (type) {
1583         case TC_SETUP_QDISC_MQPRIO:
1584                 return enetc_setup_tc_mqprio(ndev, type_data);
1585         case TC_SETUP_QDISC_TAPRIO:
1586                 return enetc_setup_tc_taprio(ndev, type_data);
1587         case TC_SETUP_QDISC_CBS:
1588                 return enetc_setup_tc_cbs(ndev, type_data);
1589         case TC_SETUP_QDISC_ETF:
1590                 return enetc_setup_tc_txtime(ndev, type_data);
1591         case TC_SETUP_BLOCK:
1592                 return enetc_setup_tc_psfp(ndev, type_data);
1593         default:
1594                 return -EOPNOTSUPP;
1595         }
1596 }
1597
1598 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
1599 {
1600         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1601         struct net_device_stats *stats = &ndev->stats;
1602         unsigned long packets = 0, bytes = 0;
1603         int i;
1604
1605         for (i = 0; i < priv->num_rx_rings; i++) {
1606                 packets += priv->rx_ring[i]->stats.packets;
1607                 bytes   += priv->rx_ring[i]->stats.bytes;
1608         }
1609
1610         stats->rx_packets = packets;
1611         stats->rx_bytes = bytes;
1612         bytes = 0;
1613         packets = 0;
1614
1615         for (i = 0; i < priv->num_tx_rings; i++) {
1616                 packets += priv->tx_ring[i]->stats.packets;
1617                 bytes   += priv->tx_ring[i]->stats.bytes;
1618         }
1619
1620         stats->tx_packets = packets;
1621         stats->tx_bytes = bytes;
1622
1623         return stats;
1624 }
1625
1626 static int enetc_set_rss(struct net_device *ndev, int en)
1627 {
1628         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1629         struct enetc_hw *hw = &priv->si->hw;
1630         u32 reg;
1631
1632         enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
1633
1634         reg = enetc_rd(hw, ENETC_SIMR);
1635         reg &= ~ENETC_SIMR_RSSE;
1636         reg |= (en) ? ENETC_SIMR_RSSE : 0;
1637         enetc_wr(hw, ENETC_SIMR, reg);
1638
1639         return 0;
1640 }
1641
1642 static int enetc_set_psfp(struct net_device *ndev, int en)
1643 {
1644         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1645         int err;
1646
1647         if (en) {
1648                 err = enetc_psfp_enable(priv);
1649                 if (err)
1650                         return err;
1651
1652                 priv->active_offloads |= ENETC_F_QCI;
1653                 return 0;
1654         }
1655
1656         err = enetc_psfp_disable(priv);
1657         if (err)
1658                 return err;
1659
1660         priv->active_offloads &= ~ENETC_F_QCI;
1661
1662         return 0;
1663 }
1664
1665 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
1666 {
1667         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1668         int i;
1669
1670         for (i = 0; i < priv->num_rx_rings; i++)
1671                 enetc_bdr_enable_rxvlan(&priv->si->hw, i, en);
1672 }
1673
1674 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
1675 {
1676         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1677         int i;
1678
1679         for (i = 0; i < priv->num_tx_rings; i++)
1680                 enetc_bdr_enable_txvlan(&priv->si->hw, i, en);
1681 }
1682
1683 int enetc_set_features(struct net_device *ndev,
1684                        netdev_features_t features)
1685 {
1686         netdev_features_t changed = ndev->features ^ features;
1687         int err = 0;
1688
1689         if (changed & NETIF_F_RXHASH)
1690                 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
1691
1692         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
1693                 enetc_enable_rxvlan(ndev,
1694                                     !!(features & NETIF_F_HW_VLAN_CTAG_RX));
1695
1696         if (changed & NETIF_F_HW_VLAN_CTAG_TX)
1697                 enetc_enable_txvlan(ndev,
1698                                     !!(features & NETIF_F_HW_VLAN_CTAG_TX));
1699
1700         if (changed & NETIF_F_HW_TC)
1701                 err = enetc_set_psfp(ndev, !!(features & NETIF_F_HW_TC));
1702
1703         return err;
1704 }
1705
1706 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1707 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
1708 {
1709         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1710         struct hwtstamp_config config;
1711         int ao;
1712
1713         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
1714                 return -EFAULT;
1715
1716         switch (config.tx_type) {
1717         case HWTSTAMP_TX_OFF:
1718                 priv->active_offloads &= ~ENETC_F_TX_TSTAMP;
1719                 break;
1720         case HWTSTAMP_TX_ON:
1721                 priv->active_offloads |= ENETC_F_TX_TSTAMP;
1722                 break;
1723         default:
1724                 return -ERANGE;
1725         }
1726
1727         ao = priv->active_offloads;
1728         switch (config.rx_filter) {
1729         case HWTSTAMP_FILTER_NONE:
1730                 priv->active_offloads &= ~ENETC_F_RX_TSTAMP;
1731                 break;
1732         default:
1733                 priv->active_offloads |= ENETC_F_RX_TSTAMP;
1734                 config.rx_filter = HWTSTAMP_FILTER_ALL;
1735         }
1736
1737         if (netif_running(ndev) && ao != priv->active_offloads) {
1738                 enetc_close(ndev);
1739                 enetc_open(ndev);
1740         }
1741
1742         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1743                -EFAULT : 0;
1744 }
1745
1746 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
1747 {
1748         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1749         struct hwtstamp_config config;
1750
1751         config.flags = 0;
1752
1753         if (priv->active_offloads & ENETC_F_TX_TSTAMP)
1754                 config.tx_type = HWTSTAMP_TX_ON;
1755         else
1756                 config.tx_type = HWTSTAMP_TX_OFF;
1757
1758         config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
1759                             HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
1760
1761         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1762                -EFAULT : 0;
1763 }
1764 #endif
1765
1766 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
1767 {
1768         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1769 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1770         if (cmd == SIOCSHWTSTAMP)
1771                 return enetc_hwtstamp_set(ndev, rq);
1772         if (cmd == SIOCGHWTSTAMP)
1773                 return enetc_hwtstamp_get(ndev, rq);
1774 #endif
1775
1776         if (!priv->phylink)
1777                 return -EOPNOTSUPP;
1778
1779         return phylink_mii_ioctl(priv->phylink, rq, cmd);
1780 }
1781
1782 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
1783 {
1784         struct pci_dev *pdev = priv->si->pdev;
1785         int v_tx_rings;
1786         int i, n, err, nvec;
1787
1788         nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
1789         /* allocate MSIX for both messaging and Rx/Tx interrupts */
1790         n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
1791
1792         if (n < 0)
1793                 return n;
1794
1795         if (n != nvec)
1796                 return -EPERM;
1797
1798         /* # of tx rings per int vector */
1799         v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
1800
1801         for (i = 0; i < priv->bdr_int_num; i++) {
1802                 struct enetc_int_vector *v;
1803                 struct enetc_bdr *bdr;
1804                 int j;
1805
1806                 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
1807                 if (!v) {
1808                         err = -ENOMEM;
1809                         goto fail;
1810                 }
1811
1812                 priv->int_vector[i] = v;
1813
1814                 /* init defaults for adaptive IC */
1815                 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
1816                         v->rx_ictt = 0x1;
1817                         v->rx_dim_en = true;
1818                 }
1819                 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
1820                 netif_napi_add(priv->ndev, &v->napi, enetc_poll,
1821                                NAPI_POLL_WEIGHT);
1822                 v->count_tx_rings = v_tx_rings;
1823
1824                 for (j = 0; j < v_tx_rings; j++) {
1825                         int idx;
1826
1827                         /* default tx ring mapping policy */
1828                         if (priv->bdr_int_num == ENETC_MAX_BDR_INT)
1829                                 idx = 2 * j + i; /* 2 CPUs */
1830                         else
1831                                 idx = j + i * v_tx_rings; /* default */
1832
1833                         __set_bit(idx, &v->tx_rings_map);
1834                         bdr = &v->tx_ring[j];
1835                         bdr->index = idx;
1836                         bdr->ndev = priv->ndev;
1837                         bdr->dev = priv->dev;
1838                         bdr->bd_count = priv->tx_bd_count;
1839                         priv->tx_ring[idx] = bdr;
1840                 }
1841
1842                 bdr = &v->rx_ring;
1843                 bdr->index = i;
1844                 bdr->ndev = priv->ndev;
1845                 bdr->dev = priv->dev;
1846                 bdr->bd_count = priv->rx_bd_count;
1847                 priv->rx_ring[i] = bdr;
1848         }
1849
1850         return 0;
1851
1852 fail:
1853         while (i--) {
1854                 netif_napi_del(&priv->int_vector[i]->napi);
1855                 cancel_work_sync(&priv->int_vector[i]->rx_dim.work);
1856                 kfree(priv->int_vector[i]);
1857         }
1858
1859         pci_free_irq_vectors(pdev);
1860
1861         return err;
1862 }
1863
1864 void enetc_free_msix(struct enetc_ndev_priv *priv)
1865 {
1866         int i;
1867
1868         for (i = 0; i < priv->bdr_int_num; i++) {
1869                 struct enetc_int_vector *v = priv->int_vector[i];
1870
1871                 netif_napi_del(&v->napi);
1872                 cancel_work_sync(&v->rx_dim.work);
1873         }
1874
1875         for (i = 0; i < priv->num_rx_rings; i++)
1876                 priv->rx_ring[i] = NULL;
1877
1878         for (i = 0; i < priv->num_tx_rings; i++)
1879                 priv->tx_ring[i] = NULL;
1880
1881         for (i = 0; i < priv->bdr_int_num; i++) {
1882                 kfree(priv->int_vector[i]);
1883                 priv->int_vector[i] = NULL;
1884         }
1885
1886         /* disable all MSIX for this device */
1887         pci_free_irq_vectors(priv->si->pdev);
1888 }
1889
1890 static void enetc_kfree_si(struct enetc_si *si)
1891 {
1892         char *p = (char *)si - si->pad;
1893
1894         kfree(p);
1895 }
1896
1897 static void enetc_detect_errata(struct enetc_si *si)
1898 {
1899         if (si->pdev->revision == ENETC_REV1)
1900                 si->errata = ENETC_ERR_TXCSUM | ENETC_ERR_VLAN_ISOL |
1901                              ENETC_ERR_UCMCSWP;
1902 }
1903
1904 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
1905 {
1906         struct enetc_si *si, *p;
1907         struct enetc_hw *hw;
1908         size_t alloc_size;
1909         int err, len;
1910
1911         pcie_flr(pdev);
1912         err = pci_enable_device_mem(pdev);
1913         if (err) {
1914                 dev_err(&pdev->dev, "device enable failed\n");
1915                 return err;
1916         }
1917
1918         /* set up for high or low dma */
1919         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1920         if (err) {
1921                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1922                 if (err) {
1923                         dev_err(&pdev->dev,
1924                                 "DMA configuration failed: 0x%x\n", err);
1925                         goto err_dma;
1926                 }
1927         }
1928
1929         err = pci_request_mem_regions(pdev, name);
1930         if (err) {
1931                 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
1932                 goto err_pci_mem_reg;
1933         }
1934
1935         pci_set_master(pdev);
1936
1937         alloc_size = sizeof(struct enetc_si);
1938         if (sizeof_priv) {
1939                 /* align priv to 32B */
1940                 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
1941                 alloc_size += sizeof_priv;
1942         }
1943         /* force 32B alignment for enetc_si */
1944         alloc_size += ENETC_SI_ALIGN - 1;
1945
1946         p = kzalloc(alloc_size, GFP_KERNEL);
1947         if (!p) {
1948                 err = -ENOMEM;
1949                 goto err_alloc_si;
1950         }
1951
1952         si = PTR_ALIGN(p, ENETC_SI_ALIGN);
1953         si->pad = (char *)si - (char *)p;
1954
1955         pci_set_drvdata(pdev, si);
1956         si->pdev = pdev;
1957         hw = &si->hw;
1958
1959         len = pci_resource_len(pdev, ENETC_BAR_REGS);
1960         hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
1961         if (!hw->reg) {
1962                 err = -ENXIO;
1963                 dev_err(&pdev->dev, "ioremap() failed\n");
1964                 goto err_ioremap;
1965         }
1966         if (len > ENETC_PORT_BASE)
1967                 hw->port = hw->reg + ENETC_PORT_BASE;
1968         if (len > ENETC_GLOBAL_BASE)
1969                 hw->global = hw->reg + ENETC_GLOBAL_BASE;
1970
1971         enetc_detect_errata(si);
1972
1973         return 0;
1974
1975 err_ioremap:
1976         enetc_kfree_si(si);
1977 err_alloc_si:
1978         pci_release_mem_regions(pdev);
1979 err_pci_mem_reg:
1980 err_dma:
1981         pci_disable_device(pdev);
1982
1983         return err;
1984 }
1985
1986 void enetc_pci_remove(struct pci_dev *pdev)
1987 {
1988         struct enetc_si *si = pci_get_drvdata(pdev);
1989         struct enetc_hw *hw = &si->hw;
1990
1991         iounmap(hw->reg);
1992         enetc_kfree_si(si);
1993         pci_release_mem_regions(pdev);
1994         pci_disable_device(pdev);
1995 }