drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_rx.c

   1 // SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause
   2
   3 /* Packet receive logic for Mellanox Gigabit Ethernet driver
   4  *
   5  * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES
   6  */
   7
   8 #include <linux/etherdevice.h>
   9 #include <linux/skbuff.h>
  10
  11 #include "mlxbf_gige.h"
  12 #include "mlxbf_gige_regs.h"
  13
  14 void mlxbf_gige_set_mac_rx_filter(struct mlxbf_gige *priv,
  15                                   unsigned int index, u64 dmac)
  16 {
  17         void __iomem *base = priv->base;
  18         u64 control;
  19
  20         /* Write destination MAC to specified MAC RX filter */
  21         writeq(dmac, base + MLXBF_GIGE_RX_MAC_FILTER +
  22                (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));
  23
  24         /* Enable MAC receive filter mask for specified index */
  25         control = readq(base + MLXBF_GIGE_CONTROL);
  26         control |= (MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC << index);
  27         writeq(control, base + MLXBF_GIGE_CONTROL);
  28 }
  29
  30 void mlxbf_gige_get_mac_rx_filter(struct mlxbf_gige *priv,
  31                                   unsigned int index, u64 *dmac)
  32 {
  33         void __iomem *base = priv->base;
  34
  35         /* Read destination MAC from specified MAC RX filter */
  36         *dmac = readq(base + MLXBF_GIGE_RX_MAC_FILTER +
  37                       (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));
  38 }
  39
  40 void mlxbf_gige_enable_promisc(struct mlxbf_gige *priv)
  41 {
  42         void __iomem *base = priv->base;
  43         u64 control;
  44         u64 end_mac;
  45
  46         /* Enable MAC_ID_RANGE match functionality */
  47         control = readq(base + MLXBF_GIGE_CONTROL);
  48         control |= MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;
  49         writeq(control, base + MLXBF_GIGE_CONTROL);
  50
  51         /* Set start of destination MAC range check to 0 */
  52         writeq(0, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_START);
  53
  54         /* Set end of destination MAC range check to all FFs */
  55         end_mac = BCAST_MAC_ADDR;
  56         writeq(end_mac, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_END);
  57 }
  58
  59 void mlxbf_gige_disable_promisc(struct mlxbf_gige *priv)
  60 {
  61         void __iomem *base = priv->base;
  62         u64 control;
  63
  64         /* Disable MAC_ID_RANGE match functionality */
  65         control = readq(base + MLXBF_GIGE_CONTROL);
  66         control &= ~MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;
  67         writeq(control, base + MLXBF_GIGE_CONTROL);
  68
  69         /* NOTE: no need to change DMAC_RANGE_START or END;
  70          * those values are ignored since MAC_ID_RANGE_EN=0
  71          */
  72 }
  73
  74 /* Receive Initialization
  75  * 1) Configures RX MAC filters via MMIO registers
  76  * 2) Allocates RX WQE array using coherent DMA mapping
  77  * 3) Initializes each element of RX WQE array with a receive
  78  *    buffer pointer (also using coherent DMA mapping)
  79  * 4) Allocates RX CQE array using coherent DMA mapping
  80  * 5) Completes other misc receive initialization
  81  */
  82 int mlxbf_gige_rx_init(struct mlxbf_gige *priv)
  83 {
  84         size_t wq_size, cq_size;
  85         dma_addr_t *rx_wqe_ptr;
  86         dma_addr_t rx_buf_dma;
  87         u64 data;
  88         int i, j;
  89
  90         /* Configure MAC RX filter #0 to allow RX of broadcast pkts */
  91         mlxbf_gige_set_mac_rx_filter(priv, MLXBF_GIGE_BCAST_MAC_FILTER_IDX,
  92                                      BCAST_MAC_ADDR);
  93
  94         wq_size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;
  95         priv->rx_wqe_base = dma_alloc_coherent(priv->dev, wq_size,
  96                                                &priv->rx_wqe_base_dma,
  97                                                GFP_KERNEL);
  98         if (!priv->rx_wqe_base)
  99                 return -ENOMEM;
 100
 101         /* Initialize 'rx_wqe_ptr' to point to first RX WQE in array
 102          * Each RX WQE is simply a receive buffer pointer, so walk
 103          * the entire array, allocating a 2KB buffer for each element
 104          */
 105         rx_wqe_ptr = priv->rx_wqe_base;
 106
 107         for (i = 0; i < priv->rx_q_entries; i++) {
 108                 priv->rx_skb[i] = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,
 109                                                        &rx_buf_dma, DMA_FROM_DEVICE);
 110                 if (!priv->rx_skb[i])
 111                         goto free_wqe_and_skb;
 112                 *rx_wqe_ptr++ = rx_buf_dma;
 113         }
 114
 115         /* Write RX WQE base address into MMIO reg */
 116         writeq(priv->rx_wqe_base_dma, priv->base + MLXBF_GIGE_RX_WQ_BASE);
 117
 118         cq_size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;
 119         priv->rx_cqe_base = dma_alloc_coherent(priv->dev, cq_size,
 120                                                &priv->rx_cqe_base_dma,
 121                                                GFP_KERNEL);
 122         if (!priv->rx_cqe_base)
 123                 goto free_wqe_and_skb;
 124
 125         for (i = 0; i < priv->rx_q_entries; i++)
 126                 priv->rx_cqe_base[i] |= MLXBF_GIGE_RX_CQE_VALID_MASK;
 127
 128         /* Write RX CQE base address into MMIO reg */
 129         writeq(priv->rx_cqe_base_dma, priv->base + MLXBF_GIGE_RX_CQ_BASE);
 130
 131         /* Write RX_WQE_PI with current number of replenished buffers */
 132         writeq(priv->rx_q_entries, priv->base + MLXBF_GIGE_RX_WQE_PI);
 133
 134         /* Enable removal of CRC during RX */
 135         data = readq(priv->base + MLXBF_GIGE_RX);
 136         data |= MLXBF_GIGE_RX_STRIP_CRC_EN;
 137         writeq(data, priv->base + MLXBF_GIGE_RX);
 138
 139         /* Enable RX MAC filter pass and discard counters */
 140         writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC_EN,
 141                priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC);
 142         writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS_EN,
 143                priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS);
 144
 145         /* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to
 146          * indicate readiness to receive interrupts
 147          */
 148         data = readq(priv->base + MLXBF_GIGE_INT_MASK);
 149         data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;
 150         writeq(data, priv->base + MLXBF_GIGE_INT_MASK);
 151
 152         /* Enable RX DMA to write new packets to memory */
 153         data = readq(priv->base + MLXBF_GIGE_RX_DMA);
 154         data |= MLXBF_GIGE_RX_DMA_EN;
 155         writeq(data, priv->base + MLXBF_GIGE_RX_DMA);
 156
 157         writeq(ilog2(priv->rx_q_entries),
 158                priv->base + MLXBF_GIGE_RX_WQE_SIZE_LOG2);
 159
 160         return 0;
 161
 162 free_wqe_and_skb:
 163         rx_wqe_ptr = priv->rx_wqe_base;
 164         for (j = 0; j < i; j++) {
 165                 dma_unmap_single(priv->dev, *rx_wqe_ptr,
 166                                  MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);
 167                 dev_kfree_skb(priv->rx_skb[j]);
 168                 rx_wqe_ptr++;
 169         }
 170         dma_free_coherent(priv->dev, wq_size,
 171                           priv->rx_wqe_base, priv->rx_wqe_base_dma);
 172         return -ENOMEM;
 173 }
 174
 175 /* Receive Deinitialization
 176  * This routine will free allocations done by mlxbf_gige_rx_init(),
 177  * namely the RX WQE and RX CQE arrays, as well as all RX buffers
 178  */
 179 void mlxbf_gige_rx_deinit(struct mlxbf_gige *priv)
 180 {
 181         dma_addr_t *rx_wqe_ptr;
 182         size_t size;
 183         u64 data;
 184         int i;
 185
 186         /* Disable RX DMA to prevent packet transfers to memory */
 187         data = readq(priv->base + MLXBF_GIGE_RX_DMA);
 188         data &= ~MLXBF_GIGE_RX_DMA_EN;
 189         writeq(data, priv->base + MLXBF_GIGE_RX_DMA);
 190
 191         rx_wqe_ptr = priv->rx_wqe_base;
 192
 193         for (i = 0; i < priv->rx_q_entries; i++) {
 194                 dma_unmap_single(priv->dev, *rx_wqe_ptr, MLXBF_GIGE_DEFAULT_BUF_SZ,
 195                                  DMA_FROM_DEVICE);
 196                 dev_kfree_skb(priv->rx_skb[i]);
 197                 rx_wqe_ptr++;
 198         }
 199
 200         size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;
 201         dma_free_coherent(priv->dev, size,
 202                           priv->rx_wqe_base, priv->rx_wqe_base_dma);
 203
 204         size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;
 205         dma_free_coherent(priv->dev, size,
 206                           priv->rx_cqe_base, priv->rx_cqe_base_dma);
 207
 208         priv->rx_wqe_base = NULL;
 209         priv->rx_wqe_base_dma = 0;
 210         priv->rx_cqe_base = NULL;
 211         priv->rx_cqe_base_dma = 0;
 212         writeq(0, priv->base + MLXBF_GIGE_RX_WQ_BASE);
 213         writeq(0, priv->base + MLXBF_GIGE_RX_CQ_BASE);
 214 }
 215
 216 static bool mlxbf_gige_rx_packet(struct mlxbf_gige *priv, int *rx_pkts)
 217 {
 218         struct net_device *netdev = priv->netdev;
 219         struct sk_buff *skb = NULL, *rx_skb;
 220         u16 rx_pi_rem, rx_ci_rem;
 221         dma_addr_t *rx_wqe_addr;
 222         dma_addr_t rx_buf_dma;
 223         u64 *rx_cqe_addr;
 224         u64 datalen;
 225         u64 rx_cqe;
 226         u16 rx_ci;
 227         u16 rx_pi;
 228
 229         /* Index into RX buffer array is rx_pi w/wrap based on RX_CQE_SIZE */
 230         rx_pi = readq(priv->base + MLXBF_GIGE_RX_WQE_PI);
 231         rx_pi_rem = rx_pi % priv->rx_q_entries;
 232
 233         rx_wqe_addr = priv->rx_wqe_base + rx_pi_rem;
 234         rx_cqe_addr = priv->rx_cqe_base + rx_pi_rem;
 235         rx_cqe = *rx_cqe_addr;
 236
 237         if ((!!(rx_cqe & MLXBF_GIGE_RX_CQE_VALID_MASK)) != priv->valid_polarity)
 238                 return false;
 239
 240         if ((rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MASK) == 0) {
 241                 /* Packet is OK, increment stats */
 242                 datalen = rx_cqe & MLXBF_GIGE_RX_CQE_PKT_LEN_MASK;
 243                 netdev->stats.rx_packets++;
 244                 netdev->stats.rx_bytes += datalen;
 245
 246                 skb = priv->rx_skb[rx_pi_rem];
 247
 248                 skb_put(skb, datalen);
 249
 250                 skb->ip_summed = CHECKSUM_NONE; /* device did not checksum packet */
 251
 252                 skb->protocol = eth_type_trans(skb, netdev);
 253
 254                 /* Alloc another RX SKB for this same index */
 255                 rx_skb = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,
 256                                               &rx_buf_dma, DMA_FROM_DEVICE);
 257                 if (!rx_skb)
 258                         return false;
 259                 priv->rx_skb[rx_pi_rem] = rx_skb;
 260                 dma_unmap_single(priv->dev, *rx_wqe_addr,
 261                                  MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);
 262                 *rx_wqe_addr = rx_buf_dma;
 263         } else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MAC_ERR) {
 264                 priv->stats.rx_mac_errors++;
 265         } else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_TRUNCATED) {
 266                 priv->stats.rx_truncate_errors++;
 267         }
 268
 269         /* Let hardware know we've replenished one buffer */
 270         rx_pi++;
 271
 272         /* Ensure completion of all writes before notifying HW of replenish */
 273         wmb();
 274         writeq(rx_pi, priv->base + MLXBF_GIGE_RX_WQE_PI);
 275
 276         (*rx_pkts)++;
 277
 278         rx_pi_rem = rx_pi % priv->rx_q_entries;
 279         if (rx_pi_rem == 0)
 280                 priv->valid_polarity ^= 1;
 281         rx_ci = readq(priv->base + MLXBF_GIGE_RX_CQE_PACKET_CI);
 282         rx_ci_rem = rx_ci % priv->rx_q_entries;
 283
 284         if (skb)
 285                 netif_receive_skb(skb);
 286
 287         return rx_pi_rem != rx_ci_rem;
 288 }
 289
 290 /* Driver poll() function called by NAPI infrastructure */
 291 int mlxbf_gige_poll(struct napi_struct *napi, int budget)
 292 {
 293         struct mlxbf_gige *priv;
 294         bool remaining_pkts;
 295         int work_done = 0;
 296         u64 data;
 297
 298         priv = container_of(napi, struct mlxbf_gige, napi);
 299
 300         mlxbf_gige_handle_tx_complete(priv);
 301
 302         do {
 303                 remaining_pkts = mlxbf_gige_rx_packet(priv, &work_done);
 304         } while (remaining_pkts && work_done < budget);
 305
 306         /* If amount of work done < budget, turn off NAPI polling
 307          * via napi_complete_done(napi, work_done) and then
 308          * re-enable interrupts.
 309          */
 310         if (work_done < budget && napi_complete_done(napi, work_done)) {
 311                 /* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to
 312                  * indicate receive readiness
 313                  */
 314                 data = readq(priv->base + MLXBF_GIGE_INT_MASK);
 315                 data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;
 316                 writeq(data, priv->base + MLXBF_GIGE_INT_MASK);
 317         }
 318
 319         return work_done;
 320 }