drivers/net/ethernet/hisilicon/hns3/hns3_enet.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 // Copyright (c) 2016-2017 Hisilicon Limited.
   3
   4 #include <linux/dma-mapping.h>
   5 #include <linux/etherdevice.h>
   6 #include <linux/interrupt.h>
   7 #ifdef CONFIG_RFS_ACCEL
   8 #include <linux/cpu_rmap.h>
   9 #endif
  10 #include <linux/if_vlan.h>
  11 #include <linux/irq.h>
  12 #include <linux/ip.h>
  13 #include <linux/ipv6.h>
  14 #include <linux/module.h>
  15 #include <linux/pci.h>
  16 #include <linux/aer.h>
  17 #include <linux/skbuff.h>
  18 #include <linux/sctp.h>
  19 #include <net/gre.h>
  20 #include <net/ip6_checksum.h>
  21 #include <net/pkt_cls.h>
  22 #include <net/tcp.h>
  23 #include <net/vxlan.h>
  24 #include <net/geneve.h>
  25
  26 #include "hnae3.h"
  27 #include "hns3_enet.h"
  28 /* All hns3 tracepoints are defined by the include below, which
  29  * must be included exactly once across the whole kernel with
  30  * CREATE_TRACE_POINTS defined
  31  */
  32 #define CREATE_TRACE_POINTS
  33 #include "hns3_trace.h"
  34
  35 #define hns3_set_field(origin, shift, val)      ((origin) |= (val) << (shift))
  36 #define hns3_tx_bd_count(S)     DIV_ROUND_UP(S, HNS3_MAX_BD_SIZE)
  37
  38 #define hns3_rl_err(fmt, ...)                                           \
  39         do {                                                            \
  40                 if (net_ratelimit())                                    \
  41                         netdev_err(fmt, ##__VA_ARGS__);                 \
  42         } while (0)
  43
  44 static void hns3_clear_all_ring(struct hnae3_handle *h, bool force);
  45
  46 static const char hns3_driver_name[] = "hns3";
  47 static const char hns3_driver_string[] =
  48                         "Hisilicon Ethernet Network Driver for Hip08 Family";
  49 static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
  50 static struct hnae3_client client;
  51
  52 static int debug = -1;
  53 module_param(debug, int, 0);
  54 MODULE_PARM_DESC(debug, " Network interface message level setting");
  55
  56 #define DEFAULT_MSG_LEVEL (NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  57                            NETIF_MSG_IFDOWN | NETIF_MSG_IFUP)
  58
  59 #define HNS3_INNER_VLAN_TAG     1
  60 #define HNS3_OUTER_VLAN_TAG     2
  61
  62 #define HNS3_MIN_TX_LEN         33U
  63
  64 /* hns3_pci_tbl - PCI Device ID Table
  65  *
  66  * Last entry must be all 0s
  67  *
  68  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  69  *   Class, Class Mask, private data (not used) }
  70  */
  71 static const struct pci_device_id hns3_pci_tbl[] = {
  72         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
  73         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
  74         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
  75          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  76         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
  77          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  78         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
  79          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  80         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
  81          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  82         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
  83          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  84         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA),
  85          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  86         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_VF), 0},
  87         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
  88          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  89         /* required last entry */
  90         {0, }
  91 };
  92 MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
  93
  94 static irqreturn_t hns3_irq_handle(int irq, void *vector)
  95 {
  96         struct hns3_enet_tqp_vector *tqp_vector = vector;
  97
  98         napi_schedule_irqoff(&tqp_vector->napi);
  99
 100         return IRQ_HANDLED;
 101 }
 102
 103 static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
 104 {
 105         struct hns3_enet_tqp_vector *tqp_vectors;
 106         unsigned int i;
 107
 108         for (i = 0; i < priv->vector_num; i++) {
 109                 tqp_vectors = &priv->tqp_vector[i];
 110
 111                 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
 112                         continue;
 113
 114                 /* clear the affinity mask */
 115                 irq_set_affinity_hint(tqp_vectors->vector_irq, NULL);
 116
 117                 /* release the irq resource */
 118                 free_irq(tqp_vectors->vector_irq, tqp_vectors);
 119                 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
 120         }
 121 }
 122
 123 static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
 124 {
 125         struct hns3_enet_tqp_vector *tqp_vectors;
 126         int txrx_int_idx = 0;
 127         int rx_int_idx = 0;
 128         int tx_int_idx = 0;
 129         unsigned int i;
 130         int ret;
 131
 132         for (i = 0; i < priv->vector_num; i++) {
 133                 tqp_vectors = &priv->tqp_vector[i];
 134
 135                 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
 136                         continue;
 137
 138                 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
 139                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
 140                                  "%s-%s-%s-%d", hns3_driver_name,
 141                                  pci_name(priv->ae_handle->pdev),
 142                                  "TxRx", txrx_int_idx++);
 143                         txrx_int_idx++;
 144                 } else if (tqp_vectors->rx_group.ring) {
 145                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
 146                                  "%s-%s-%s-%d", hns3_driver_name,
 147                                  pci_name(priv->ae_handle->pdev),
 148                                  "Rx", rx_int_idx++);
 149                 } else if (tqp_vectors->tx_group.ring) {
 150                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
 151                                  "%s-%s-%s-%d", hns3_driver_name,
 152                                  pci_name(priv->ae_handle->pdev),
 153                                  "Tx", tx_int_idx++);
 154                 } else {
 155                         /* Skip this unused q_vector */
 156                         continue;
 157                 }
 158
 159                 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
 160
 161                 irq_set_status_flags(tqp_vectors->vector_irq, IRQ_NOAUTOEN);
 162                 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
 163                                   tqp_vectors->name, tqp_vectors);
 164                 if (ret) {
 165                         netdev_err(priv->netdev, "request irq(%d) fail\n",
 166                                    tqp_vectors->vector_irq);
 167                         hns3_nic_uninit_irq(priv);
 168                         return ret;
 169                 }
 170
 171                 irq_set_affinity_hint(tqp_vectors->vector_irq,
 172                                       &tqp_vectors->affinity_mask);
 173
 174                 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
 175         }
 176
 177         return 0;
 178 }
 179
 180 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
 181                                  u32 mask_en)
 182 {
 183         writel(mask_en, tqp_vector->mask_addr);
 184 }
 185
 186 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
 187 {
 188         napi_enable(&tqp_vector->napi);
 189         enable_irq(tqp_vector->vector_irq);
 190
 191         /* enable vector */
 192         hns3_mask_vector_irq(tqp_vector, 1);
 193 }
 194
 195 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
 196 {
 197         /* disable vector */
 198         hns3_mask_vector_irq(tqp_vector, 0);
 199
 200         disable_irq(tqp_vector->vector_irq);
 201         napi_disable(&tqp_vector->napi);
 202 }
 203
 204 void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
 205                                  u32 rl_value)
 206 {
 207         u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
 208
 209         /* this defines the configuration for RL (Interrupt Rate Limiter).
 210          * Rl defines rate of interrupts i.e. number of interrupts-per-second
 211          * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
 212          */
 213         if (rl_reg > 0 && !tqp_vector->tx_group.coal.adapt_enable &&
 214             !tqp_vector->rx_group.coal.adapt_enable)
 215                 /* According to the hardware, the range of rl_reg is
 216                  * 0-59 and the unit is 4.
 217                  */
 218                 rl_reg |=  HNS3_INT_RL_ENABLE_MASK;
 219
 220         writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
 221 }
 222
 223 void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
 224                                     u32 gl_value)
 225 {
 226         u32 new_val;
 227
 228         if (tqp_vector->rx_group.coal.unit_1us)
 229                 new_val = gl_value | HNS3_INT_GL_1US;
 230         else
 231                 new_val = hns3_gl_usec_to_reg(gl_value);
 232
 233         writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
 234 }
 235
 236 void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
 237                                     u32 gl_value)
 238 {
 239         u32 new_val;
 240
 241         if (tqp_vector->tx_group.coal.unit_1us)
 242                 new_val = gl_value | HNS3_INT_GL_1US;
 243         else
 244                 new_val = hns3_gl_usec_to_reg(gl_value);
 245
 246         writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
 247 }
 248
 249 void hns3_set_vector_coalesce_tx_ql(struct hns3_enet_tqp_vector *tqp_vector,
 250                                     u32 ql_value)
 251 {
 252         writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_TX_QL_OFFSET);
 253 }
 254
 255 void hns3_set_vector_coalesce_rx_ql(struct hns3_enet_tqp_vector *tqp_vector,
 256                                     u32 ql_value)
 257 {
 258         writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_RX_QL_OFFSET);
 259 }
 260
 261 static void hns3_vector_coalesce_init(struct hns3_enet_tqp_vector *tqp_vector,
 262                                       struct hns3_nic_priv *priv)
 263 {
 264         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);
 265         struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;
 266         struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;
 267         struct hns3_enet_coalesce *ptx_coal = &priv->tx_coal;
 268         struct hns3_enet_coalesce *prx_coal = &priv->rx_coal;
 269
 270         tx_coal->adapt_enable = ptx_coal->adapt_enable;
 271         rx_coal->adapt_enable = prx_coal->adapt_enable;
 272
 273         tx_coal->int_gl = ptx_coal->int_gl;
 274         rx_coal->int_gl = prx_coal->int_gl;
 275
 276         rx_coal->flow_level = prx_coal->flow_level;
 277         tx_coal->flow_level = ptx_coal->flow_level;
 278
 279         /* device version above V3(include V3), GL can configure 1us
 280          * unit, so uses 1us unit.
 281          */
 282         if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3) {
 283                 tx_coal->unit_1us = 1;
 284                 rx_coal->unit_1us = 1;
 285         }
 286
 287         if (ae_dev->dev_specs.int_ql_max) {
 288                 tx_coal->ql_enable = 1;
 289                 rx_coal->ql_enable = 1;
 290                 tx_coal->int_ql_max = ae_dev->dev_specs.int_ql_max;
 291                 rx_coal->int_ql_max = ae_dev->dev_specs.int_ql_max;
 292                 tx_coal->int_ql = ptx_coal->int_ql;
 293                 rx_coal->int_ql = prx_coal->int_ql;
 294         }
 295 }
 296
 297 static void
 298 hns3_vector_coalesce_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
 299                              struct hns3_nic_priv *priv)
 300 {
 301         struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;
 302         struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;
 303         struct hnae3_handle *h = priv->ae_handle;
 304
 305         hns3_set_vector_coalesce_tx_gl(tqp_vector, tx_coal->int_gl);
 306         hns3_set_vector_coalesce_rx_gl(tqp_vector, rx_coal->int_gl);
 307         hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
 308
 309         if (tx_coal->ql_enable)
 310                 hns3_set_vector_coalesce_tx_ql(tqp_vector, tx_coal->int_ql);
 311
 312         if (rx_coal->ql_enable)
 313                 hns3_set_vector_coalesce_rx_ql(tqp_vector, rx_coal->int_ql);
 314 }
 315
 316 static int hns3_nic_set_real_num_queue(struct net_device *netdev)
 317 {
 318         struct hnae3_handle *h = hns3_get_handle(netdev);
 319         struct hnae3_knic_private_info *kinfo = &h->kinfo;
 320         struct hnae3_tc_info *tc_info = &kinfo->tc_info;
 321         unsigned int queue_size = kinfo->num_tqps;
 322         int i, ret;
 323
 324         if (tc_info->num_tc <= 1 && !tc_info->mqprio_active) {
 325                 netdev_reset_tc(netdev);
 326         } else {
 327                 ret = netdev_set_num_tc(netdev, tc_info->num_tc);
 328                 if (ret) {
 329                         netdev_err(netdev,
 330                                    "netdev_set_num_tc fail, ret=%d!\n", ret);
 331                         return ret;
 332                 }
 333
 334                 for (i = 0; i < HNAE3_MAX_TC; i++) {
 335                         if (!test_bit(i, &tc_info->tc_en))
 336                                 continue;
 337
 338                         netdev_set_tc_queue(netdev, i, tc_info->tqp_count[i],
 339                                             tc_info->tqp_offset[i]);
 340                 }
 341         }
 342
 343         ret = netif_set_real_num_tx_queues(netdev, queue_size);
 344         if (ret) {
 345                 netdev_err(netdev,
 346                            "netif_set_real_num_tx_queues fail, ret=%d!\n", ret);
 347                 return ret;
 348         }
 349
 350         ret = netif_set_real_num_rx_queues(netdev, queue_size);
 351         if (ret) {
 352                 netdev_err(netdev,
 353                            "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
 354                 return ret;
 355         }
 356
 357         return 0;
 358 }
 359
 360 static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
 361 {
 362         u16 alloc_tqps, max_rss_size, rss_size;
 363
 364         h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size);
 365         rss_size = alloc_tqps / h->kinfo.tc_info.num_tc;
 366
 367         return min_t(u16, rss_size, max_rss_size);
 368 }
 369
 370 static void hns3_tqp_enable(struct hnae3_queue *tqp)
 371 {
 372         u32 rcb_reg;
 373
 374         rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
 375         rcb_reg |= BIT(HNS3_RING_EN_B);
 376         hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
 377 }
 378
 379 static void hns3_tqp_disable(struct hnae3_queue *tqp)
 380 {
 381         u32 rcb_reg;
 382
 383         rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
 384         rcb_reg &= ~BIT(HNS3_RING_EN_B);
 385         hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
 386 }
 387
 388 static void hns3_free_rx_cpu_rmap(struct net_device *netdev)
 389 {
 390 #ifdef CONFIG_RFS_ACCEL
 391         free_irq_cpu_rmap(netdev->rx_cpu_rmap);
 392         netdev->rx_cpu_rmap = NULL;
 393 #endif
 394 }
 395
 396 static int hns3_set_rx_cpu_rmap(struct net_device *netdev)
 397 {
 398 #ifdef CONFIG_RFS_ACCEL
 399         struct hns3_nic_priv *priv = netdev_priv(netdev);
 400         struct hns3_enet_tqp_vector *tqp_vector;
 401         int i, ret;
 402
 403         if (!netdev->rx_cpu_rmap) {
 404                 netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(priv->vector_num);
 405                 if (!netdev->rx_cpu_rmap)
 406                         return -ENOMEM;
 407         }
 408
 409         for (i = 0; i < priv->vector_num; i++) {
 410                 tqp_vector = &priv->tqp_vector[i];
 411                 ret = irq_cpu_rmap_add(netdev->rx_cpu_rmap,
 412                                        tqp_vector->vector_irq);
 413                 if (ret) {
 414                         hns3_free_rx_cpu_rmap(netdev);
 415                         return ret;
 416                 }
 417         }
 418 #endif
 419         return 0;
 420 }
 421
 422 static int hns3_nic_net_up(struct net_device *netdev)
 423 {
 424         struct hns3_nic_priv *priv = netdev_priv(netdev);
 425         struct hnae3_handle *h = priv->ae_handle;
 426         int i, j;
 427         int ret;
 428
 429         ret = hns3_nic_reset_all_ring(h);
 430         if (ret)
 431                 return ret;
 432
 433         clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
 434
 435         /* enable the vectors */
 436         for (i = 0; i < priv->vector_num; i++)
 437                 hns3_vector_enable(&priv->tqp_vector[i]);
 438
 439         /* enable rcb */
 440         for (j = 0; j < h->kinfo.num_tqps; j++)
 441                 hns3_tqp_enable(h->kinfo.tqp[j]);
 442
 443         /* start the ae_dev */
 444         ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
 445         if (ret) {
 446                 set_bit(HNS3_NIC_STATE_DOWN, &priv->state);
 447                 while (j--)
 448                         hns3_tqp_disable(h->kinfo.tqp[j]);
 449
 450                 for (j = i - 1; j >= 0; j--)
 451                         hns3_vector_disable(&priv->tqp_vector[j]);
 452         }
 453
 454         return ret;
 455 }
 456
 457 static void hns3_config_xps(struct hns3_nic_priv *priv)
 458 {
 459         int i;
 460
 461         for (i = 0; i < priv->vector_num; i++) {
 462                 struct hns3_enet_tqp_vector *tqp_vector = &priv->tqp_vector[i];
 463                 struct hns3_enet_ring *ring = tqp_vector->tx_group.ring;
 464
 465                 while (ring) {
 466                         int ret;
 467
 468                         ret = netif_set_xps_queue(priv->netdev,
 469                                                   &tqp_vector->affinity_mask,
 470                                                   ring->tqp->tqp_index);
 471                         if (ret)
 472                                 netdev_warn(priv->netdev,
 473                                             "set xps queue failed: %d", ret);
 474
 475                         ring = ring->next;
 476                 }
 477         }
 478 }
 479
 480 static int hns3_nic_net_open(struct net_device *netdev)
 481 {
 482         struct hns3_nic_priv *priv = netdev_priv(netdev);
 483         struct hnae3_handle *h = hns3_get_handle(netdev);
 484         struct hnae3_knic_private_info *kinfo;
 485         int i, ret;
 486
 487         if (hns3_nic_resetting(netdev))
 488                 return -EBUSY;
 489
 490         netif_carrier_off(netdev);
 491
 492         ret = hns3_nic_set_real_num_queue(netdev);
 493         if (ret)
 494                 return ret;
 495
 496         ret = hns3_nic_net_up(netdev);
 497         if (ret) {
 498                 netdev_err(netdev, "net up fail, ret=%d!\n", ret);
 499                 return ret;
 500         }
 501
 502         kinfo = &h->kinfo;
 503         for (i = 0; i < HNAE3_MAX_USER_PRIO; i++)
 504                 netdev_set_prio_tc_map(netdev, i, kinfo->tc_info.prio_tc[i]);
 505
 506         if (h->ae_algo->ops->set_timer_task)
 507                 h->ae_algo->ops->set_timer_task(priv->ae_handle, true);
 508
 509         hns3_config_xps(priv);
 510
 511         netif_dbg(h, drv, netdev, "net open\n");
 512
 513         return 0;
 514 }
 515
 516 static void hns3_reset_tx_queue(struct hnae3_handle *h)
 517 {
 518         struct net_device *ndev = h->kinfo.netdev;
 519         struct hns3_nic_priv *priv = netdev_priv(ndev);
 520         struct netdev_queue *dev_queue;
 521         u32 i;
 522
 523         for (i = 0; i < h->kinfo.num_tqps; i++) {
 524                 dev_queue = netdev_get_tx_queue(ndev,
 525                                                 priv->ring[i].queue_index);
 526                 netdev_tx_reset_queue(dev_queue);
 527         }
 528 }
 529
 530 static void hns3_nic_net_down(struct net_device *netdev)
 531 {
 532         struct hns3_nic_priv *priv = netdev_priv(netdev);
 533         struct hnae3_handle *h = hns3_get_handle(netdev);
 534         const struct hnae3_ae_ops *ops;
 535         int i;
 536
 537         /* disable vectors */
 538         for (i = 0; i < priv->vector_num; i++)
 539                 hns3_vector_disable(&priv->tqp_vector[i]);
 540
 541         /* disable rcb */
 542         for (i = 0; i < h->kinfo.num_tqps; i++)
 543                 hns3_tqp_disable(h->kinfo.tqp[i]);
 544
 545         /* stop ae_dev */
 546         ops = priv->ae_handle->ae_algo->ops;
 547         if (ops->stop)
 548                 ops->stop(priv->ae_handle);
 549
 550         /* delay ring buffer clearing to hns3_reset_notify_uninit_enet
 551          * during reset process, because driver may not be able
 552          * to disable the ring through firmware when downing the netdev.
 553          */
 554         if (!hns3_nic_resetting(netdev))
 555                 hns3_clear_all_ring(priv->ae_handle, false);
 556
 557         hns3_reset_tx_queue(priv->ae_handle);
 558 }
 559
 560 static int hns3_nic_net_stop(struct net_device *netdev)
 561 {
 562         struct hns3_nic_priv *priv = netdev_priv(netdev);
 563         struct hnae3_handle *h = hns3_get_handle(netdev);
 564
 565         if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
 566                 return 0;
 567
 568         netif_dbg(h, drv, netdev, "net stop\n");
 569
 570         if (h->ae_algo->ops->set_timer_task)
 571                 h->ae_algo->ops->set_timer_task(priv->ae_handle, false);
 572
 573         netif_carrier_off(netdev);
 574         netif_tx_disable(netdev);
 575
 576         hns3_nic_net_down(netdev);
 577
 578         return 0;
 579 }
 580
 581 static int hns3_nic_uc_sync(struct net_device *netdev,
 582                             const unsigned char *addr)
 583 {
 584         struct hnae3_handle *h = hns3_get_handle(netdev);
 585
 586         if (h->ae_algo->ops->add_uc_addr)
 587                 return h->ae_algo->ops->add_uc_addr(h, addr);
 588
 589         return 0;
 590 }
 591
 592 static int hns3_nic_uc_unsync(struct net_device *netdev,
 593                               const unsigned char *addr)
 594 {
 595         struct hnae3_handle *h = hns3_get_handle(netdev);
 596
 597         /* need ignore the request of removing device address, because
 598          * we store the device address and other addresses of uc list
 599          * in the function's mac filter list.
 600          */
 601         if (ether_addr_equal(addr, netdev->dev_addr))
 602                 return 0;
 603
 604         if (h->ae_algo->ops->rm_uc_addr)
 605                 return h->ae_algo->ops->rm_uc_addr(h, addr);
 606
 607         return 0;
 608 }
 609
 610 static int hns3_nic_mc_sync(struct net_device *netdev,
 611                             const unsigned char *addr)
 612 {
 613         struct hnae3_handle *h = hns3_get_handle(netdev);
 614
 615         if (h->ae_algo->ops->add_mc_addr)
 616                 return h->ae_algo->ops->add_mc_addr(h, addr);
 617
 618         return 0;
 619 }
 620
 621 static int hns3_nic_mc_unsync(struct net_device *netdev,
 622                               const unsigned char *addr)
 623 {
 624         struct hnae3_handle *h = hns3_get_handle(netdev);
 625
 626         if (h->ae_algo->ops->rm_mc_addr)
 627                 return h->ae_algo->ops->rm_mc_addr(h, addr);
 628
 629         return 0;
 630 }
 631
 632 static u8 hns3_get_netdev_flags(struct net_device *netdev)
 633 {
 634         u8 flags = 0;
 635
 636         if (netdev->flags & IFF_PROMISC) {
 637                 flags = HNAE3_USER_UPE | HNAE3_USER_MPE | HNAE3_BPE;
 638         } else {
 639                 flags |= HNAE3_VLAN_FLTR;
 640                 if (netdev->flags & IFF_ALLMULTI)
 641                         flags |= HNAE3_USER_MPE;
 642         }
 643
 644         return flags;
 645 }
 646
 647 static void hns3_nic_set_rx_mode(struct net_device *netdev)
 648 {
 649         struct hnae3_handle *h = hns3_get_handle(netdev);
 650         u8 new_flags;
 651
 652         new_flags = hns3_get_netdev_flags(netdev);
 653
 654         __dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync);
 655         __dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync);
 656
 657         /* User mode Promisc mode enable and vlan filtering is disabled to
 658          * let all packets in.
 659          */
 660         h->netdev_flags = new_flags;
 661         hns3_request_update_promisc_mode(h);
 662 }
 663
 664 void hns3_request_update_promisc_mode(struct hnae3_handle *handle)
 665 {
 666         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 667
 668         if (ops->request_update_promisc_mode)
 669                 ops->request_update_promisc_mode(handle);
 670 }
 671
 672 void hns3_enable_vlan_filter(struct net_device *netdev, bool enable)
 673 {
 674         struct hns3_nic_priv *priv = netdev_priv(netdev);
 675         struct hnae3_handle *h = priv->ae_handle;
 676         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev);
 677         bool last_state;
 678
 679         if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2 &&
 680             h->ae_algo->ops->enable_vlan_filter) {
 681                 last_state = h->netdev_flags & HNAE3_VLAN_FLTR ? true : false;
 682                 if (enable != last_state) {
 683                         netdev_info(netdev,
 684                                     "%s vlan filter\n",
 685                                     enable ? "enable" : "disable");
 686                         h->ae_algo->ops->enable_vlan_filter(h, enable);
 687                 }
 688         }
 689 }
 690
 691 static int hns3_set_tso(struct sk_buff *skb, u32 *paylen_fdop_ol4cs,
 692                         u16 *mss, u32 *type_cs_vlan_tso, u32 *send_bytes)
 693 {
 694         u32 l4_offset, hdr_len;
 695         union l3_hdr_info l3;
 696         union l4_hdr_info l4;
 697         u32 l4_paylen;
 698         int ret;
 699
 700         if (!skb_is_gso(skb))
 701                 return 0;
 702
 703         ret = skb_cow_head(skb, 0);
 704         if (unlikely(ret < 0))
 705                 return ret;
 706
 707         l3.hdr = skb_network_header(skb);
 708         l4.hdr = skb_transport_header(skb);
 709
 710         /* Software should clear the IPv4's checksum field when tso is
 711          * needed.
 712          */
 713         if (l3.v4->version == 4)
 714                 l3.v4->check = 0;
 715
 716         /* tunnel packet */
 717         if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
 718                                          SKB_GSO_GRE_CSUM |
 719                                          SKB_GSO_UDP_TUNNEL |
 720                                          SKB_GSO_UDP_TUNNEL_CSUM)) {
 721                 /* reset l3&l4 pointers from outer to inner headers */
 722                 l3.hdr = skb_inner_network_header(skb);
 723                 l4.hdr = skb_inner_transport_header(skb);
 724
 725                 /* Software should clear the IPv4's checksum field when
 726                  * tso is needed.
 727                  */
 728                 if (l3.v4->version == 4)
 729                         l3.v4->check = 0;
 730         }
 731
 732         /* normal or tunnel packet */
 733         l4_offset = l4.hdr - skb->data;
 734
 735         /* remove payload length from inner pseudo checksum when tso */
 736         l4_paylen = skb->len - l4_offset;
 737
 738         if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
 739                 hdr_len = sizeof(*l4.udp) + l4_offset;
 740                 csum_replace_by_diff(&l4.udp->check,
 741                                      (__force __wsum)htonl(l4_paylen));
 742         } else {
 743                 hdr_len = (l4.tcp->doff << 2) + l4_offset;
 744                 csum_replace_by_diff(&l4.tcp->check,
 745                                      (__force __wsum)htonl(l4_paylen));
 746         }
 747
 748         *send_bytes = (skb_shinfo(skb)->gso_segs - 1) * hdr_len + skb->len;
 749
 750         /* find the txbd field values */
 751         *paylen_fdop_ol4cs = skb->len - hdr_len;
 752         hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_TSO_B, 1);
 753
 754         /* offload outer UDP header checksum */
 755         if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)
 756                 hns3_set_field(*paylen_fdop_ol4cs, HNS3_TXD_OL4CS_B, 1);
 757
 758         /* get MSS for TSO */
 759         *mss = skb_shinfo(skb)->gso_size;
 760
 761         trace_hns3_tso(skb);
 762
 763         return 0;
 764 }
 765
 766 static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
 767                                 u8 *il4_proto)
 768 {
 769         union l3_hdr_info l3;
 770         unsigned char *l4_hdr;
 771         unsigned char *exthdr;
 772         u8 l4_proto_tmp;
 773         __be16 frag_off;
 774
 775         /* find outer header point */
 776         l3.hdr = skb_network_header(skb);
 777         l4_hdr = skb_transport_header(skb);
 778
 779         if (skb->protocol == htons(ETH_P_IPV6)) {
 780                 exthdr = l3.hdr + sizeof(*l3.v6);
 781                 l4_proto_tmp = l3.v6->nexthdr;
 782                 if (l4_hdr != exthdr)
 783                         ipv6_skip_exthdr(skb, exthdr - skb->data,
 784                                          &l4_proto_tmp, &frag_off);
 785         } else if (skb->protocol == htons(ETH_P_IP)) {
 786                 l4_proto_tmp = l3.v4->protocol;
 787         } else {
 788                 return -EINVAL;
 789         }
 790
 791         *ol4_proto = l4_proto_tmp;
 792
 793         /* tunnel packet */
 794         if (!skb->encapsulation) {
 795                 *il4_proto = 0;
 796                 return 0;
 797         }
 798
 799         /* find inner header point */
 800         l3.hdr = skb_inner_network_header(skb);
 801         l4_hdr = skb_inner_transport_header(skb);
 802
 803         if (l3.v6->version == 6) {
 804                 exthdr = l3.hdr + sizeof(*l3.v6);
 805                 l4_proto_tmp = l3.v6->nexthdr;
 806                 if (l4_hdr != exthdr)
 807                         ipv6_skip_exthdr(skb, exthdr - skb->data,
 808                                          &l4_proto_tmp, &frag_off);
 809         } else if (l3.v4->version == 4) {
 810                 l4_proto_tmp = l3.v4->protocol;
 811         }
 812
 813         *il4_proto = l4_proto_tmp;
 814
 815         return 0;
 816 }
 817
 818 /* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
 819  * and it is udp packet, which has a dest port as the IANA assigned.
 820  * the hardware is expected to do the checksum offload, but the
 821  * hardware will not do the checksum offload when udp dest port is
 822  * 4789, 4790 or 6081.
 823  */
 824 static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
 825 {
 826         struct hns3_nic_priv *priv = netdev_priv(skb->dev);
 827         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);
 828         union l4_hdr_info l4;
 829
 830         /* device version above V3(include V3), the hardware can
 831          * do this checksum offload.
 832          */
 833         if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3)
 834                 return false;
 835
 836         l4.hdr = skb_transport_header(skb);
 837
 838         if (!(!skb->encapsulation &&
 839               (l4.udp->dest == htons(IANA_VXLAN_UDP_PORT) ||
 840               l4.udp->dest == htons(GENEVE_UDP_PORT) ||
 841               l4.udp->dest == htons(4790))))
 842                 return false;
 843
 844         return true;
 845 }
 846
 847 static void hns3_set_outer_l2l3l4(struct sk_buff *skb, u8 ol4_proto,
 848                                   u32 *ol_type_vlan_len_msec)
 849 {
 850         u32 l2_len, l3_len, l4_len;
 851         unsigned char *il2_hdr;
 852         union l3_hdr_info l3;
 853         union l4_hdr_info l4;
 854
 855         l3.hdr = skb_network_header(skb);
 856         l4.hdr = skb_transport_header(skb);
 857
 858         /* compute OL2 header size, defined in 2 Bytes */
 859         l2_len = l3.hdr - skb->data;
 860         hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L2LEN_S, l2_len >> 1);
 861
 862         /* compute OL3 header size, defined in 4 Bytes */
 863         l3_len = l4.hdr - l3.hdr;
 864         hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_S, l3_len >> 2);
 865
 866         il2_hdr = skb_inner_mac_header(skb);
 867         /* compute OL4 header size, defined in 4 Bytes */
 868         l4_len = il2_hdr - l4.hdr;
 869         hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_S, l4_len >> 2);
 870
 871         /* define outer network header type */
 872         if (skb->protocol == htons(ETH_P_IP)) {
 873                 if (skb_is_gso(skb))
 874                         hns3_set_field(*ol_type_vlan_len_msec,
 875                                        HNS3_TXD_OL3T_S,
 876                                        HNS3_OL3T_IPV4_CSUM);
 877                 else
 878                         hns3_set_field(*ol_type_vlan_len_msec,
 879                                        HNS3_TXD_OL3T_S,
 880                                        HNS3_OL3T_IPV4_NO_CSUM);
 881         } else if (skb->protocol == htons(ETH_P_IPV6)) {
 882                 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_S,
 883                                HNS3_OL3T_IPV6);
 884         }
 885
 886         if (ol4_proto == IPPROTO_UDP)
 887                 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_TUNTYPE_S,
 888                                HNS3_TUN_MAC_IN_UDP);
 889         else if (ol4_proto == IPPROTO_GRE)
 890                 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_TUNTYPE_S,
 891                                HNS3_TUN_NVGRE);
 892 }
 893
 894 static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto,
 895                            u8 il4_proto, u32 *type_cs_vlan_tso,
 896                            u32 *ol_type_vlan_len_msec)
 897 {
 898         unsigned char *l2_hdr = skb->data;
 899         u32 l4_proto = ol4_proto;
 900         union l4_hdr_info l4;
 901         union l3_hdr_info l3;
 902         u32 l2_len, l3_len;
 903
 904         l4.hdr = skb_transport_header(skb);
 905         l3.hdr = skb_network_header(skb);
 906
 907         /* handle encapsulation skb */
 908         if (skb->encapsulation) {
 909                 /* If this is a not UDP/GRE encapsulation skb */
 910                 if (!(ol4_proto == IPPROTO_UDP || ol4_proto == IPPROTO_GRE)) {
 911                         /* drop the skb tunnel packet if hardware don't support,
 912                          * because hardware can't calculate csum when TSO.
 913                          */
 914                         if (skb_is_gso(skb))
 915                                 return -EDOM;
 916
 917                         /* the stack computes the IP header already,
 918                          * driver calculate l4 checksum when not TSO.
 919                          */
 920                         return skb_checksum_help(skb);
 921                 }
 922
 923                 hns3_set_outer_l2l3l4(skb, ol4_proto, ol_type_vlan_len_msec);
 924
 925                 /* switch to inner header */
 926                 l2_hdr = skb_inner_mac_header(skb);
 927                 l3.hdr = skb_inner_network_header(skb);
 928                 l4.hdr = skb_inner_transport_header(skb);
 929                 l4_proto = il4_proto;
 930         }
 931
 932         if (l3.v4->version == 4) {
 933                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S,
 934                                HNS3_L3T_IPV4);
 935
 936                 /* the stack computes the IP header already, the only time we
 937                  * need the hardware to recompute it is in the case of TSO.
 938                  */
 939                 if (skb_is_gso(skb))
 940                         hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
 941         } else if (l3.v6->version == 6) {
 942                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S,
 943                                HNS3_L3T_IPV6);
 944         }
 945
 946         /* compute inner(/normal) L2 header size, defined in 2 Bytes */
 947         l2_len = l3.hdr - l2_hdr;
 948         hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_S, l2_len >> 1);
 949
 950         /* compute inner(/normal) L3 header size, defined in 4 Bytes */
 951         l3_len = l4.hdr - l3.hdr;
 952         hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_S, l3_len >> 2);
 953
 954         /* compute inner(/normal) L4 header size, defined in 4 Bytes */
 955         switch (l4_proto) {
 956         case IPPROTO_TCP:
 957                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 958                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
 959                                HNS3_L4T_TCP);
 960                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
 961                                l4.tcp->doff);
 962                 break;
 963         case IPPROTO_UDP:
 964                 if (hns3_tunnel_csum_bug(skb))
 965                         return skb_checksum_help(skb);
 966
 967                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 968                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
 969                                HNS3_L4T_UDP);
 970                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
 971                                (sizeof(struct udphdr) >> 2));
 972                 break;
 973         case IPPROTO_SCTP:
 974                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 975                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
 976                                HNS3_L4T_SCTP);
 977                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
 978                                (sizeof(struct sctphdr) >> 2));
 979                 break;
 980         default:
 981                 /* drop the skb tunnel packet if hardware don't support,
 982                  * because hardware can't calculate csum when TSO.
 983                  */
 984                 if (skb_is_gso(skb))
 985                         return -EDOM;
 986
 987                 /* the stack computes the IP header already,
 988                  * driver calculate l4 checksum when not TSO.
 989                  */
 990                 return skb_checksum_help(skb);
 991         }
 992
 993         return 0;
 994 }
 995
 996 static int hns3_handle_vtags(struct hns3_enet_ring *tx_ring,
 997                              struct sk_buff *skb)
 998 {
 999         struct hnae3_handle *handle = tx_ring->tqp->handle;
1000         struct hnae3_ae_dev *ae_dev;
1001         struct vlan_ethhdr *vhdr;
1002         int rc;
1003
1004         if (!(skb->protocol == htons(ETH_P_8021Q) ||
1005               skb_vlan_tag_present(skb)))
1006                 return 0;
1007
1008         /* For HW limitation on HNAE3_DEVICE_VERSION_V2, if port based insert
1009          * VLAN enabled, only one VLAN header is allowed in skb, otherwise it
1010          * will cause RAS error.
1011          */
1012         ae_dev = pci_get_drvdata(handle->pdev);
1013         if (unlikely(skb_vlan_tagged_multi(skb) &&
1014                      ae_dev->dev_version <= HNAE3_DEVICE_VERSION_V2 &&
1015                      handle->port_base_vlan_state ==
1016                      HNAE3_PORT_BASE_VLAN_ENABLE))
1017                 return -EINVAL;
1018
1019         if (skb->protocol == htons(ETH_P_8021Q) &&
1020             !(handle->kinfo.netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) {
1021                 /* When HW VLAN acceleration is turned off, and the stack
1022                  * sets the protocol to 802.1q, the driver just need to
1023                  * set the protocol to the encapsulated ethertype.
1024                  */
1025                 skb->protocol = vlan_get_protocol(skb);
1026                 return 0;
1027         }
1028
1029         if (skb_vlan_tag_present(skb)) {
1030                 /* Based on hw strategy, use out_vtag in two layer tag case,
1031                  * and use inner_vtag in one tag case.
1032                  */
1033                 if (skb->protocol == htons(ETH_P_8021Q) &&
1034                     handle->port_base_vlan_state ==
1035                     HNAE3_PORT_BASE_VLAN_DISABLE)
1036                         rc = HNS3_OUTER_VLAN_TAG;
1037                 else
1038                         rc = HNS3_INNER_VLAN_TAG;
1039
1040                 skb->protocol = vlan_get_protocol(skb);
1041                 return rc;
1042         }
1043
1044         rc = skb_cow_head(skb, 0);
1045         if (unlikely(rc < 0))
1046                 return rc;
1047
1048         vhdr = (struct vlan_ethhdr *)skb->data;
1049         vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority << VLAN_PRIO_SHIFT)
1050                                          & VLAN_PRIO_MASK);
1051
1052         skb->protocol = vlan_get_protocol(skb);
1053         return 0;
1054 }
1055
1056 /* check if the hardware is capable of checksum offloading */
1057 static bool hns3_check_hw_tx_csum(struct sk_buff *skb)
1058 {
1059         struct hns3_nic_priv *priv = netdev_priv(skb->dev);
1060
1061         /* Kindly note, due to backward compatibility of the TX descriptor,
1062          * HW checksum of the non-IP packets and GSO packets is handled at
1063          * different place in the following code
1064          */
1065         if (skb_csum_is_sctp(skb) || skb_is_gso(skb) ||
1066             !test_bit(HNS3_NIC_STATE_HW_TX_CSUM_ENABLE, &priv->state))
1067                 return false;
1068
1069         return true;
1070 }
1071
1072 static int hns3_fill_skb_desc(struct hns3_enet_ring *ring,
1073                               struct sk_buff *skb, struct hns3_desc *desc,
1074                               struct hns3_desc_cb *desc_cb)
1075 {
1076         u32 ol_type_vlan_len_msec = 0;
1077         u32 paylen_ol4cs = skb->len;
1078         u32 type_cs_vlan_tso = 0;
1079         u16 mss_hw_csum = 0;
1080         u16 inner_vtag = 0;
1081         u16 out_vtag = 0;
1082         int ret;
1083
1084         ret = hns3_handle_vtags(ring, skb);
1085         if (unlikely(ret < 0)) {
1086                 u64_stats_update_begin(&ring->syncp);
1087                 ring->stats.tx_vlan_err++;
1088                 u64_stats_update_end(&ring->syncp);
1089                 return ret;
1090         } else if (ret == HNS3_INNER_VLAN_TAG) {
1091                 inner_vtag = skb_vlan_tag_get(skb);
1092                 inner_vtag |= (skb->priority << VLAN_PRIO_SHIFT) &
1093                                 VLAN_PRIO_MASK;
1094                 hns3_set_field(type_cs_vlan_tso, HNS3_TXD_VLAN_B, 1);
1095         } else if (ret == HNS3_OUTER_VLAN_TAG) {
1096                 out_vtag = skb_vlan_tag_get(skb);
1097                 out_vtag |= (skb->priority << VLAN_PRIO_SHIFT) &
1098                                 VLAN_PRIO_MASK;
1099                 hns3_set_field(ol_type_vlan_len_msec, HNS3_TXD_OVLAN_B,
1100                                1);
1101         }
1102
1103         desc_cb->send_bytes = skb->len;
1104
1105         if (skb->ip_summed == CHECKSUM_PARTIAL) {
1106                 u8 ol4_proto, il4_proto;
1107
1108                 if (hns3_check_hw_tx_csum(skb)) {
1109                         /* set checksum start and offset, defined in 2 Bytes */
1110                         hns3_set_field(type_cs_vlan_tso, HNS3_TXD_CSUM_START_S,
1111                                        skb_checksum_start_offset(skb) >> 1);
1112                         hns3_set_field(ol_type_vlan_len_msec,
1113                                        HNS3_TXD_CSUM_OFFSET_S,
1114                                        skb->csum_offset >> 1);
1115                         mss_hw_csum |= BIT(HNS3_TXD_HW_CS_B);
1116                         goto out_hw_tx_csum;
1117                 }
1118
1119                 skb_reset_mac_len(skb);
1120
1121                 ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
1122                 if (unlikely(ret < 0)) {
1123                         u64_stats_update_begin(&ring->syncp);
1124                         ring->stats.tx_l4_proto_err++;
1125                         u64_stats_update_end(&ring->syncp);
1126                         return ret;
1127                 }
1128
1129                 ret = hns3_set_l2l3l4(skb, ol4_proto, il4_proto,
1130                                       &type_cs_vlan_tso,
1131                                       &ol_type_vlan_len_msec);
1132                 if (unlikely(ret < 0)) {
1133                         u64_stats_update_begin(&ring->syncp);
1134                         ring->stats.tx_l2l3l4_err++;
1135                         u64_stats_update_end(&ring->syncp);
1136                         return ret;
1137                 }
1138
1139                 ret = hns3_set_tso(skb, &paylen_ol4cs, &mss_hw_csum,
1140                                    &type_cs_vlan_tso, &desc_cb->send_bytes);
1141                 if (unlikely(ret < 0)) {
1142                         u64_stats_update_begin(&ring->syncp);
1143                         ring->stats.tx_tso_err++;
1144                         u64_stats_update_end(&ring->syncp);
1145                         return ret;
1146                 }
1147         }
1148
1149 out_hw_tx_csum:
1150         /* Set txbd */
1151         desc->tx.ol_type_vlan_len_msec =
1152                 cpu_to_le32(ol_type_vlan_len_msec);
1153         desc->tx.type_cs_vlan_tso_len = cpu_to_le32(type_cs_vlan_tso);
1154         desc->tx.paylen_ol4cs = cpu_to_le32(paylen_ol4cs);
1155         desc->tx.mss_hw_csum = cpu_to_le16(mss_hw_csum);
1156         desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
1157         desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
1158
1159         return 0;
1160 }
1161
1162 static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
1163                           unsigned int size, enum hns_desc_type type)
1164 {
1165 #define HNS3_LIKELY_BD_NUM      1
1166
1167         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
1168         struct hns3_desc *desc = &ring->desc[ring->next_to_use];
1169         struct device *dev = ring_to_dev(ring);
1170         skb_frag_t *frag;
1171         unsigned int frag_buf_num;
1172         int k, sizeoflast;
1173         dma_addr_t dma;
1174
1175         if (type == DESC_TYPE_FRAGLIST_SKB ||
1176             type == DESC_TYPE_SKB) {
1177                 struct sk_buff *skb = (struct sk_buff *)priv;
1178
1179                 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1180         } else {
1181                 frag = (skb_frag_t *)priv;
1182                 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1183         }
1184
1185         if (unlikely(dma_mapping_error(dev, dma))) {
1186                 u64_stats_update_begin(&ring->syncp);
1187                 ring->stats.sw_err_cnt++;
1188                 u64_stats_update_end(&ring->syncp);
1189                 return -ENOMEM;
1190         }
1191
1192         desc_cb->priv = priv;
1193         desc_cb->length = size;
1194         desc_cb->dma = dma;
1195         desc_cb->type = type;
1196
1197         if (likely(size <= HNS3_MAX_BD_SIZE)) {
1198                 desc->addr = cpu_to_le64(dma);
1199                 desc->tx.send_size = cpu_to_le16(size);
1200                 desc->tx.bdtp_fe_sc_vld_ra_ri =
1201                         cpu_to_le16(BIT(HNS3_TXD_VLD_B));
1202
1203                 trace_hns3_tx_desc(ring, ring->next_to_use);
1204                 ring_ptr_move_fw(ring, next_to_use);
1205                 return HNS3_LIKELY_BD_NUM;
1206         }
1207
1208         frag_buf_num = hns3_tx_bd_count(size);
1209         sizeoflast = size % HNS3_MAX_BD_SIZE;
1210         sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
1211
1212         /* When frag size is bigger than hardware limit, split this frag */
1213         for (k = 0; k < frag_buf_num; k++) {
1214                 /* now, fill the descriptor */
1215                 desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k);
1216                 desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ?
1217                                      (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE);
1218                 desc->tx.bdtp_fe_sc_vld_ra_ri =
1219                                 cpu_to_le16(BIT(HNS3_TXD_VLD_B));
1220
1221                 trace_hns3_tx_desc(ring, ring->next_to_use);
1222                 /* move ring pointer to next */
1223                 ring_ptr_move_fw(ring, next_to_use);
1224
1225                 desc = &ring->desc[ring->next_to_use];
1226         }
1227
1228         return frag_buf_num;
1229 }
1230
1231 static unsigned int hns3_skb_bd_num(struct sk_buff *skb, unsigned int *bd_size,
1232                                     unsigned int bd_num)
1233 {
1234         unsigned int size;
1235         int i;
1236
1237         size = skb_headlen(skb);
1238         while (size > HNS3_MAX_BD_SIZE) {
1239                 bd_size[bd_num++] = HNS3_MAX_BD_SIZE;
1240                 size -= HNS3_MAX_BD_SIZE;
1241
1242                 if (bd_num > HNS3_MAX_TSO_BD_NUM)
1243                         return bd_num;
1244         }
1245
1246         if (size) {
1247                 bd_size[bd_num++] = size;
1248                 if (bd_num > HNS3_MAX_TSO_BD_NUM)
1249                         return bd_num;
1250         }
1251
1252         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1253                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1254                 size = skb_frag_size(frag);
1255                 if (!size)
1256                         continue;
1257
1258                 while (size > HNS3_MAX_BD_SIZE) {
1259                         bd_size[bd_num++] = HNS3_MAX_BD_SIZE;
1260                         size -= HNS3_MAX_BD_SIZE;
1261
1262                         if (bd_num > HNS3_MAX_TSO_BD_NUM)
1263                                 return bd_num;
1264                 }
1265
1266                 bd_size[bd_num++] = size;
1267                 if (bd_num > HNS3_MAX_TSO_BD_NUM)
1268                         return bd_num;
1269         }
1270
1271         return bd_num;
1272 }
1273
1274 static unsigned int hns3_tx_bd_num(struct sk_buff *skb, unsigned int *bd_size,
1275                                    u8 max_non_tso_bd_num, unsigned int bd_num,
1276                                    unsigned int recursion_level)
1277 {
1278 #define HNS3_MAX_RECURSION_LEVEL        24
1279
1280         struct sk_buff *frag_skb;
1281
1282         /* If the total len is within the max bd limit */
1283         if (likely(skb->len <= HNS3_MAX_BD_SIZE && !recursion_level &&
1284                    !skb_has_frag_list(skb) &&
1285                    skb_shinfo(skb)->nr_frags < max_non_tso_bd_num))
1286                 return skb_shinfo(skb)->nr_frags + 1U;
1287
1288         if (unlikely(recursion_level >= HNS3_MAX_RECURSION_LEVEL))
1289                 return UINT_MAX;
1290
1291         bd_num = hns3_skb_bd_num(skb, bd_size, bd_num);
1292         if (!skb_has_frag_list(skb) || bd_num > HNS3_MAX_TSO_BD_NUM)
1293                 return bd_num;
1294
1295         skb_walk_frags(skb, frag_skb) {
1296                 bd_num = hns3_tx_bd_num(frag_skb, bd_size, max_non_tso_bd_num,
1297                                         bd_num, recursion_level + 1);
1298                 if (bd_num > HNS3_MAX_TSO_BD_NUM)
1299                         return bd_num;
1300         }
1301
1302         return bd_num;
1303 }
1304
1305 static unsigned int hns3_gso_hdr_len(struct sk_buff *skb)
1306 {
1307         if (!skb->encapsulation)
1308                 return skb_transport_offset(skb) + tcp_hdrlen(skb);
1309
1310         return skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb);
1311 }
1312
1313 /* HW need every continuous max_non_tso_bd_num buffer data to be larger
1314  * than MSS, we simplify it by ensuring skb_headlen + the first continuous
1315  * max_non_tso_bd_num - 1 frags to be larger than gso header len + mss,
1316  * and the remaining continuous max_non_tso_bd_num - 1 frags to be larger
1317  * than MSS except the last max_non_tso_bd_num - 1 frags.
1318  */
1319 static bool hns3_skb_need_linearized(struct sk_buff *skb, unsigned int *bd_size,
1320                                      unsigned int bd_num, u8 max_non_tso_bd_num)
1321 {
1322         unsigned int tot_len = 0;
1323         int i;
1324
1325         for (i = 0; i < max_non_tso_bd_num - 1U; i++)
1326                 tot_len += bd_size[i];
1327
1328         /* ensure the first max_non_tso_bd_num frags is greater than
1329          * mss + header
1330          */
1331         if (tot_len + bd_size[max_non_tso_bd_num - 1U] <
1332             skb_shinfo(skb)->gso_size + hns3_gso_hdr_len(skb))
1333                 return true;
1334
1335         /* ensure every continuous max_non_tso_bd_num - 1 buffer is greater
1336          * than mss except the last one.
1337          */
1338         for (i = 0; i < bd_num - max_non_tso_bd_num; i++) {
1339                 tot_len -= bd_size[i];
1340                 tot_len += bd_size[i + max_non_tso_bd_num - 1U];
1341
1342                 if (tot_len < skb_shinfo(skb)->gso_size)
1343                         return true;
1344         }
1345
1346         return false;
1347 }
1348
1349 void hns3_shinfo_pack(struct skb_shared_info *shinfo, __u32 *size)
1350 {
1351         int i;
1352
1353         for (i = 0; i < MAX_SKB_FRAGS; i++)
1354                 size[i] = skb_frag_size(&shinfo->frags[i]);
1355 }
1356
1357 static int hns3_skb_linearize(struct hns3_enet_ring *ring,
1358                               struct sk_buff *skb,
1359                               u8 max_non_tso_bd_num,
1360                               unsigned int bd_num)
1361 {
1362         /* 'bd_num == UINT_MAX' means the skb' fraglist has a
1363          * recursion level of over HNS3_MAX_RECURSION_LEVEL.
1364          */
1365         if (bd_num == UINT_MAX) {
1366                 u64_stats_update_begin(&ring->syncp);
1367                 ring->stats.over_max_recursion++;
1368                 u64_stats_update_end(&ring->syncp);
1369                 return -ENOMEM;
1370         }
1371
1372         /* The skb->len has exceeded the hw limitation, linearization
1373          * will not help.
1374          */
1375         if (skb->len > HNS3_MAX_TSO_SIZE ||
1376             (!skb_is_gso(skb) && skb->len >
1377              HNS3_MAX_NON_TSO_SIZE(max_non_tso_bd_num))) {
1378                 u64_stats_update_begin(&ring->syncp);
1379                 ring->stats.hw_limitation++;
1380                 u64_stats_update_end(&ring->syncp);
1381                 return -ENOMEM;
1382         }
1383
1384         if (__skb_linearize(skb)) {
1385                 u64_stats_update_begin(&ring->syncp);
1386                 ring->stats.sw_err_cnt++;
1387                 u64_stats_update_end(&ring->syncp);
1388                 return -ENOMEM;
1389         }
1390
1391         return 0;
1392 }
1393
1394 static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring,
1395                                   struct net_device *netdev,
1396                                   struct sk_buff *skb)
1397 {
1398         struct hns3_nic_priv *priv = netdev_priv(netdev);
1399         u8 max_non_tso_bd_num = priv->max_non_tso_bd_num;
1400         unsigned int bd_size[HNS3_MAX_TSO_BD_NUM + 1U];
1401         unsigned int bd_num;
1402
1403         bd_num = hns3_tx_bd_num(skb, bd_size, max_non_tso_bd_num, 0, 0);
1404         if (unlikely(bd_num > max_non_tso_bd_num)) {
1405                 if (bd_num <= HNS3_MAX_TSO_BD_NUM && skb_is_gso(skb) &&
1406                     !hns3_skb_need_linearized(skb, bd_size, bd_num,
1407                                               max_non_tso_bd_num)) {
1408                         trace_hns3_over_max_bd(skb);
1409                         goto out;
1410                 }
1411
1412                 if (hns3_skb_linearize(ring, skb, max_non_tso_bd_num,
1413                                        bd_num))
1414                         return -ENOMEM;
1415
1416                 bd_num = hns3_tx_bd_count(skb->len);
1417
1418                 u64_stats_update_begin(&ring->syncp);
1419                 ring->stats.tx_copy++;
1420                 u64_stats_update_end(&ring->syncp);
1421         }
1422
1423 out:
1424         if (likely(ring_space(ring) >= bd_num))
1425                 return bd_num;
1426
1427         netif_stop_subqueue(netdev, ring->queue_index);
1428         smp_mb(); /* Memory barrier before checking ring_space */
1429
1430         /* Start queue in case hns3_clean_tx_ring has just made room
1431          * available and has not seen the queue stopped state performed
1432          * by netif_stop_subqueue above.
1433          */
1434         if (ring_space(ring) >= bd_num && netif_carrier_ok(netdev) &&
1435             !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
1436                 netif_start_subqueue(netdev, ring->queue_index);
1437                 return bd_num;
1438         }
1439
1440         u64_stats_update_begin(&ring->syncp);
1441         ring->stats.tx_busy++;
1442         u64_stats_update_end(&ring->syncp);
1443
1444         return -EBUSY;
1445 }
1446
1447 static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)
1448 {
1449         struct device *dev = ring_to_dev(ring);
1450         unsigned int i;
1451
1452         for (i = 0; i < ring->desc_num; i++) {
1453                 struct hns3_desc *desc = &ring->desc[ring->next_to_use];
1454
1455                 memset(desc, 0, sizeof(*desc));
1456
1457                 /* check if this is where we started */
1458                 if (ring->next_to_use == next_to_use_orig)
1459                         break;
1460
1461                 /* rollback one */
1462                 ring_ptr_move_bw(ring, next_to_use);
1463
1464                 if (!ring->desc_cb[ring->next_to_use].dma)
1465                         continue;
1466
1467                 /* unmap the descriptor dma address */
1468                 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB ||
1469                     ring->desc_cb[ring->next_to_use].type ==
1470                     DESC_TYPE_FRAGLIST_SKB)
1471                         dma_unmap_single(dev,
1472                                          ring->desc_cb[ring->next_to_use].dma,
1473                                         ring->desc_cb[ring->next_to_use].length,
1474                                         DMA_TO_DEVICE);
1475                 else if (ring->desc_cb[ring->next_to_use].length)
1476                         dma_unmap_page(dev,
1477                                        ring->desc_cb[ring->next_to_use].dma,
1478                                        ring->desc_cb[ring->next_to_use].length,
1479                                        DMA_TO_DEVICE);
1480
1481                 ring->desc_cb[ring->next_to_use].length = 0;
1482                 ring->desc_cb[ring->next_to_use].dma = 0;
1483                 ring->desc_cb[ring->next_to_use].type = DESC_TYPE_UNKNOWN;
1484         }
1485 }
1486
1487 static int hns3_fill_skb_to_desc(struct hns3_enet_ring *ring,
1488                                  struct sk_buff *skb, enum hns_desc_type type)
1489 {
1490         unsigned int size = skb_headlen(skb);
1491         struct sk_buff *frag_skb;
1492         int i, ret, bd_num = 0;
1493
1494         if (size) {
1495                 ret = hns3_fill_desc(ring, skb, size, type);
1496                 if (unlikely(ret < 0))
1497                         return ret;
1498
1499                 bd_num += ret;
1500         }
1501
1502         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1503                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1504
1505                 size = skb_frag_size(frag);
1506                 if (!size)
1507                         continue;
1508
1509                 ret = hns3_fill_desc(ring, frag, size, DESC_TYPE_PAGE);
1510                 if (unlikely(ret < 0))
1511                         return ret;
1512
1513                 bd_num += ret;
1514         }
1515
1516         skb_walk_frags(skb, frag_skb) {
1517                 ret = hns3_fill_skb_to_desc(ring, frag_skb,
1518                                             DESC_TYPE_FRAGLIST_SKB);
1519                 if (unlikely(ret < 0))
1520                         return ret;
1521
1522                 bd_num += ret;
1523         }
1524
1525         return bd_num;
1526 }
1527
1528 static void hns3_tx_doorbell(struct hns3_enet_ring *ring, int num,
1529                              bool doorbell)
1530 {
1531         ring->pending_buf += num;
1532
1533         if (!doorbell) {
1534                 u64_stats_update_begin(&ring->syncp);
1535                 ring->stats.tx_more++;
1536                 u64_stats_update_end(&ring->syncp);
1537                 return;
1538         }
1539
1540         if (!ring->pending_buf)
1541                 return;
1542
1543         writel(ring->pending_buf,
1544                ring->tqp->io_base + HNS3_RING_TX_RING_TAIL_REG);
1545         ring->pending_buf = 0;
1546         WRITE_ONCE(ring->last_to_use, ring->next_to_use);
1547 }
1548
1549 netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
1550 {
1551         struct hns3_nic_priv *priv = netdev_priv(netdev);
1552         struct hns3_enet_ring *ring = &priv->ring[skb->queue_mapping];
1553         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
1554         struct netdev_queue *dev_queue;
1555         int pre_ntu, next_to_use_head;
1556         bool doorbell;
1557         int ret;
1558
1559         /* Hardware can only handle short frames above 32 bytes */
1560         if (skb_put_padto(skb, HNS3_MIN_TX_LEN)) {
1561                 hns3_tx_doorbell(ring, 0, !netdev_xmit_more());
1562
1563                 u64_stats_update_begin(&ring->syncp);
1564                 ring->stats.sw_err_cnt++;
1565                 u64_stats_update_end(&ring->syncp);
1566
1567                 return NETDEV_TX_OK;
1568         }
1569
1570         /* Prefetch the data used later */
1571         prefetch(skb->data);
1572
1573         ret = hns3_nic_maybe_stop_tx(ring, netdev, skb);
1574         if (unlikely(ret <= 0)) {
1575                 if (ret == -EBUSY) {
1576                         hns3_tx_doorbell(ring, 0, true);
1577                         return NETDEV_TX_BUSY;
1578                 }
1579
1580                 hns3_rl_err(netdev, "xmit error: %d!\n", ret);
1581                 goto out_err_tx_ok;
1582         }
1583
1584         next_to_use_head = ring->next_to_use;
1585
1586         ret = hns3_fill_skb_desc(ring, skb, &ring->desc[ring->next_to_use],
1587                                  desc_cb);
1588         if (unlikely(ret < 0))
1589                 goto fill_err;
1590
1591         /* 'ret < 0' means filling error, 'ret == 0' means skb->len is
1592          * zero, which is unlikely, and 'ret > 0' means how many tx desc
1593          * need to be notified to the hw.
1594          */
1595         ret = hns3_fill_skb_to_desc(ring, skb, DESC_TYPE_SKB);
1596         if (unlikely(ret <= 0))
1597                 goto fill_err;
1598
1599         pre_ntu = ring->next_to_use ? (ring->next_to_use - 1) :
1600                                         (ring->desc_num - 1);
1601         ring->desc[pre_ntu].tx.bdtp_fe_sc_vld_ra_ri |=
1602                                 cpu_to_le16(BIT(HNS3_TXD_FE_B));
1603         trace_hns3_tx_desc(ring, pre_ntu);
1604
1605         /* Complete translate all packets */
1606         dev_queue = netdev_get_tx_queue(netdev, ring->queue_index);
1607         doorbell = __netdev_tx_sent_queue(dev_queue, desc_cb->send_bytes,
1608                                           netdev_xmit_more());
1609         hns3_tx_doorbell(ring, ret, doorbell);
1610
1611         return NETDEV_TX_OK;
1612
1613 fill_err:
1614         hns3_clear_desc(ring, next_to_use_head);
1615
1616 out_err_tx_ok:
1617         dev_kfree_skb_any(skb);
1618         hns3_tx_doorbell(ring, 0, !netdev_xmit_more());
1619         return NETDEV_TX_OK;
1620 }
1621
1622 static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1623 {
1624         struct hnae3_handle *h = hns3_get_handle(netdev);
1625         struct sockaddr *mac_addr = p;
1626         int ret;
1627
1628         if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1629                 return -EADDRNOTAVAIL;
1630
1631         if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
1632                 netdev_info(netdev, "already using mac address %pM\n",
1633                             mac_addr->sa_data);
1634                 return 0;
1635         }
1636
1637         /* For VF device, if there is a perm_addr, then the user will not
1638          * be allowed to change the address.
1639          */
1640         if (!hns3_is_phys_func(h->pdev) &&
1641             !is_zero_ether_addr(netdev->perm_addr)) {
1642                 netdev_err(netdev, "has permanent MAC %pM, user MAC %pM not allow\n",
1643                            netdev->perm_addr, mac_addr->sa_data);
1644                 return -EPERM;
1645         }
1646
1647         ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
1648         if (ret) {
1649                 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1650                 return ret;
1651         }
1652
1653         ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1654
1655         return 0;
1656 }
1657
1658 static int hns3_nic_do_ioctl(struct net_device *netdev,
1659                              struct ifreq *ifr, int cmd)
1660 {
1661         struct hnae3_handle *h = hns3_get_handle(netdev);
1662
1663         if (!netif_running(netdev))
1664                 return -EINVAL;
1665
1666         if (!h->ae_algo->ops->do_ioctl)
1667                 return -EOPNOTSUPP;
1668
1669         return h->ae_algo->ops->do_ioctl(h, ifr, cmd);
1670 }
1671
1672 static int hns3_nic_set_features(struct net_device *netdev,
1673                                  netdev_features_t features)
1674 {
1675         netdev_features_t changed = netdev->features ^ features;
1676         struct hns3_nic_priv *priv = netdev_priv(netdev);
1677         struct hnae3_handle *h = priv->ae_handle;
1678         bool enable;
1679         int ret;
1680
1681         if (changed & (NETIF_F_GRO_HW) && h->ae_algo->ops->set_gro_en) {
1682                 enable = !!(features & NETIF_F_GRO_HW);
1683                 ret = h->ae_algo->ops->set_gro_en(h, enable);
1684                 if (ret)
1685                         return ret;
1686         }
1687
1688         if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1689             h->ae_algo->ops->enable_hw_strip_rxvtag) {
1690                 enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
1691                 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, enable);
1692                 if (ret)
1693                         return ret;
1694         }
1695
1696         if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) {
1697                 enable = !!(features & NETIF_F_NTUPLE);
1698                 h->ae_algo->ops->enable_fd(h, enable);
1699         }
1700
1701         if ((netdev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC) &&
1702             h->ae_algo->ops->cls_flower_active(h)) {
1703                 netdev_err(netdev,
1704                            "there are offloaded TC filters active, cannot disable HW TC offload");
1705                 return -EINVAL;
1706         }
1707
1708         netdev->features = features;
1709         return 0;
1710 }
1711
1712 static netdev_features_t hns3_features_check(struct sk_buff *skb,
1713                                              struct net_device *dev,
1714                                              netdev_features_t features)
1715 {
1716 #define HNS3_MAX_HDR_LEN        480U
1717 #define HNS3_MAX_L4_HDR_LEN     60U
1718
1719         size_t len;
1720
1721         if (skb->ip_summed != CHECKSUM_PARTIAL)
1722                 return features;
1723
1724         if (skb->encapsulation)
1725                 len = skb_inner_transport_header(skb) - skb->data;
1726         else
1727                 len = skb_transport_header(skb) - skb->data;
1728
1729         /* Assume L4 is 60 byte as TCP is the only protocol with a
1730          * a flexible value, and it's max len is 60 bytes.
1731          */
1732         len += HNS3_MAX_L4_HDR_LEN;
1733
1734         /* Hardware only supports checksum on the skb with a max header
1735          * len of 480 bytes.
1736          */
1737         if (len > HNS3_MAX_HDR_LEN)
1738                 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
1739
1740         return features;
1741 }
1742
1743 static void hns3_nic_get_stats64(struct net_device *netdev,
1744                                  struct rtnl_link_stats64 *stats)
1745 {
1746         struct hns3_nic_priv *priv = netdev_priv(netdev);
1747         int queue_num = priv->ae_handle->kinfo.num_tqps;
1748         struct hnae3_handle *handle = priv->ae_handle;
1749         struct hns3_enet_ring *ring;
1750         u64 rx_length_errors = 0;
1751         u64 rx_crc_errors = 0;
1752         u64 rx_multicast = 0;
1753         unsigned int start;
1754         u64 tx_errors = 0;
1755         u64 rx_errors = 0;
1756         unsigned int idx;
1757         u64 tx_bytes = 0;
1758         u64 rx_bytes = 0;
1759         u64 tx_pkts = 0;
1760         u64 rx_pkts = 0;
1761         u64 tx_drop = 0;
1762         u64 rx_drop = 0;
1763
1764         if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1765                 return;
1766
1767         handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1768
1769         for (idx = 0; idx < queue_num; idx++) {
1770                 /* fetch the tx stats */
1771                 ring = &priv->ring[idx];
1772                 do {
1773                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1774                         tx_bytes += ring->stats.tx_bytes;
1775                         tx_pkts += ring->stats.tx_pkts;
1776                         tx_drop += ring->stats.sw_err_cnt;
1777                         tx_drop += ring->stats.tx_vlan_err;
1778                         tx_drop += ring->stats.tx_l4_proto_err;
1779                         tx_drop += ring->stats.tx_l2l3l4_err;
1780                         tx_drop += ring->stats.tx_tso_err;
1781                         tx_drop += ring->stats.over_max_recursion;
1782                         tx_drop += ring->stats.hw_limitation;
1783                         tx_errors += ring->stats.sw_err_cnt;
1784                         tx_errors += ring->stats.tx_vlan_err;
1785                         tx_errors += ring->stats.tx_l4_proto_err;
1786                         tx_errors += ring->stats.tx_l2l3l4_err;
1787                         tx_errors += ring->stats.tx_tso_err;
1788                         tx_errors += ring->stats.over_max_recursion;
1789                         tx_errors += ring->stats.hw_limitation;
1790                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1791
1792                 /* fetch the rx stats */
1793                 ring = &priv->ring[idx + queue_num];
1794                 do {
1795                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1796                         rx_bytes += ring->stats.rx_bytes;
1797                         rx_pkts += ring->stats.rx_pkts;
1798                         rx_drop += ring->stats.l2_err;
1799                         rx_errors += ring->stats.l2_err;
1800                         rx_errors += ring->stats.l3l4_csum_err;
1801                         rx_crc_errors += ring->stats.l2_err;
1802                         rx_multicast += ring->stats.rx_multicast;
1803                         rx_length_errors += ring->stats.err_pkt_len;
1804                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1805         }
1806
1807         stats->tx_bytes = tx_bytes;
1808         stats->tx_packets = tx_pkts;
1809         stats->rx_bytes = rx_bytes;
1810         stats->rx_packets = rx_pkts;
1811
1812         stats->rx_errors = rx_errors;
1813         stats->multicast = rx_multicast;
1814         stats->rx_length_errors = rx_length_errors;
1815         stats->rx_crc_errors = rx_crc_errors;
1816         stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1817
1818         stats->tx_errors = tx_errors;
1819         stats->rx_dropped = rx_drop;
1820         stats->tx_dropped = tx_drop;
1821         stats->collisions = netdev->stats.collisions;
1822         stats->rx_over_errors = netdev->stats.rx_over_errors;
1823         stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1824         stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1825         stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1826         stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1827         stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1828         stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1829         stats->tx_window_errors = netdev->stats.tx_window_errors;
1830         stats->rx_compressed = netdev->stats.rx_compressed;
1831         stats->tx_compressed = netdev->stats.tx_compressed;
1832 }
1833
1834 static int hns3_setup_tc(struct net_device *netdev, void *type_data)
1835 {
1836         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
1837         struct hnae3_knic_private_info *kinfo;
1838         u8 tc = mqprio_qopt->qopt.num_tc;
1839         u16 mode = mqprio_qopt->mode;
1840         u8 hw = mqprio_qopt->qopt.hw;
1841         struct hnae3_handle *h;
1842
1843         if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1844                mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1845                 return -EOPNOTSUPP;
1846
1847         if (tc > HNAE3_MAX_TC)
1848                 return -EINVAL;
1849
1850         if (!netdev)
1851                 return -EINVAL;
1852
1853         h = hns3_get_handle(netdev);
1854         kinfo = &h->kinfo;
1855
1856         netif_dbg(h, drv, netdev, "setup tc: num_tc=%u\n", tc);
1857
1858         return (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1859                 kinfo->dcb_ops->setup_tc(h, mqprio_qopt) : -EOPNOTSUPP;
1860 }
1861
1862 static int hns3_setup_tc_cls_flower(struct hns3_nic_priv *priv,
1863                                     struct flow_cls_offload *flow)
1864 {
1865         int tc = tc_classid_to_hwtc(priv->netdev, flow->classid);
1866         struct hnae3_handle *h = hns3_get_handle(priv->netdev);
1867
1868         switch (flow->command) {
1869         case FLOW_CLS_REPLACE:
1870                 if (h->ae_algo->ops->add_cls_flower)
1871                         return h->ae_algo->ops->add_cls_flower(h, flow, tc);
1872                 break;
1873         case FLOW_CLS_DESTROY:
1874                 if (h->ae_algo->ops->del_cls_flower)
1875                         return h->ae_algo->ops->del_cls_flower(h, flow);
1876                 break;
1877         default:
1878                 break;
1879         }
1880
1881         return -EOPNOTSUPP;
1882 }
1883
1884 static int hns3_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
1885                                   void *cb_priv)
1886 {
1887         struct hns3_nic_priv *priv = cb_priv;
1888
1889         if (!tc_cls_can_offload_and_chain0(priv->netdev, type_data))
1890                 return -EOPNOTSUPP;
1891
1892         switch (type) {
1893         case TC_SETUP_CLSFLOWER:
1894                 return hns3_setup_tc_cls_flower(priv, type_data);
1895         default:
1896                 return -EOPNOTSUPP;
1897         }
1898 }
1899
1900 static LIST_HEAD(hns3_block_cb_list);
1901
1902 static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
1903                              void *type_data)
1904 {
1905         struct hns3_nic_priv *priv = netdev_priv(dev);
1906         int ret;
1907
1908         switch (type) {
1909         case TC_SETUP_QDISC_MQPRIO:
1910                 ret = hns3_setup_tc(dev, type_data);
1911                 break;
1912         case TC_SETUP_BLOCK:
1913                 ret = flow_block_cb_setup_simple(type_data,
1914                                                  &hns3_block_cb_list,
1915                                                  hns3_setup_tc_block_cb,
1916                                                  priv, priv, true);
1917                 break;
1918         default:
1919                 return -EOPNOTSUPP;
1920         }
1921
1922         return ret;
1923 }
1924
1925 static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1926                                 __be16 proto, u16 vid)
1927 {
1928         struct hnae3_handle *h = hns3_get_handle(netdev);
1929         int ret = -EIO;
1930
1931         if (h->ae_algo->ops->set_vlan_filter)
1932                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1933
1934         return ret;
1935 }
1936
1937 static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1938                                  __be16 proto, u16 vid)
1939 {
1940         struct hnae3_handle *h = hns3_get_handle(netdev);
1941         int ret = -EIO;
1942
1943         if (h->ae_algo->ops->set_vlan_filter)
1944                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1945
1946         return ret;
1947 }
1948
1949 static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1950                                 u8 qos, __be16 vlan_proto)
1951 {
1952         struct hnae3_handle *h = hns3_get_handle(netdev);
1953         int ret = -EIO;
1954
1955         netif_dbg(h, drv, netdev,
1956                   "set vf vlan: vf=%d, vlan=%u, qos=%u, vlan_proto=0x%x\n",
1957                   vf, vlan, qos, ntohs(vlan_proto));
1958
1959         if (h->ae_algo->ops->set_vf_vlan_filter)
1960                 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1961                                                           qos, vlan_proto);
1962
1963         return ret;
1964 }
1965
1966 static int hns3_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable)
1967 {
1968         struct hnae3_handle *handle = hns3_get_handle(netdev);
1969
1970         if (hns3_nic_resetting(netdev))
1971                 return -EBUSY;
1972
1973         if (!handle->ae_algo->ops->set_vf_spoofchk)
1974                 return -EOPNOTSUPP;
1975
1976         return handle->ae_algo->ops->set_vf_spoofchk(handle, vf, enable);
1977 }
1978
1979 static int hns3_set_vf_trust(struct net_device *netdev, int vf, bool enable)
1980 {
1981         struct hnae3_handle *handle = hns3_get_handle(netdev);
1982
1983         if (!handle->ae_algo->ops->set_vf_trust)
1984                 return -EOPNOTSUPP;
1985
1986         return handle->ae_algo->ops->set_vf_trust(handle, vf, enable);
1987 }
1988
1989 static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1990 {
1991         struct hnae3_handle *h = hns3_get_handle(netdev);
1992         int ret;
1993
1994         if (hns3_nic_resetting(netdev))
1995                 return -EBUSY;
1996
1997         if (!h->ae_algo->ops->set_mtu)
1998                 return -EOPNOTSUPP;
1999
2000         netif_dbg(h, drv, netdev,
2001                   "change mtu from %u to %d\n", netdev->mtu, new_mtu);
2002
2003         ret = h->ae_algo->ops->set_mtu(h, new_mtu);
2004         if (ret)
2005                 netdev_err(netdev, "failed to change MTU in hardware %d\n",
2006                            ret);
2007         else
2008                 netdev->mtu = new_mtu;
2009
2010         return ret;
2011 }
2012
2013 static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
2014 {
2015         struct hns3_nic_priv *priv = netdev_priv(ndev);
2016         struct hnae3_handle *h = hns3_get_handle(ndev);
2017         struct hns3_enet_ring *tx_ring;
2018         struct napi_struct *napi;
2019         int timeout_queue = 0;
2020         int hw_head, hw_tail;
2021         int fbd_num, fbd_oft;
2022         int ebd_num, ebd_oft;
2023         int bd_num, bd_err;
2024         int ring_en, tc;
2025         int i;
2026
2027         /* Find the stopped queue the same way the stack does */
2028         for (i = 0; i < ndev->num_tx_queues; i++) {
2029                 struct netdev_queue *q;
2030                 unsigned long trans_start;
2031
2032                 q = netdev_get_tx_queue(ndev, i);
2033                 trans_start = q->trans_start;
2034                 if (netif_xmit_stopped(q) &&
2035                     time_after(jiffies,
2036                                (trans_start + ndev->watchdog_timeo))) {
2037                         timeout_queue = i;
2038                         netdev_info(ndev, "queue state: 0x%lx, delta msecs: %u\n",
2039                                     q->state,
2040                                     jiffies_to_msecs(jiffies - trans_start));
2041                         break;
2042                 }
2043         }
2044
2045         if (i == ndev->num_tx_queues) {
2046                 netdev_info(ndev,
2047                             "no netdev TX timeout queue found, timeout count: %llu\n",
2048                             priv->tx_timeout_count);
2049                 return false;
2050         }
2051
2052         priv->tx_timeout_count++;
2053
2054         tx_ring = &priv->ring[timeout_queue];
2055         napi = &tx_ring->tqp_vector->napi;
2056
2057         netdev_info(ndev,
2058                     "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, napi state: %lu\n",
2059                     priv->tx_timeout_count, timeout_queue, tx_ring->next_to_use,
2060                     tx_ring->next_to_clean, napi->state);
2061
2062         netdev_info(ndev,
2063                     "tx_pkts: %llu, tx_bytes: %llu, sw_err_cnt: %llu, tx_pending: %d\n",
2064                     tx_ring->stats.tx_pkts, tx_ring->stats.tx_bytes,
2065                     tx_ring->stats.sw_err_cnt, tx_ring->pending_buf);
2066
2067         netdev_info(ndev,
2068                     "seg_pkt_cnt: %llu, tx_more: %llu, restart_queue: %llu, tx_busy: %llu\n",
2069                     tx_ring->stats.seg_pkt_cnt, tx_ring->stats.tx_more,
2070                     tx_ring->stats.restart_queue, tx_ring->stats.tx_busy);
2071
2072         /* When mac received many pause frames continuous, it's unable to send
2073          * packets, which may cause tx timeout
2074          */
2075         if (h->ae_algo->ops->get_mac_stats) {
2076                 struct hns3_mac_stats mac_stats;
2077
2078                 h->ae_algo->ops->get_mac_stats(h, &mac_stats);
2079                 netdev_info(ndev, "tx_pause_cnt: %llu, rx_pause_cnt: %llu\n",
2080                             mac_stats.tx_pause_cnt, mac_stats.rx_pause_cnt);
2081         }
2082
2083         hw_head = readl_relaxed(tx_ring->tqp->io_base +
2084                                 HNS3_RING_TX_RING_HEAD_REG);
2085         hw_tail = readl_relaxed(tx_ring->tqp->io_base +
2086                                 HNS3_RING_TX_RING_TAIL_REG);
2087         fbd_num = readl_relaxed(tx_ring->tqp->io_base +
2088                                 HNS3_RING_TX_RING_FBDNUM_REG);
2089         fbd_oft = readl_relaxed(tx_ring->tqp->io_base +
2090                                 HNS3_RING_TX_RING_OFFSET_REG);
2091         ebd_num = readl_relaxed(tx_ring->tqp->io_base +
2092                                 HNS3_RING_TX_RING_EBDNUM_REG);
2093         ebd_oft = readl_relaxed(tx_ring->tqp->io_base +
2094                                 HNS3_RING_TX_RING_EBD_OFFSET_REG);
2095         bd_num = readl_relaxed(tx_ring->tqp->io_base +
2096                                HNS3_RING_TX_RING_BD_NUM_REG);
2097         bd_err = readl_relaxed(tx_ring->tqp->io_base +
2098                                HNS3_RING_TX_RING_BD_ERR_REG);
2099         ring_en = readl_relaxed(tx_ring->tqp->io_base + HNS3_RING_EN_REG);
2100         tc = readl_relaxed(tx_ring->tqp->io_base + HNS3_RING_TX_RING_TC_REG);
2101
2102         netdev_info(ndev,
2103                     "BD_NUM: 0x%x HW_HEAD: 0x%x, HW_TAIL: 0x%x, BD_ERR: 0x%x, INT: 0x%x\n",
2104                     bd_num, hw_head, hw_tail, bd_err,
2105                     readl(tx_ring->tqp_vector->mask_addr));
2106         netdev_info(ndev,
2107                     "RING_EN: 0x%x, TC: 0x%x, FBD_NUM: 0x%x FBD_OFT: 0x%x, EBD_NUM: 0x%x, EBD_OFT: 0x%x\n",
2108                     ring_en, tc, fbd_num, fbd_oft, ebd_num, ebd_oft);
2109
2110         return true;
2111 }
2112
2113 static void hns3_nic_net_timeout(struct net_device *ndev, unsigned int txqueue)
2114 {
2115         struct hns3_nic_priv *priv = netdev_priv(ndev);
2116         struct hnae3_handle *h = priv->ae_handle;
2117
2118         if (!hns3_get_tx_timeo_queue_info(ndev))
2119                 return;
2120
2121         /* request the reset, and let the hclge to determine
2122          * which reset level should be done
2123          */
2124         if (h->ae_algo->ops->reset_event)
2125                 h->ae_algo->ops->reset_event(h->pdev, h);
2126 }
2127
2128 #ifdef CONFIG_RFS_ACCEL
2129 static int hns3_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
2130                               u16 rxq_index, u32 flow_id)
2131 {
2132         struct hnae3_handle *h = hns3_get_handle(dev);
2133         struct flow_keys fkeys;
2134
2135         if (!h->ae_algo->ops->add_arfs_entry)
2136                 return -EOPNOTSUPP;
2137
2138         if (skb->encapsulation)
2139                 return -EPROTONOSUPPORT;
2140
2141         if (!skb_flow_dissect_flow_keys(skb, &fkeys, 0))
2142                 return -EPROTONOSUPPORT;
2143
2144         if ((fkeys.basic.n_proto != htons(ETH_P_IP) &&
2145              fkeys.basic.n_proto != htons(ETH_P_IPV6)) ||
2146             (fkeys.basic.ip_proto != IPPROTO_TCP &&
2147              fkeys.basic.ip_proto != IPPROTO_UDP))
2148                 return -EPROTONOSUPPORT;
2149
2150         return h->ae_algo->ops->add_arfs_entry(h, rxq_index, flow_id, &fkeys);
2151 }
2152 #endif
2153
2154 static int hns3_nic_get_vf_config(struct net_device *ndev, int vf,
2155                                   struct ifla_vf_info *ivf)
2156 {
2157         struct hnae3_handle *h = hns3_get_handle(ndev);
2158
2159         if (!h->ae_algo->ops->get_vf_config)
2160                 return -EOPNOTSUPP;
2161
2162         return h->ae_algo->ops->get_vf_config(h, vf, ivf);
2163 }
2164
2165 static int hns3_nic_set_vf_link_state(struct net_device *ndev, int vf,
2166                                       int link_state)
2167 {
2168         struct hnae3_handle *h = hns3_get_handle(ndev);
2169
2170         if (!h->ae_algo->ops->set_vf_link_state)
2171                 return -EOPNOTSUPP;
2172
2173         return h->ae_algo->ops->set_vf_link_state(h, vf, link_state);
2174 }
2175
2176 static int hns3_nic_set_vf_rate(struct net_device *ndev, int vf,
2177                                 int min_tx_rate, int max_tx_rate)
2178 {
2179         struct hnae3_handle *h = hns3_get_handle(ndev);
2180
2181         if (!h->ae_algo->ops->set_vf_rate)
2182                 return -EOPNOTSUPP;
2183
2184         return h->ae_algo->ops->set_vf_rate(h, vf, min_tx_rate, max_tx_rate,
2185                                             false);
2186 }
2187
2188 static int hns3_nic_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac)
2189 {
2190         struct hnae3_handle *h = hns3_get_handle(netdev);
2191
2192         if (!h->ae_algo->ops->set_vf_mac)
2193                 return -EOPNOTSUPP;
2194
2195         if (is_multicast_ether_addr(mac)) {
2196                 netdev_err(netdev,
2197                            "Invalid MAC:%pM specified. Could not set MAC\n",
2198                            mac);
2199                 return -EINVAL;
2200         }
2201
2202         return h->ae_algo->ops->set_vf_mac(h, vf_id, mac);
2203 }
2204
2205 static const struct net_device_ops hns3_nic_netdev_ops = {
2206         .ndo_open               = hns3_nic_net_open,
2207         .ndo_stop               = hns3_nic_net_stop,
2208         .ndo_start_xmit         = hns3_nic_net_xmit,
2209         .ndo_tx_timeout         = hns3_nic_net_timeout,
2210         .ndo_set_mac_address    = hns3_nic_net_set_mac_address,
2211         .ndo_do_ioctl           = hns3_nic_do_ioctl,
2212         .ndo_change_mtu         = hns3_nic_change_mtu,
2213         .ndo_set_features       = hns3_nic_set_features,
2214         .ndo_features_check     = hns3_features_check,
2215         .ndo_get_stats64        = hns3_nic_get_stats64,
2216         .ndo_setup_tc           = hns3_nic_setup_tc,
2217         .ndo_set_rx_mode        = hns3_nic_set_rx_mode,
2218         .ndo_vlan_rx_add_vid    = hns3_vlan_rx_add_vid,
2219         .ndo_vlan_rx_kill_vid   = hns3_vlan_rx_kill_vid,
2220         .ndo_set_vf_vlan        = hns3_ndo_set_vf_vlan,
2221         .ndo_set_vf_spoofchk    = hns3_set_vf_spoofchk,
2222         .ndo_set_vf_trust       = hns3_set_vf_trust,
2223 #ifdef CONFIG_RFS_ACCEL
2224         .ndo_rx_flow_steer      = hns3_rx_flow_steer,
2225 #endif
2226         .ndo_get_vf_config      = hns3_nic_get_vf_config,
2227         .ndo_set_vf_link_state  = hns3_nic_set_vf_link_state,
2228         .ndo_set_vf_rate        = hns3_nic_set_vf_rate,
2229         .ndo_set_vf_mac         = hns3_nic_set_vf_mac,
2230 };
2231
2232 bool hns3_is_phys_func(struct pci_dev *pdev)
2233 {
2234         u32 dev_id = pdev->device;
2235
2236         switch (dev_id) {
2237         case HNAE3_DEV_ID_GE:
2238         case HNAE3_DEV_ID_25GE:
2239         case HNAE3_DEV_ID_25GE_RDMA:
2240         case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
2241         case HNAE3_DEV_ID_50GE_RDMA:
2242         case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
2243         case HNAE3_DEV_ID_100G_RDMA_MACSEC:
2244         case HNAE3_DEV_ID_200G_RDMA:
2245                 return true;
2246         case HNAE3_DEV_ID_VF:
2247         case HNAE3_DEV_ID_RDMA_DCB_PFC_VF:
2248                 return false;
2249         default:
2250                 dev_warn(&pdev->dev, "un-recognized pci device-id %u",
2251                          dev_id);
2252         }
2253
2254         return false;
2255 }
2256
2257 static void hns3_disable_sriov(struct pci_dev *pdev)
2258 {
2259         /* If our VFs are assigned we cannot shut down SR-IOV
2260          * without causing issues, so just leave the hardware
2261          * available but disabled
2262          */
2263         if (pci_vfs_assigned(pdev)) {
2264                 dev_warn(&pdev->dev,
2265                          "disabling driver while VFs are assigned\n");
2266                 return;
2267         }
2268
2269         pci_disable_sriov(pdev);
2270 }
2271
2272 /* hns3_probe - Device initialization routine
2273  * @pdev: PCI device information struct
2274  * @ent: entry in hns3_pci_tbl
2275  *
2276  * hns3_probe initializes a PF identified by a pci_dev structure.
2277  * The OS initialization, configuring of the PF private structure,
2278  * and a hardware reset occur.
2279  *
2280  * Returns 0 on success, negative on failure
2281  */
2282 static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2283 {
2284         struct hnae3_ae_dev *ae_dev;
2285         int ret;
2286
2287         ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev), GFP_KERNEL);
2288         if (!ae_dev)
2289                 return -ENOMEM;
2290
2291         ae_dev->pdev = pdev;
2292         ae_dev->flag = ent->driver_data;
2293         pci_set_drvdata(pdev, ae_dev);
2294
2295         ret = hnae3_register_ae_dev(ae_dev);
2296         if (ret)
2297                 pci_set_drvdata(pdev, NULL);
2298
2299         return ret;
2300 }
2301
2302 /* hns3_remove - Device removal routine
2303  * @pdev: PCI device information struct
2304  */
2305 static void hns3_remove(struct pci_dev *pdev)
2306 {
2307         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2308
2309         if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
2310                 hns3_disable_sriov(pdev);
2311
2312         hnae3_unregister_ae_dev(ae_dev);
2313         pci_set_drvdata(pdev, NULL);
2314 }
2315
2316 /**
2317  * hns3_pci_sriov_configure
2318  * @pdev: pointer to a pci_dev structure
2319  * @num_vfs: number of VFs to allocate
2320  *
2321  * Enable or change the number of VFs. Called when the user updates the number
2322  * of VFs in sysfs.
2323  **/
2324 static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
2325 {
2326         int ret;
2327
2328         if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
2329                 dev_warn(&pdev->dev, "Can not config SRIOV\n");
2330                 return -EINVAL;
2331         }
2332
2333         if (num_vfs) {
2334                 ret = pci_enable_sriov(pdev, num_vfs);
2335                 if (ret)
2336                         dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
2337                 else
2338                         return num_vfs;
2339         } else if (!pci_vfs_assigned(pdev)) {
2340                 pci_disable_sriov(pdev);
2341         } else {
2342                 dev_warn(&pdev->dev,
2343                          "Unable to free VFs because some are assigned to VMs.\n");
2344         }
2345
2346         return 0;
2347 }
2348
2349 static void hns3_shutdown(struct pci_dev *pdev)
2350 {
2351         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2352
2353         hnae3_unregister_ae_dev(ae_dev);
2354         pci_set_drvdata(pdev, NULL);
2355
2356         if (system_state == SYSTEM_POWER_OFF)
2357                 pci_set_power_state(pdev, PCI_D3hot);
2358 }
2359
2360 static int __maybe_unused hns3_suspend(struct device *dev)
2361 {
2362         struct hnae3_ae_dev *ae_dev = dev_get_drvdata(dev);
2363
2364         if (ae_dev && hns3_is_phys_func(ae_dev->pdev)) {
2365                 dev_info(dev, "Begin to suspend.\n");
2366                 if (ae_dev->ops && ae_dev->ops->reset_prepare)
2367                         ae_dev->ops->reset_prepare(ae_dev, HNAE3_FUNC_RESET);
2368         }
2369
2370         return 0;
2371 }
2372
2373 static int __maybe_unused hns3_resume(struct device *dev)
2374 {
2375         struct hnae3_ae_dev *ae_dev = dev_get_drvdata(dev);
2376
2377         if (ae_dev && hns3_is_phys_func(ae_dev->pdev)) {
2378                 dev_info(dev, "Begin to resume.\n");
2379                 if (ae_dev->ops && ae_dev->ops->reset_done)
2380                         ae_dev->ops->reset_done(ae_dev);
2381         }
2382
2383         return 0;
2384 }
2385
2386 static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev,
2387                                             pci_channel_state_t state)
2388 {
2389         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2390         pci_ers_result_t ret;
2391
2392         dev_info(&pdev->dev, "PCI error detected, state(=%u)!!\n", state);
2393
2394         if (state == pci_channel_io_perm_failure)
2395                 return PCI_ERS_RESULT_DISCONNECT;
2396
2397         if (!ae_dev || !ae_dev->ops) {
2398                 dev_err(&pdev->dev,
2399                         "Can't recover - error happened before device initialized\n");
2400                 return PCI_ERS_RESULT_NONE;
2401         }
2402
2403         if (ae_dev->ops->handle_hw_ras_error)
2404                 ret = ae_dev->ops->handle_hw_ras_error(ae_dev);
2405         else
2406                 return PCI_ERS_RESULT_NONE;
2407
2408         return ret;
2409 }
2410
2411 static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev)
2412 {
2413         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2414         const struct hnae3_ae_ops *ops;
2415         enum hnae3_reset_type reset_type;
2416         struct device *dev = &pdev->dev;
2417
2418         if (!ae_dev || !ae_dev->ops)
2419                 return PCI_ERS_RESULT_NONE;
2420
2421         ops = ae_dev->ops;
2422         /* request the reset */
2423         if (ops->reset_event && ops->get_reset_level &&
2424             ops->set_default_reset_request) {
2425                 if (ae_dev->hw_err_reset_req) {
2426                         reset_type = ops->get_reset_level(ae_dev,
2427                                                 &ae_dev->hw_err_reset_req);
2428                         ops->set_default_reset_request(ae_dev, reset_type);
2429                         dev_info(dev, "requesting reset due to PCI error\n");
2430                         ops->reset_event(pdev, NULL);
2431                 }
2432
2433                 return PCI_ERS_RESULT_RECOVERED;
2434         }
2435
2436         return PCI_ERS_RESULT_DISCONNECT;
2437 }
2438
2439 static void hns3_reset_prepare(struct pci_dev *pdev)
2440 {
2441         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2442
2443         dev_info(&pdev->dev, "FLR prepare\n");
2444         if (ae_dev && ae_dev->ops && ae_dev->ops->reset_prepare)
2445                 ae_dev->ops->reset_prepare(ae_dev, HNAE3_FLR_RESET);
2446 }
2447
2448 static void hns3_reset_done(struct pci_dev *pdev)
2449 {
2450         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2451
2452         dev_info(&pdev->dev, "FLR done\n");
2453         if (ae_dev && ae_dev->ops && ae_dev->ops->reset_done)
2454                 ae_dev->ops->reset_done(ae_dev);
2455 }
2456
2457 static const struct pci_error_handlers hns3_err_handler = {
2458         .error_detected = hns3_error_detected,
2459         .slot_reset     = hns3_slot_reset,
2460         .reset_prepare  = hns3_reset_prepare,
2461         .reset_done     = hns3_reset_done,
2462 };
2463
2464 static SIMPLE_DEV_PM_OPS(hns3_pm_ops, hns3_suspend, hns3_resume);
2465
2466 static struct pci_driver hns3_driver = {
2467         .name     = hns3_driver_name,
2468         .id_table = hns3_pci_tbl,
2469         .probe    = hns3_probe,
2470         .remove   = hns3_remove,
2471         .shutdown = hns3_shutdown,
2472         .driver.pm  = &hns3_pm_ops,
2473         .sriov_configure = hns3_pci_sriov_configure,
2474         .err_handler    = &hns3_err_handler,
2475 };
2476
2477 /* set default feature to hns3 */
2478 static void hns3_set_default_feature(struct net_device *netdev)
2479 {
2480         struct hnae3_handle *h = hns3_get_handle(netdev);
2481         struct pci_dev *pdev = h->pdev;
2482         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2483
2484         netdev->priv_flags |= IFF_UNICAST_FLT;
2485
2486         netdev->hw_enc_features |= NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
2487                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
2488                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
2489                 NETIF_F_SCTP_CRC | NETIF_F_TSO_MANGLEID | NETIF_F_FRAGLIST;
2490
2491         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
2492
2493         netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
2494                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
2495                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
2496                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
2497                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
2498                 NETIF_F_SCTP_CRC | NETIF_F_FRAGLIST;
2499
2500         netdev->vlan_features |= NETIF_F_RXCSUM |
2501                 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
2502                 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
2503                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
2504                 NETIF_F_SCTP_CRC | NETIF_F_FRAGLIST;
2505
2506         netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX |
2507                 NETIF_F_HW_VLAN_CTAG_RX |
2508                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
2509                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
2510                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
2511                 NETIF_F_SCTP_CRC | NETIF_F_FRAGLIST;
2512
2513         if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2) {
2514                 netdev->hw_features |= NETIF_F_GRO_HW;
2515                 netdev->features |= NETIF_F_GRO_HW;
2516
2517                 if (!(h->flags & HNAE3_SUPPORT_VF)) {
2518                         netdev->hw_features |= NETIF_F_NTUPLE;
2519                         netdev->features |= NETIF_F_NTUPLE;
2520                 }
2521         }
2522
2523         if (test_bit(HNAE3_DEV_SUPPORT_UDP_GSO_B, ae_dev->caps)) {
2524                 netdev->hw_features |= NETIF_F_GSO_UDP_L4;
2525                 netdev->features |= NETIF_F_GSO_UDP_L4;
2526                 netdev->vlan_features |= NETIF_F_GSO_UDP_L4;
2527                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_L4;
2528         }
2529
2530         if (test_bit(HNAE3_DEV_SUPPORT_HW_TX_CSUM_B, ae_dev->caps)) {
2531                 netdev->hw_features |= NETIF_F_HW_CSUM;
2532                 netdev->features |= NETIF_F_HW_CSUM;
2533                 netdev->vlan_features |= NETIF_F_HW_CSUM;
2534                 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
2535         } else {
2536                 netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2537                 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2538                 netdev->vlan_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2539                 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2540         }
2541
2542         if (test_bit(HNAE3_DEV_SUPPORT_UDP_TUNNEL_CSUM_B, ae_dev->caps)) {
2543                 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
2544                 netdev->features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
2545                 netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
2546                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
2547         }
2548
2549         if (test_bit(HNAE3_DEV_SUPPORT_FD_FORWARD_TC_B, ae_dev->caps)) {
2550                 netdev->hw_features |= NETIF_F_HW_TC;
2551                 netdev->features |= NETIF_F_HW_TC;
2552         }
2553 }
2554
2555 static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
2556                              struct hns3_desc_cb *cb)
2557 {
2558         unsigned int order = hns3_page_order(ring);
2559         struct page *p;
2560
2561         p = dev_alloc_pages(order);
2562         if (!p)
2563                 return -ENOMEM;
2564
2565         cb->priv = p;
2566         cb->page_offset = 0;
2567         cb->reuse_flag = 0;
2568         cb->buf  = page_address(p);
2569         cb->length = hns3_page_size(ring);
2570         cb->type = DESC_TYPE_PAGE;
2571         page_ref_add(p, USHRT_MAX - 1);
2572         cb->pagecnt_bias = USHRT_MAX;
2573
2574         return 0;
2575 }
2576
2577 static void hns3_free_buffer(struct hns3_enet_ring *ring,
2578                              struct hns3_desc_cb *cb, int budget)
2579 {
2580         if (cb->type == DESC_TYPE_SKB)
2581                 napi_consume_skb(cb->priv, budget);
2582         else if (!HNAE3_IS_TX_RING(ring) && cb->pagecnt_bias)
2583                 __page_frag_cache_drain(cb->priv, cb->pagecnt_bias);
2584         memset(cb, 0, sizeof(*cb));
2585 }
2586
2587 static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
2588 {
2589         cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
2590                                cb->length, ring_to_dma_dir(ring));
2591
2592         if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma)))
2593                 return -EIO;
2594
2595         return 0;
2596 }
2597
2598 static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
2599                               struct hns3_desc_cb *cb)
2600 {
2601         if (cb->type == DESC_TYPE_SKB || cb->type == DESC_TYPE_FRAGLIST_SKB)
2602                 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
2603                                  ring_to_dma_dir(ring));
2604         else if (cb->length)
2605                 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
2606                                ring_to_dma_dir(ring));
2607 }
2608
2609 static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
2610 {
2611         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
2612         ring->desc[i].addr = 0;
2613 }
2614
2615 static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i,
2616                                     int budget)
2617 {
2618         struct hns3_desc_cb *cb = &ring->desc_cb[i];
2619
2620         if (!ring->desc_cb[i].dma)
2621                 return;
2622
2623         hns3_buffer_detach(ring, i);
2624         hns3_free_buffer(ring, cb, budget);
2625 }
2626
2627 static void hns3_free_buffers(struct hns3_enet_ring *ring)
2628 {
2629         int i;
2630
2631         for (i = 0; i < ring->desc_num; i++)
2632                 hns3_free_buffer_detach(ring, i, 0);
2633 }
2634
2635 /* free desc along with its attached buffer */
2636 static void hns3_free_desc(struct hns3_enet_ring *ring)
2637 {
2638         int size = ring->desc_num * sizeof(ring->desc[0]);
2639
2640         hns3_free_buffers(ring);
2641
2642         if (ring->desc) {
2643                 dma_free_coherent(ring_to_dev(ring), size,
2644                                   ring->desc, ring->desc_dma_addr);
2645                 ring->desc = NULL;
2646         }
2647 }
2648
2649 static int hns3_alloc_desc(struct hns3_enet_ring *ring)
2650 {
2651         int size = ring->desc_num * sizeof(ring->desc[0]);
2652
2653         ring->desc = dma_alloc_coherent(ring_to_dev(ring), size,
2654                                         &ring->desc_dma_addr, GFP_KERNEL);
2655         if (!ring->desc)
2656                 return -ENOMEM;
2657
2658         return 0;
2659 }
2660
2661 static int hns3_alloc_and_map_buffer(struct hns3_enet_ring *ring,
2662                                    struct hns3_desc_cb *cb)
2663 {
2664         int ret;
2665
2666         ret = hns3_alloc_buffer(ring, cb);
2667         if (ret)
2668                 goto out;
2669
2670         ret = hns3_map_buffer(ring, cb);
2671         if (ret)
2672                 goto out_with_buf;
2673
2674         return 0;
2675
2676 out_with_buf:
2677         hns3_free_buffer(ring, cb, 0);
2678 out:
2679         return ret;
2680 }
2681
2682 static int hns3_alloc_and_attach_buffer(struct hns3_enet_ring *ring, int i)
2683 {
2684         int ret = hns3_alloc_and_map_buffer(ring, &ring->desc_cb[i]);
2685
2686         if (ret)
2687                 return ret;
2688
2689         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2690
2691         return 0;
2692 }
2693
2694 /* Allocate memory for raw pkg, and map with dma */
2695 static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
2696 {
2697         int i, j, ret;
2698
2699         for (i = 0; i < ring->desc_num; i++) {
2700                 ret = hns3_alloc_and_attach_buffer(ring, i);
2701                 if (ret)
2702                         goto out_buffer_fail;
2703         }
2704
2705         return 0;
2706
2707 out_buffer_fail:
2708         for (j = i - 1; j >= 0; j--)
2709                 hns3_free_buffer_detach(ring, j, 0);
2710         return ret;
2711 }
2712
2713 /* detach a in-used buffer and replace with a reserved one */
2714 static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
2715                                 struct hns3_desc_cb *res_cb)
2716 {
2717         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
2718         ring->desc_cb[i] = *res_cb;
2719         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2720         ring->desc[i].rx.bd_base_info = 0;
2721 }
2722
2723 static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
2724 {
2725         ring->desc_cb[i].reuse_flag = 0;
2726         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma +
2727                                          ring->desc_cb[i].page_offset);
2728         ring->desc[i].rx.bd_base_info = 0;
2729
2730         dma_sync_single_for_device(ring_to_dev(ring),
2731                         ring->desc_cb[i].dma + ring->desc_cb[i].page_offset,
2732                         hns3_buf_size(ring),
2733                         DMA_FROM_DEVICE);
2734 }
2735
2736 static bool hns3_nic_reclaim_desc(struct hns3_enet_ring *ring,
2737                                   int *bytes, int *pkts, int budget)
2738 {
2739         /* pair with ring->last_to_use update in hns3_tx_doorbell(),
2740          * smp_store_release() is not used in hns3_tx_doorbell() because
2741          * the doorbell operation already have the needed barrier operation.
2742          */
2743         int ltu = smp_load_acquire(&ring->last_to_use);
2744         int ntc = ring->next_to_clean;
2745         struct hns3_desc_cb *desc_cb;
2746         bool reclaimed = false;
2747         struct hns3_desc *desc;
2748
2749         while (ltu != ntc) {
2750                 desc = &ring->desc[ntc];
2751
2752                 if (le16_to_cpu(desc->tx.bdtp_fe_sc_vld_ra_ri) &
2753                                 BIT(HNS3_TXD_VLD_B))
2754                         break;
2755
2756                 desc_cb = &ring->desc_cb[ntc];
2757
2758                 if (desc_cb->type == DESC_TYPE_SKB) {
2759                         (*pkts)++;
2760                         (*bytes) += desc_cb->send_bytes;
2761                 }
2762
2763                 /* desc_cb will be cleaned, after hnae3_free_buffer_detach */
2764                 hns3_free_buffer_detach(ring, ntc, budget);
2765
2766                 if (++ntc == ring->desc_num)
2767                         ntc = 0;
2768
2769                 /* Issue prefetch for next Tx descriptor */
2770                 prefetch(&ring->desc_cb[ntc]);
2771                 reclaimed = true;
2772         }
2773
2774         if (unlikely(!reclaimed))
2775                 return false;
2776
2777         /* This smp_store_release() pairs with smp_load_acquire() in
2778          * ring_space called by hns3_nic_net_xmit.
2779          */
2780         smp_store_release(&ring->next_to_clean, ntc);
2781         return true;
2782 }
2783
2784 void hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)
2785 {
2786         struct net_device *netdev = ring_to_netdev(ring);
2787         struct hns3_nic_priv *priv = netdev_priv(netdev);
2788         struct netdev_queue *dev_queue;
2789         int bytes, pkts;
2790
2791         bytes = 0;
2792         pkts = 0;
2793
2794         if (unlikely(!hns3_nic_reclaim_desc(ring, &bytes, &pkts, budget)))
2795                 return;
2796
2797         ring->tqp_vector->tx_group.total_bytes += bytes;
2798         ring->tqp_vector->tx_group.total_packets += pkts;
2799
2800         u64_stats_update_begin(&ring->syncp);
2801         ring->stats.tx_bytes += bytes;
2802         ring->stats.tx_pkts += pkts;
2803         u64_stats_update_end(&ring->syncp);
2804
2805         dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
2806         netdev_tx_completed_queue(dev_queue, pkts, bytes);
2807
2808         if (unlikely(netif_carrier_ok(netdev) &&
2809                      ring_space(ring) > HNS3_MAX_TSO_BD_NUM)) {
2810                 /* Make sure that anybody stopping the queue after this
2811                  * sees the new next_to_clean.
2812                  */
2813                 smp_mb();
2814                 if (netif_tx_queue_stopped(dev_queue) &&
2815                     !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
2816                         netif_tx_wake_queue(dev_queue);
2817                         ring->stats.restart_queue++;
2818                 }
2819         }
2820 }
2821
2822 static int hns3_desc_unused(struct hns3_enet_ring *ring)
2823 {
2824         int ntc = ring->next_to_clean;
2825         int ntu = ring->next_to_use;
2826
2827         return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
2828 }
2829
2830 static void hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring,
2831                                       int cleand_count)
2832 {
2833         struct hns3_desc_cb *desc_cb;
2834         struct hns3_desc_cb res_cbs;
2835         int i, ret;
2836
2837         for (i = 0; i < cleand_count; i++) {
2838                 desc_cb = &ring->desc_cb[ring->next_to_use];
2839                 if (desc_cb->reuse_flag) {
2840                         u64_stats_update_begin(&ring->syncp);
2841                         ring->stats.reuse_pg_cnt++;
2842                         u64_stats_update_end(&ring->syncp);
2843
2844                         hns3_reuse_buffer(ring, ring->next_to_use);
2845                 } else {
2846                         ret = hns3_alloc_and_map_buffer(ring, &res_cbs);
2847                         if (ret) {
2848                                 u64_stats_update_begin(&ring->syncp);
2849                                 ring->stats.sw_err_cnt++;
2850                                 u64_stats_update_end(&ring->syncp);
2851
2852                                 hns3_rl_err(ring_to_netdev(ring),
2853                                             "alloc rx buffer failed: %d\n",
2854                                             ret);
2855                                 break;
2856                         }
2857                         hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
2858
2859                         u64_stats_update_begin(&ring->syncp);
2860                         ring->stats.non_reuse_pg++;
2861                         u64_stats_update_end(&ring->syncp);
2862                 }
2863
2864                 ring_ptr_move_fw(ring, next_to_use);
2865         }
2866
2867         writel(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
2868 }
2869
2870 static bool hns3_can_reuse_page(struct hns3_desc_cb *cb)
2871 {
2872         return (page_count(cb->priv) - cb->pagecnt_bias) == 1;
2873 }
2874
2875 static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2876                                 struct hns3_enet_ring *ring, int pull_len,
2877                                 struct hns3_desc_cb *desc_cb)
2878 {
2879         struct hns3_desc *desc = &ring->desc[ring->next_to_clean];
2880         int size = le16_to_cpu(desc->rx.size);
2881         u32 truesize = hns3_buf_size(ring);
2882
2883         desc_cb->pagecnt_bias--;
2884         skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
2885                         size - pull_len, truesize);
2886
2887         /* Avoid re-using remote and pfmemalloc pages, or the stack is still
2888          * using the page when page_offset rollback to zero, flag default
2889          * unreuse
2890          */
2891         if (!dev_page_is_reusable(desc_cb->priv) ||
2892             (!desc_cb->page_offset && !hns3_can_reuse_page(desc_cb))) {
2893                 __page_frag_cache_drain(desc_cb->priv, desc_cb->pagecnt_bias);
2894                 return;
2895         }
2896
2897         /* Move offset up to the next cache line */
2898         desc_cb->page_offset += truesize;
2899
2900         if (desc_cb->page_offset + truesize <= hns3_page_size(ring)) {
2901                 desc_cb->reuse_flag = 1;
2902         } else if (hns3_can_reuse_page(desc_cb)) {
2903                 desc_cb->reuse_flag = 1;
2904                 desc_cb->page_offset = 0;
2905         } else if (desc_cb->pagecnt_bias) {
2906                 __page_frag_cache_drain(desc_cb->priv, desc_cb->pagecnt_bias);
2907                 return;
2908         }
2909
2910         if (unlikely(!desc_cb->pagecnt_bias)) {
2911                 page_ref_add(desc_cb->priv, USHRT_MAX);
2912                 desc_cb->pagecnt_bias = USHRT_MAX;
2913         }
2914 }
2915
2916 static int hns3_gro_complete(struct sk_buff *skb, u32 l234info)
2917 {
2918         __be16 type = skb->protocol;
2919         struct tcphdr *th;
2920         int depth = 0;
2921
2922         while (eth_type_vlan(type)) {
2923                 struct vlan_hdr *vh;
2924
2925                 if ((depth + VLAN_HLEN) > skb_headlen(skb))
2926                         return -EFAULT;
2927
2928                 vh = (struct vlan_hdr *)(skb->data + depth);
2929                 type = vh->h_vlan_encapsulated_proto;
2930                 depth += VLAN_HLEN;
2931         }
2932
2933         skb_set_network_header(skb, depth);
2934
2935         if (type == htons(ETH_P_IP)) {
2936                 const struct iphdr *iph = ip_hdr(skb);
2937
2938                 depth += sizeof(struct iphdr);
2939                 skb_set_transport_header(skb, depth);
2940                 th = tcp_hdr(skb);
2941                 th->check = ~tcp_v4_check(skb->len - depth, iph->saddr,
2942                                           iph->daddr, 0);
2943         } else if (type == htons(ETH_P_IPV6)) {
2944                 const struct ipv6hdr *iph = ipv6_hdr(skb);
2945
2946                 depth += sizeof(struct ipv6hdr);
2947                 skb_set_transport_header(skb, depth);
2948                 th = tcp_hdr(skb);
2949                 th->check = ~tcp_v6_check(skb->len - depth, &iph->saddr,
2950                                           &iph->daddr, 0);
2951         } else {
2952                 hns3_rl_err(skb->dev,
2953                             "Error: FW GRO supports only IPv4/IPv6, not 0x%04x, depth: %d\n",
2954                             be16_to_cpu(type), depth);
2955                 return -EFAULT;
2956         }
2957
2958         skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
2959         if (th->cwr)
2960                 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
2961
2962         if (l234info & BIT(HNS3_RXD_GRO_FIXID_B))
2963                 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_FIXEDID;
2964
2965         skb->csum_start = (unsigned char *)th - skb->head;
2966         skb->csum_offset = offsetof(struct tcphdr, check);
2967         skb->ip_summed = CHECKSUM_PARTIAL;
2968
2969         trace_hns3_gro(skb);
2970
2971         return 0;
2972 }
2973
2974 static void hns3_checksum_complete(struct hns3_enet_ring *ring,
2975                                    struct sk_buff *skb, u32 l234info)
2976 {
2977         u32 lo, hi;
2978
2979         u64_stats_update_begin(&ring->syncp);
2980         ring->stats.csum_complete++;
2981         u64_stats_update_end(&ring->syncp);
2982         skb->ip_summed = CHECKSUM_COMPLETE;
2983         lo = hnae3_get_field(l234info, HNS3_RXD_L2_CSUM_L_M,
2984                              HNS3_RXD_L2_CSUM_L_S);
2985         hi = hnae3_get_field(l234info, HNS3_RXD_L2_CSUM_H_M,
2986                              HNS3_RXD_L2_CSUM_H_S);
2987         skb->csum = csum_unfold((__force __sum16)(lo | hi << 8));
2988 }
2989
2990 static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2991                              u32 l234info, u32 bd_base_info, u32 ol_info)
2992 {
2993         struct net_device *netdev = ring_to_netdev(ring);
2994         int l3_type, l4_type;
2995         int ol4_type;
2996
2997         skb->ip_summed = CHECKSUM_NONE;
2998
2999         skb_checksum_none_assert(skb);
3000
3001         if (!(netdev->features & NETIF_F_RXCSUM))
3002                 return;
3003
3004         if (l234info & BIT(HNS3_RXD_L2_CSUM_B)) {
3005                 hns3_checksum_complete(ring, skb, l234info);
3006                 return;
3007         }
3008
3009         /* check if hardware has done checksum */
3010         if (!(bd_base_info & BIT(HNS3_RXD_L3L4P_B)))
3011                 return;
3012
3013         if (unlikely(l234info & (BIT(HNS3_RXD_L3E_B) | BIT(HNS3_RXD_L4E_B) |
3014                                  BIT(HNS3_RXD_OL3E_B) |
3015                                  BIT(HNS3_RXD_OL4E_B)))) {
3016                 u64_stats_update_begin(&ring->syncp);
3017                 ring->stats.l3l4_csum_err++;
3018                 u64_stats_update_end(&ring->syncp);
3019
3020                 return;
3021         }
3022
3023         ol4_type = hnae3_get_field(ol_info, HNS3_RXD_OL4ID_M,
3024                                    HNS3_RXD_OL4ID_S);
3025         switch (ol4_type) {
3026         case HNS3_OL4_TYPE_MAC_IN_UDP:
3027         case HNS3_OL4_TYPE_NVGRE:
3028                 skb->csum_level = 1;
3029                 fallthrough;
3030         case HNS3_OL4_TYPE_NO_TUN:
3031                 l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
3032                                           HNS3_RXD_L3ID_S);
3033                 l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,
3034                                           HNS3_RXD_L4ID_S);
3035                 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
3036                 if ((l3_type == HNS3_L3_TYPE_IPV4 ||
3037                      l3_type == HNS3_L3_TYPE_IPV6) &&
3038                     (l4_type == HNS3_L4_TYPE_UDP ||
3039                      l4_type == HNS3_L4_TYPE_TCP ||
3040                      l4_type == HNS3_L4_TYPE_SCTP))
3041                         skb->ip_summed = CHECKSUM_UNNECESSARY;
3042                 break;
3043         default:
3044                 break;
3045         }
3046 }
3047
3048 static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
3049 {
3050         if (skb_has_frag_list(skb))
3051                 napi_gro_flush(&ring->tqp_vector->napi, false);
3052
3053         napi_gro_receive(&ring->tqp_vector->napi, skb);
3054 }
3055
3056 static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
3057                                 struct hns3_desc *desc, u32 l234info,
3058                                 u16 *vlan_tag)
3059 {
3060         struct hnae3_handle *handle = ring->tqp->handle;
3061         struct pci_dev *pdev = ring->tqp->handle->pdev;
3062         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
3063
3064         if (unlikely(ae_dev->dev_version < HNAE3_DEVICE_VERSION_V2)) {
3065                 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
3066                 if (!(*vlan_tag & VLAN_VID_MASK))
3067                         *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
3068
3069                 return (*vlan_tag != 0);
3070         }
3071
3072 #define HNS3_STRP_OUTER_VLAN    0x1
3073 #define HNS3_STRP_INNER_VLAN    0x2
3074 #define HNS3_STRP_BOTH          0x3
3075
3076         /* Hardware always insert VLAN tag into RX descriptor when
3077          * remove the tag from packet, driver needs to determine
3078          * reporting which tag to stack.
3079          */
3080         switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
3081                                 HNS3_RXD_STRP_TAGP_S)) {
3082         case HNS3_STRP_OUTER_VLAN:
3083                 if (handle->port_base_vlan_state !=
3084                                 HNAE3_PORT_BASE_VLAN_DISABLE)
3085                         return false;
3086
3087                 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
3088                 return true;
3089         case HNS3_STRP_INNER_VLAN:
3090                 if (handle->port_base_vlan_state !=
3091                                 HNAE3_PORT_BASE_VLAN_DISABLE)
3092                         return false;
3093
3094                 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
3095                 return true;
3096         case HNS3_STRP_BOTH:
3097                 if (handle->port_base_vlan_state ==
3098                                 HNAE3_PORT_BASE_VLAN_DISABLE)
3099                         *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
3100                 else
3101                         *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
3102
3103                 return true;
3104         default:
3105                 return false;
3106         }
3107 }
3108
3109 static void hns3_rx_ring_move_fw(struct hns3_enet_ring *ring)
3110 {
3111         ring->desc[ring->next_to_clean].rx.bd_base_info &=
3112                 cpu_to_le32(~BIT(HNS3_RXD_VLD_B));
3113         ring->next_to_clean += 1;
3114
3115         if (unlikely(ring->next_to_clean == ring->desc_num))
3116                 ring->next_to_clean = 0;
3117 }
3118
3119 static int hns3_alloc_skb(struct hns3_enet_ring *ring, unsigned int length,
3120                           unsigned char *va)
3121 {
3122         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
3123         struct net_device *netdev = ring_to_netdev(ring);
3124         struct sk_buff *skb;
3125
3126         ring->skb = napi_alloc_skb(&ring->tqp_vector->napi, HNS3_RX_HEAD_SIZE);
3127         skb = ring->skb;
3128         if (unlikely(!skb)) {
3129                 hns3_rl_err(netdev, "alloc rx skb fail\n");
3130
3131                 u64_stats_update_begin(&ring->syncp);
3132                 ring->stats.sw_err_cnt++;
3133                 u64_stats_update_end(&ring->syncp);
3134
3135                 return -ENOMEM;
3136         }
3137
3138         trace_hns3_rx_desc(ring);
3139         prefetchw(skb->data);
3140
3141         ring->pending_buf = 1;
3142         ring->frag_num = 0;
3143         ring->tail_skb = NULL;
3144         if (length <= HNS3_RX_HEAD_SIZE) {
3145                 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
3146
3147                 /* We can reuse buffer as-is, just make sure it is reusable */
3148                 if (dev_page_is_reusable(desc_cb->priv))
3149                         desc_cb->reuse_flag = 1;
3150                 else /* This page cannot be reused so discard it */
3151                         __page_frag_cache_drain(desc_cb->priv,
3152                                                 desc_cb->pagecnt_bias);
3153
3154                 hns3_rx_ring_move_fw(ring);
3155                 return 0;
3156         }
3157         u64_stats_update_begin(&ring->syncp);
3158         ring->stats.seg_pkt_cnt++;
3159         u64_stats_update_end(&ring->syncp);
3160
3161         ring->pull_len = eth_get_headlen(netdev, va, HNS3_RX_HEAD_SIZE);
3162         __skb_put(skb, ring->pull_len);
3163         hns3_nic_reuse_page(skb, ring->frag_num++, ring, ring->pull_len,
3164                             desc_cb);
3165         hns3_rx_ring_move_fw(ring);
3166
3167         return 0;
3168 }
3169
3170 static int hns3_add_frag(struct hns3_enet_ring *ring)
3171 {
3172         struct sk_buff *skb = ring->skb;
3173         struct sk_buff *head_skb = skb;
3174         struct sk_buff *new_skb;
3175         struct hns3_desc_cb *desc_cb;
3176         struct hns3_desc *desc;
3177         u32 bd_base_info;
3178
3179         do {
3180                 desc = &ring->desc[ring->next_to_clean];
3181                 desc_cb = &ring->desc_cb[ring->next_to_clean];
3182                 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
3183                 /* make sure HW write desc complete */
3184                 dma_rmb();
3185                 if (!(bd_base_info & BIT(HNS3_RXD_VLD_B)))
3186                         return -ENXIO;
3187
3188                 if (unlikely(ring->frag_num >= MAX_SKB_FRAGS)) {
3189                         new_skb = napi_alloc_skb(&ring->tqp_vector->napi, 0);
3190                         if (unlikely(!new_skb)) {
3191                                 hns3_rl_err(ring_to_netdev(ring),
3192                                             "alloc rx fraglist skb fail\n");
3193                                 return -ENXIO;
3194                         }
3195                         ring->frag_num = 0;
3196
3197                         if (ring->tail_skb) {
3198                                 ring->tail_skb->next = new_skb;
3199                                 ring->tail_skb = new_skb;
3200                         } else {
3201                                 skb_shinfo(skb)->frag_list = new_skb;
3202                                 ring->tail_skb = new_skb;
3203                         }
3204                 }
3205
3206                 if (ring->tail_skb) {
3207                         head_skb->truesize += hns3_buf_size(ring);
3208                         head_skb->data_len += le16_to_cpu(desc->rx.size);
3209                         head_skb->len += le16_to_cpu(desc->rx.size);
3210                         skb = ring->tail_skb;
3211                 }
3212
3213                 dma_sync_single_for_cpu(ring_to_dev(ring),
3214                                 desc_cb->dma + desc_cb->page_offset,
3215                                 hns3_buf_size(ring),
3216                                 DMA_FROM_DEVICE);
3217
3218                 hns3_nic_reuse_page(skb, ring->frag_num++, ring, 0, desc_cb);
3219                 trace_hns3_rx_desc(ring);
3220                 hns3_rx_ring_move_fw(ring);
3221                 ring->pending_buf++;
3222         } while (!(bd_base_info & BIT(HNS3_RXD_FE_B)));
3223
3224         return 0;
3225 }
3226
3227 static int hns3_set_gro_and_checksum(struct hns3_enet_ring *ring,
3228                                      struct sk_buff *skb, u32 l234info,
3229                                      u32 bd_base_info, u32 ol_info)
3230 {
3231         u32 l3_type;
3232
3233         skb_shinfo(skb)->gso_size = hnae3_get_field(bd_base_info,
3234                                                     HNS3_RXD_GRO_SIZE_M,
3235                                                     HNS3_RXD_GRO_SIZE_S);
3236         /* if there is no HW GRO, do not set gro params */
3237         if (!skb_shinfo(skb)->gso_size) {
3238                 hns3_rx_checksum(ring, skb, l234info, bd_base_info, ol_info);
3239                 return 0;
3240         }
3241
3242         NAPI_GRO_CB(skb)->count = hnae3_get_field(l234info,
3243                                                   HNS3_RXD_GRO_COUNT_M,
3244                                                   HNS3_RXD_GRO_COUNT_S);
3245
3246         l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S);
3247         if (l3_type == HNS3_L3_TYPE_IPV4)
3248                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
3249         else if (l3_type == HNS3_L3_TYPE_IPV6)
3250                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
3251         else
3252                 return -EFAULT;
3253
3254         return  hns3_gro_complete(skb, l234info);
3255 }
3256
3257 static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring,
3258                                      struct sk_buff *skb, u32 rss_hash)
3259 {
3260         struct hnae3_handle *handle = ring->tqp->handle;
3261         enum pkt_hash_types rss_type;
3262
3263         if (rss_hash)
3264                 rss_type = handle->kinfo.rss_type;
3265         else
3266                 rss_type = PKT_HASH_TYPE_NONE;
3267
3268         skb_set_hash(skb, rss_hash, rss_type);
3269 }
3270
3271 static int hns3_handle_bdinfo(struct hns3_enet_ring *ring, struct sk_buff *skb)
3272 {
3273         struct net_device *netdev = ring_to_netdev(ring);
3274         enum hns3_pkt_l2t_type l2_frame_type;
3275         u32 bd_base_info, l234info, ol_info;
3276         struct hns3_desc *desc;
3277         unsigned int len;
3278         int pre_ntc, ret;
3279
3280         /* bdinfo handled below is only valid on the last BD of the
3281          * current packet, and ring->next_to_clean indicates the first
3282          * descriptor of next packet, so need - 1 below.
3283          */
3284         pre_ntc = ring->next_to_clean ? (ring->next_to_clean - 1) :
3285                                         (ring->desc_num - 1);
3286         desc = &ring->desc[pre_ntc];
3287         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
3288         l234info = le32_to_cpu(desc->rx.l234_info);
3289         ol_info = le32_to_cpu(desc->rx.ol_info);
3290
3291         /* Based on hw strategy, the tag offloaded will be stored at
3292          * ot_vlan_tag in two layer tag case, and stored at vlan_tag
3293          * in one layer tag case.
3294          */
3295         if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
3296                 u16 vlan_tag;
3297
3298                 if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag))
3299                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
3300                                                vlan_tag);
3301         }
3302
3303         if (unlikely(!desc->rx.pkt_len || (l234info & (BIT(HNS3_RXD_TRUNCAT_B) |
3304                                   BIT(HNS3_RXD_L2E_B))))) {
3305                 u64_stats_update_begin(&ring->syncp);
3306                 if (l234info & BIT(HNS3_RXD_L2E_B))
3307                         ring->stats.l2_err++;
3308                 else
3309                         ring->stats.err_pkt_len++;
3310                 u64_stats_update_end(&ring->syncp);
3311
3312                 return -EFAULT;
3313         }
3314
3315         len = skb->len;
3316
3317         /* Do update ip stack process */
3318         skb->protocol = eth_type_trans(skb, netdev);
3319
3320         /* This is needed in order to enable forwarding support */
3321         ret = hns3_set_gro_and_checksum(ring, skb, l234info,
3322                                         bd_base_info, ol_info);
3323         if (unlikely(ret)) {
3324                 u64_stats_update_begin(&ring->syncp);
3325                 ring->stats.rx_err_cnt++;
3326                 u64_stats_update_end(&ring->syncp);
3327                 return ret;
3328         }
3329
3330         l2_frame_type = hnae3_get_field(l234info, HNS3_RXD_DMAC_M,
3331                                         HNS3_RXD_DMAC_S);
3332
3333         u64_stats_update_begin(&ring->syncp);
3334         ring->stats.rx_pkts++;
3335         ring->stats.rx_bytes += len;
3336
3337         if (l2_frame_type == HNS3_L2_TYPE_MULTICAST)
3338                 ring->stats.rx_multicast++;
3339
3340         u64_stats_update_end(&ring->syncp);
3341
3342         ring->tqp_vector->rx_group.total_bytes += len;
3343
3344         hns3_set_rx_skb_rss_type(ring, skb, le32_to_cpu(desc->rx.rss_hash));
3345         return 0;
3346 }
3347
3348 static int hns3_handle_rx_bd(struct hns3_enet_ring *ring)
3349 {
3350         struct sk_buff *skb = ring->skb;
3351         struct hns3_desc_cb *desc_cb;
3352         struct hns3_desc *desc;
3353         unsigned int length;
3354         u32 bd_base_info;
3355         int ret;
3356
3357         desc = &ring->desc[ring->next_to_clean];
3358         desc_cb = &ring->desc_cb[ring->next_to_clean];
3359
3360         prefetch(desc);
3361
3362         if (!skb) {
3363                 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
3364                 /* Check valid BD */
3365                 if (unlikely(!(bd_base_info & BIT(HNS3_RXD_VLD_B))))
3366                         return -ENXIO;
3367
3368                 dma_rmb();
3369                 length = le16_to_cpu(desc->rx.size);
3370
3371                 ring->va = desc_cb->buf + desc_cb->page_offset;
3372
3373                 dma_sync_single_for_cpu(ring_to_dev(ring),
3374                                 desc_cb->dma + desc_cb->page_offset,
3375                                 hns3_buf_size(ring),
3376                                 DMA_FROM_DEVICE);
3377
3378                 /* Prefetch first cache line of first page.
3379                  * Idea is to cache few bytes of the header of the packet.
3380                  * Our L1 Cache line size is 64B so need to prefetch twice to make
3381                  * it 128B. But in actual we can have greater size of caches with
3382                  * 128B Level 1 cache lines. In such a case, single fetch would
3383                  * suffice to cache in the relevant part of the header.
3384                  */
3385                 net_prefetch(ring->va);
3386
3387                 ret = hns3_alloc_skb(ring, length, ring->va);
3388                 skb = ring->skb;
3389
3390                 if (ret < 0) /* alloc buffer fail */
3391                         return ret;
3392                 if (!(bd_base_info & BIT(HNS3_RXD_FE_B))) { /* need add frag */
3393                         ret = hns3_add_frag(ring);
3394                         if (ret)
3395                                 return ret;
3396                 }
3397         } else {
3398                 ret = hns3_add_frag(ring);
3399                 if (ret)
3400                         return ret;
3401         }
3402
3403         /* As the head data may be changed when GRO enable, copy
3404          * the head data in after other data rx completed
3405          */
3406         if (skb->len > HNS3_RX_HEAD_SIZE)
3407                 memcpy(skb->data, ring->va,
3408                        ALIGN(ring->pull_len, sizeof(long)));
3409
3410         ret = hns3_handle_bdinfo(ring, skb);
3411         if (unlikely(ret)) {
3412                 dev_kfree_skb_any(skb);
3413                 return ret;
3414         }
3415
3416         skb_record_rx_queue(skb, ring->tqp->tqp_index);
3417         return 0;
3418 }
3419
3420 int hns3_clean_rx_ring(struct hns3_enet_ring *ring, int budget,
3421                        void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
3422 {
3423 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
3424         int unused_count = hns3_desc_unused(ring);
3425         int recv_pkts = 0;
3426         int err;
3427
3428         unused_count -= ring->pending_buf;
3429
3430         while (recv_pkts < budget) {
3431                 /* Reuse or realloc buffers */
3432                 if (unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
3433                         hns3_nic_alloc_rx_buffers(ring, unused_count);
3434                         unused_count = hns3_desc_unused(ring) -
3435                                         ring->pending_buf;
3436                 }
3437
3438                 /* Poll one pkt */
3439                 err = hns3_handle_rx_bd(ring);
3440                 /* Do not get FE for the packet or failed to alloc skb */
3441                 if (unlikely(!ring->skb || err == -ENXIO)) {
3442                         goto out;
3443                 } else if (likely(!err)) {
3444                         rx_fn(ring, ring->skb);
3445                         recv_pkts++;
3446                 }
3447
3448                 unused_count += ring->pending_buf;
3449                 ring->skb = NULL;
3450                 ring->pending_buf = 0;
3451         }
3452
3453 out:
3454         /* Make all data has been write before submit */
3455         if (unused_count > 0)
3456                 hns3_nic_alloc_rx_buffers(ring, unused_count);
3457
3458         return recv_pkts;
3459 }
3460
3461 static bool hns3_get_new_flow_lvl(struct hns3_enet_ring_group *ring_group)
3462 {
3463 #define HNS3_RX_LOW_BYTE_RATE 10000
3464 #define HNS3_RX_MID_BYTE_RATE 20000
3465 #define HNS3_RX_ULTRA_PACKET_RATE 40
3466
3467         enum hns3_flow_level_range new_flow_level;
3468         struct hns3_enet_tqp_vector *tqp_vector;
3469         int packets_per_msecs, bytes_per_msecs;
3470         u32 time_passed_ms;
3471
3472         tqp_vector = ring_group->ring->tqp_vector;
3473         time_passed_ms =
3474                 jiffies_to_msecs(jiffies - tqp_vector->last_jiffies);
3475         if (!time_passed_ms)
3476                 return false;
3477
3478         do_div(ring_group->total_packets, time_passed_ms);
3479         packets_per_msecs = ring_group->total_packets;
3480
3481         do_div(ring_group->total_bytes, time_passed_ms);
3482         bytes_per_msecs = ring_group->total_bytes;
3483
3484         new_flow_level = ring_group->coal.flow_level;
3485
3486         /* Simple throttlerate management
3487          * 0-10MB/s   lower     (50000 ints/s)
3488          * 10-20MB/s   middle    (20000 ints/s)
3489          * 20-1249MB/s high      (18000 ints/s)
3490          * > 40000pps  ultra     (8000 ints/s)
3491          */
3492         switch (new_flow_level) {
3493         case HNS3_FLOW_LOW:
3494                 if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE)
3495                         new_flow_level = HNS3_FLOW_MID;
3496                 break;
3497         case HNS3_FLOW_MID:
3498                 if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE)
3499                         new_flow_level = HNS3_FLOW_HIGH;
3500                 else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE)
3501                         new_flow_level = HNS3_FLOW_LOW;
3502                 break;
3503         case HNS3_FLOW_HIGH:
3504         case HNS3_FLOW_ULTRA:
3505         default:
3506                 if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE)
3507                         new_flow_level = HNS3_FLOW_MID;
3508                 break;
3509         }
3510
3511         if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE &&
3512             &tqp_vector->rx_group == ring_group)
3513                 new_flow_level = HNS3_FLOW_ULTRA;
3514
3515         ring_group->total_bytes = 0;
3516         ring_group->total_packets = 0;
3517         ring_group->coal.flow_level = new_flow_level;
3518
3519         return true;
3520 }
3521
3522 static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
3523 {
3524         struct hns3_enet_tqp_vector *tqp_vector;
3525         u16 new_int_gl;
3526
3527         if (!ring_group->ring)
3528                 return false;
3529
3530         tqp_vector = ring_group->ring->tqp_vector;
3531         if (!tqp_vector->last_jiffies)
3532                 return false;
3533
3534         if (ring_group->total_packets == 0) {
3535                 ring_group->coal.int_gl = HNS3_INT_GL_50K;
3536                 ring_group->coal.flow_level = HNS3_FLOW_LOW;
3537                 return true;
3538         }
3539
3540         if (!hns3_get_new_flow_lvl(ring_group))
3541                 return false;
3542
3543         new_int_gl = ring_group->coal.int_gl;
3544         switch (ring_group->coal.flow_level) {
3545         case HNS3_FLOW_LOW:
3546                 new_int_gl = HNS3_INT_GL_50K;
3547                 break;
3548         case HNS3_FLOW_MID:
3549                 new_int_gl = HNS3_INT_GL_20K;
3550                 break;
3551         case HNS3_FLOW_HIGH:
3552                 new_int_gl = HNS3_INT_GL_18K;
3553                 break;
3554         case HNS3_FLOW_ULTRA:
3555                 new_int_gl = HNS3_INT_GL_8K;
3556                 break;
3557         default:
3558                 break;
3559         }
3560
3561         if (new_int_gl != ring_group->coal.int_gl) {
3562                 ring_group->coal.int_gl = new_int_gl;
3563                 return true;
3564         }
3565         return false;
3566 }
3567
3568 static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
3569 {
3570         struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
3571         struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
3572         bool rx_update, tx_update;
3573
3574         /* update param every 1000ms */
3575         if (time_before(jiffies,
3576                         tqp_vector->last_jiffies + msecs_to_jiffies(1000)))
3577                 return;
3578
3579         if (rx_group->coal.adapt_enable) {
3580                 rx_update = hns3_get_new_int_gl(rx_group);
3581                 if (rx_update)
3582                         hns3_set_vector_coalesce_rx_gl(tqp_vector,
3583                                                        rx_group->coal.int_gl);
3584         }
3585
3586         if (tx_group->coal.adapt_enable) {
3587                 tx_update = hns3_get_new_int_gl(tx_group);
3588                 if (tx_update)
3589                         hns3_set_vector_coalesce_tx_gl(tqp_vector,
3590                                                        tx_group->coal.int_gl);
3591         }
3592
3593         tqp_vector->last_jiffies = jiffies;
3594 }
3595
3596 static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
3597 {
3598         struct hns3_nic_priv *priv = netdev_priv(napi->dev);
3599         struct hns3_enet_ring *ring;
3600         int rx_pkt_total = 0;
3601
3602         struct hns3_enet_tqp_vector *tqp_vector =
3603                 container_of(napi, struct hns3_enet_tqp_vector, napi);
3604         bool clean_complete = true;
3605         int rx_budget = budget;
3606
3607         if (unlikely(test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {
3608                 napi_complete(napi);
3609                 return 0;
3610         }
3611
3612         /* Since the actual Tx work is minimal, we can give the Tx a larger
3613          * budget and be more aggressive about cleaning up the Tx descriptors.
3614          */
3615         hns3_for_each_ring(ring, tqp_vector->tx_group)
3616                 hns3_clean_tx_ring(ring, budget);
3617
3618         /* make sure rx ring budget not smaller than 1 */
3619         if (tqp_vector->num_tqps > 1)
3620                 rx_budget = max(budget / tqp_vector->num_tqps, 1);
3621
3622         hns3_for_each_ring(ring, tqp_vector->rx_group) {
3623                 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
3624                                                     hns3_rx_skb);
3625                 if (rx_cleaned >= rx_budget)
3626                         clean_complete = false;
3627
3628                 rx_pkt_total += rx_cleaned;
3629         }
3630
3631         tqp_vector->rx_group.total_packets += rx_pkt_total;
3632
3633         if (!clean_complete)
3634                 return budget;
3635
3636         if (napi_complete(napi) &&
3637             likely(!test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {
3638                 hns3_update_new_int_gl(tqp_vector);
3639                 hns3_mask_vector_irq(tqp_vector, 1);
3640         }
3641
3642         return rx_pkt_total;
3643 }
3644
3645 static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
3646                                       struct hnae3_ring_chain_node *head)
3647 {
3648         struct pci_dev *pdev = tqp_vector->handle->pdev;
3649         struct hnae3_ring_chain_node *cur_chain = head;
3650         struct hnae3_ring_chain_node *chain;
3651         struct hns3_enet_ring *tx_ring;
3652         struct hns3_enet_ring *rx_ring;
3653
3654         tx_ring = tqp_vector->tx_group.ring;
3655         if (tx_ring) {
3656                 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
3657                 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
3658                               HNAE3_RING_TYPE_TX);
3659                 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
3660                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
3661
3662                 cur_chain->next = NULL;
3663
3664                 while (tx_ring->next) {
3665                         tx_ring = tx_ring->next;
3666
3667                         chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
3668                                              GFP_KERNEL);
3669                         if (!chain)
3670                                 goto err_free_chain;
3671
3672                         cur_chain->next = chain;
3673                         chain->tqp_index = tx_ring->tqp->tqp_index;
3674                         hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
3675                                       HNAE3_RING_TYPE_TX);
3676                         hnae3_set_field(chain->int_gl_idx,
3677                                         HNAE3_RING_GL_IDX_M,
3678                                         HNAE3_RING_GL_IDX_S,
3679                                         HNAE3_RING_GL_TX);
3680
3681                         cur_chain = chain;
3682                 }
3683         }
3684
3685         rx_ring = tqp_vector->rx_group.ring;
3686         if (!tx_ring && rx_ring) {
3687                 cur_chain->next = NULL;
3688                 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
3689                 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
3690                               HNAE3_RING_TYPE_RX);
3691                 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
3692                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
3693
3694                 rx_ring = rx_ring->next;
3695         }
3696
3697         while (rx_ring) {
3698                 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
3699                 if (!chain)
3700                         goto err_free_chain;
3701
3702                 cur_chain->next = chain;
3703                 chain->tqp_index = rx_ring->tqp->tqp_index;
3704                 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
3705                               HNAE3_RING_TYPE_RX);
3706                 hnae3_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
3707                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
3708
3709                 cur_chain = chain;
3710
3711                 rx_ring = rx_ring->next;
3712         }
3713
3714         return 0;
3715
3716 err_free_chain:
3717         cur_chain = head->next;
3718         while (cur_chain) {
3719                 chain = cur_chain->next;
3720                 devm_kfree(&pdev->dev, cur_chain);
3721                 cur_chain = chain;
3722         }
3723         head->next = NULL;
3724
3725         return -ENOMEM;
3726 }
3727
3728 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
3729                                         struct hnae3_ring_chain_node *head)
3730 {
3731         struct pci_dev *pdev = tqp_vector->handle->pdev;
3732         struct hnae3_ring_chain_node *chain_tmp, *chain;
3733
3734         chain = head->next;
3735
3736         while (chain) {
3737                 chain_tmp = chain->next;
3738                 devm_kfree(&pdev->dev, chain);
3739                 chain = chain_tmp;
3740         }
3741 }
3742
3743 static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
3744                                    struct hns3_enet_ring *ring)
3745 {
3746         ring->next = group->ring;
3747         group->ring = ring;
3748
3749         group->count++;
3750 }
3751
3752 static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv)
3753 {
3754         struct pci_dev *pdev = priv->ae_handle->pdev;
3755         struct hns3_enet_tqp_vector *tqp_vector;
3756         int num_vectors = priv->vector_num;
3757         int numa_node;
3758         int vector_i;
3759
3760         numa_node = dev_to_node(&pdev->dev);
3761
3762         for (vector_i = 0; vector_i < num_vectors; vector_i++) {
3763                 tqp_vector = &priv->tqp_vector[vector_i];
3764                 cpumask_set_cpu(cpumask_local_spread(vector_i, numa_node),
3765                                 &tqp_vector->affinity_mask);
3766         }
3767 }
3768
3769 static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
3770 {
3771         struct hnae3_handle *h = priv->ae_handle;
3772         struct hns3_enet_tqp_vector *tqp_vector;
3773         int ret;
3774         int i;
3775
3776         hns3_nic_set_cpumask(priv);
3777
3778         for (i = 0; i < priv->vector_num; i++) {
3779                 tqp_vector = &priv->tqp_vector[i];
3780                 hns3_vector_coalesce_init_hw(tqp_vector, priv);
3781                 tqp_vector->num_tqps = 0;
3782         }
3783
3784         for (i = 0; i < h->kinfo.num_tqps; i++) {
3785                 u16 vector_i = i % priv->vector_num;
3786                 u16 tqp_num = h->kinfo.num_tqps;
3787
3788                 tqp_vector = &priv->tqp_vector[vector_i];
3789
3790                 hns3_add_ring_to_group(&tqp_vector->tx_group,
3791                                        &priv->ring[i]);
3792
3793                 hns3_add_ring_to_group(&tqp_vector->rx_group,
3794                                        &priv->ring[i + tqp_num]);
3795
3796                 priv->ring[i].tqp_vector = tqp_vector;
3797                 priv->ring[i + tqp_num].tqp_vector = tqp_vector;
3798                 tqp_vector->num_tqps++;
3799         }
3800
3801         for (i = 0; i < priv->vector_num; i++) {
3802                 struct hnae3_ring_chain_node vector_ring_chain;
3803
3804                 tqp_vector = &priv->tqp_vector[i];
3805
3806                 tqp_vector->rx_group.total_bytes = 0;
3807                 tqp_vector->rx_group.total_packets = 0;
3808                 tqp_vector->tx_group.total_bytes = 0;
3809                 tqp_vector->tx_group.total_packets = 0;
3810                 tqp_vector->handle = h;
3811
3812                 ret = hns3_get_vector_ring_chain(tqp_vector,
3813                                                  &vector_ring_chain);
3814                 if (ret)
3815                         goto map_ring_fail;
3816
3817                 ret = h->ae_algo->ops->map_ring_to_vector(h,
3818                         tqp_vector->vector_irq, &vector_ring_chain);
3819
3820                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
3821
3822                 if (ret)
3823                         goto map_ring_fail;
3824
3825                 netif_napi_add(priv->netdev, &tqp_vector->napi,
3826                                hns3_nic_common_poll, NAPI_POLL_WEIGHT);
3827         }
3828
3829         return 0;
3830
3831 map_ring_fail:
3832         while (i--)
3833                 netif_napi_del(&priv->tqp_vector[i].napi);
3834
3835         return ret;
3836 }
3837
3838 static void hns3_nic_init_coal_cfg(struct hns3_nic_priv *priv)
3839 {
3840         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);
3841         struct hns3_enet_coalesce *tx_coal = &priv->tx_coal;
3842         struct hns3_enet_coalesce *rx_coal = &priv->rx_coal;
3843
3844         /* initialize the configuration for interrupt coalescing.
3845          * 1. GL (Interrupt Gap Limiter)
3846          * 2. RL (Interrupt Rate Limiter)
3847          * 3. QL (Interrupt Quantity Limiter)
3848          *
3849          * Default: enable interrupt coalescing self-adaptive and GL
3850          */
3851         tx_coal->adapt_enable = 1;
3852         rx_coal->adapt_enable = 1;
3853
3854         tx_coal->int_gl = HNS3_INT_GL_50K;
3855         rx_coal->int_gl = HNS3_INT_GL_50K;
3856
3857         rx_coal->flow_level = HNS3_FLOW_LOW;
3858         tx_coal->flow_level = HNS3_FLOW_LOW;
3859
3860         if (ae_dev->dev_specs.int_ql_max) {
3861                 tx_coal->int_ql = HNS3_INT_QL_DEFAULT_CFG;
3862                 rx_coal->int_ql = HNS3_INT_QL_DEFAULT_CFG;
3863         }
3864 }
3865
3866 static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
3867 {
3868         struct hnae3_handle *h = priv->ae_handle;
3869         struct hns3_enet_tqp_vector *tqp_vector;
3870         struct hnae3_vector_info *vector;
3871         struct pci_dev *pdev = h->pdev;
3872         u16 tqp_num = h->kinfo.num_tqps;
3873         u16 vector_num;
3874         int ret = 0;
3875         u16 i;
3876
3877         /* RSS size, cpu online and vector_num should be the same */
3878         /* Should consider 2p/4p later */
3879         vector_num = min_t(u16, num_online_cpus(), tqp_num);
3880
3881         vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
3882                               GFP_KERNEL);
3883         if (!vector)
3884                 return -ENOMEM;
3885
3886         /* save the actual available vector number */
3887         vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
3888
3889         priv->vector_num = vector_num;
3890         priv->tqp_vector = (struct hns3_enet_tqp_vector *)
3891                 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
3892                              GFP_KERNEL);
3893         if (!priv->tqp_vector) {
3894                 ret = -ENOMEM;
3895                 goto out;
3896         }
3897
3898         for (i = 0; i < priv->vector_num; i++) {
3899                 tqp_vector = &priv->tqp_vector[i];
3900                 tqp_vector->idx = i;
3901                 tqp_vector->mask_addr = vector[i].io_addr;
3902                 tqp_vector->vector_irq = vector[i].vector;
3903                 hns3_vector_coalesce_init(tqp_vector, priv);
3904         }
3905
3906 out:
3907         devm_kfree(&pdev->dev, vector);
3908         return ret;
3909 }
3910
3911 static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
3912 {
3913         group->ring = NULL;
3914         group->count = 0;
3915 }
3916
3917 static void hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
3918 {
3919         struct hnae3_ring_chain_node vector_ring_chain;
3920         struct hnae3_handle *h = priv->ae_handle;
3921         struct hns3_enet_tqp_vector *tqp_vector;
3922         int i;
3923
3924         for (i = 0; i < priv->vector_num; i++) {
3925                 tqp_vector = &priv->tqp_vector[i];
3926
3927                 if (!tqp_vector->rx_group.ring && !tqp_vector->tx_group.ring)
3928                         continue;
3929
3930                 /* Since the mapping can be overwritten, when fail to get the
3931                  * chain between vector and ring, we should go on to deal with
3932                  * the remaining options.
3933                  */
3934                 if (hns3_get_vector_ring_chain(tqp_vector, &vector_ring_chain))
3935                         dev_warn(priv->dev, "failed to get ring chain\n");
3936
3937                 h->ae_algo->ops->unmap_ring_from_vector(h,
3938                         tqp_vector->vector_irq, &vector_ring_chain);
3939
3940                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
3941
3942                 hns3_clear_ring_group(&tqp_vector->rx_group);
3943                 hns3_clear_ring_group(&tqp_vector->tx_group);
3944                 netif_napi_del(&priv->tqp_vector[i].napi);
3945         }
3946 }
3947
3948 static void hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
3949 {
3950         struct hnae3_handle *h = priv->ae_handle;
3951         struct pci_dev *pdev = h->pdev;
3952         int i, ret;
3953
3954         for (i = 0; i < priv->vector_num; i++) {
3955                 struct hns3_enet_tqp_vector *tqp_vector;
3956
3957                 tqp_vector = &priv->tqp_vector[i];
3958                 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
3959                 if (ret)
3960                         return;
3961         }
3962
3963         devm_kfree(&pdev->dev, priv->tqp_vector);
3964 }
3965
3966 static void hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
3967                               unsigned int ring_type)
3968 {
3969         int queue_num = priv->ae_handle->kinfo.num_tqps;
3970         struct hns3_enet_ring *ring;
3971         int desc_num;
3972
3973         if (ring_type == HNAE3_RING_TYPE_TX) {
3974                 ring = &priv->ring[q->tqp_index];
3975                 desc_num = priv->ae_handle->kinfo.num_tx_desc;
3976                 ring->queue_index = q->tqp_index;
3977         } else {
3978                 ring = &priv->ring[q->tqp_index + queue_num];
3979                 desc_num = priv->ae_handle->kinfo.num_rx_desc;
3980                 ring->queue_index = q->tqp_index;
3981         }
3982
3983         hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
3984
3985         ring->tqp = q;
3986         ring->desc = NULL;
3987         ring->desc_cb = NULL;
3988         ring->dev = priv->dev;
3989         ring->desc_dma_addr = 0;
3990         ring->buf_size = q->buf_size;
3991         ring->desc_num = desc_num;
3992         ring->next_to_use = 0;
3993         ring->next_to_clean = 0;
3994         ring->last_to_use = 0;
3995 }
3996
3997 static void hns3_queue_to_ring(struct hnae3_queue *tqp,
3998                                struct hns3_nic_priv *priv)
3999 {
4000         hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
4001         hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
4002 }
4003
4004 static int hns3_get_ring_config(struct hns3_nic_priv *priv)
4005 {
4006         struct hnae3_handle *h = priv->ae_handle;
4007         struct pci_dev *pdev = h->pdev;
4008         int i;
4009
4010         priv->ring = devm_kzalloc(&pdev->dev,
4011                                   array3_size(h->kinfo.num_tqps,
4012                                               sizeof(*priv->ring), 2),
4013                                   GFP_KERNEL);
4014         if (!priv->ring)
4015                 return -ENOMEM;
4016
4017         for (i = 0; i < h->kinfo.num_tqps; i++)
4018                 hns3_queue_to_ring(h->kinfo.tqp[i], priv);
4019
4020         return 0;
4021 }
4022
4023 static void hns3_put_ring_config(struct hns3_nic_priv *priv)
4024 {
4025         if (!priv->ring)
4026                 return;
4027
4028         devm_kfree(priv->dev, priv->ring);
4029         priv->ring = NULL;
4030 }
4031
4032 static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
4033 {
4034         int ret;
4035
4036         if (ring->desc_num <= 0 || ring->buf_size <= 0)
4037                 return -EINVAL;
4038
4039         ring->desc_cb = devm_kcalloc(ring_to_dev(ring), ring->desc_num,
4040                                      sizeof(ring->desc_cb[0]), GFP_KERNEL);
4041         if (!ring->desc_cb) {
4042                 ret = -ENOMEM;
4043                 goto out;
4044         }
4045
4046         ret = hns3_alloc_desc(ring);
4047         if (ret)
4048                 goto out_with_desc_cb;
4049
4050         if (!HNAE3_IS_TX_RING(ring)) {
4051                 ret = hns3_alloc_ring_buffers(ring);
4052                 if (ret)
4053                         goto out_with_desc;
4054         }
4055
4056         return 0;
4057
4058 out_with_desc:
4059         hns3_free_desc(ring);
4060 out_with_desc_cb:
4061         devm_kfree(ring_to_dev(ring), ring->desc_cb);
4062         ring->desc_cb = NULL;
4063 out:
4064         return ret;
4065 }
4066
4067 void hns3_fini_ring(struct hns3_enet_ring *ring)
4068 {
4069         hns3_free_desc(ring);
4070         devm_kfree(ring_to_dev(ring), ring->desc_cb);
4071         ring->desc_cb = NULL;
4072         ring->next_to_clean = 0;
4073         ring->next_to_use = 0;
4074         ring->last_to_use = 0;
4075         ring->pending_buf = 0;
4076         if (ring->skb) {
4077                 dev_kfree_skb_any(ring->skb);
4078                 ring->skb = NULL;
4079         }
4080 }
4081
4082 static int hns3_buf_size2type(u32 buf_size)
4083 {
4084         int bd_size_type;
4085
4086         switch (buf_size) {
4087         case 512:
4088                 bd_size_type = HNS3_BD_SIZE_512_TYPE;
4089                 break;
4090         case 1024:
4091                 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
4092                 break;
4093         case 2048:
4094                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
4095                 break;
4096         case 4096:
4097                 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
4098                 break;
4099         default:
4100                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
4101         }
4102
4103         return bd_size_type;
4104 }
4105
4106 static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
4107 {
4108         dma_addr_t dma = ring->desc_dma_addr;
4109         struct hnae3_queue *q = ring->tqp;
4110
4111         if (!HNAE3_IS_TX_RING(ring)) {
4112                 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG, (u32)dma);
4113                 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
4114                                (u32)((dma >> 31) >> 1));
4115
4116                 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
4117                                hns3_buf_size2type(ring->buf_size));
4118                 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
4119                                ring->desc_num / 8 - 1);
4120         } else {
4121                 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
4122                                (u32)dma);
4123                 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
4124                                (u32)((dma >> 31) >> 1));
4125
4126                 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
4127                                ring->desc_num / 8 - 1);
4128         }
4129 }
4130
4131 static void hns3_init_tx_ring_tc(struct hns3_nic_priv *priv)
4132 {
4133         struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;
4134         struct hnae3_tc_info *tc_info = &kinfo->tc_info;
4135         int i;
4136
4137         for (i = 0; i < HNAE3_MAX_TC; i++) {
4138                 int j;
4139
4140                 if (!test_bit(i, &tc_info->tc_en))
4141                         continue;
4142
4143                 for (j = 0; j < tc_info->tqp_count[i]; j++) {
4144                         struct hnae3_queue *q;
4145
4146                         q = priv->ring[tc_info->tqp_offset[i] + j].tqp;
4147                         hns3_write_dev(q, HNS3_RING_TX_RING_TC_REG, i);
4148                 }
4149         }
4150 }
4151
4152 int hns3_init_all_ring(struct hns3_nic_priv *priv)
4153 {
4154         struct hnae3_handle *h = priv->ae_handle;
4155         int ring_num = h->kinfo.num_tqps * 2;
4156         int i, j;
4157         int ret;
4158
4159         for (i = 0; i < ring_num; i++) {
4160                 ret = hns3_alloc_ring_memory(&priv->ring[i]);
4161                 if (ret) {
4162                         dev_err(priv->dev,
4163                                 "Alloc ring memory fail! ret=%d\n", ret);
4164                         goto out_when_alloc_ring_memory;
4165                 }
4166
4167                 u64_stats_init(&priv->ring[i].syncp);
4168         }
4169
4170         return 0;
4171
4172 out_when_alloc_ring_memory:
4173         for (j = i - 1; j >= 0; j--)
4174                 hns3_fini_ring(&priv->ring[j]);
4175
4176         return -ENOMEM;
4177 }
4178
4179 static void hns3_uninit_all_ring(struct hns3_nic_priv *priv)
4180 {
4181         struct hnae3_handle *h = priv->ae_handle;
4182         int i;
4183
4184         for (i = 0; i < h->kinfo.num_tqps; i++) {
4185                 hns3_fini_ring(&priv->ring[i]);
4186                 hns3_fini_ring(&priv->ring[i + h->kinfo.num_tqps]);
4187         }
4188 }
4189
4190 /* Set mac addr if it is configured. or leave it to the AE driver */
4191 static int hns3_init_mac_addr(struct net_device *netdev)
4192 {
4193         struct hns3_nic_priv *priv = netdev_priv(netdev);
4194         struct hnae3_handle *h = priv->ae_handle;
4195         u8 mac_addr_temp[ETH_ALEN];
4196         int ret = 0;
4197
4198         if (h->ae_algo->ops->get_mac_addr)
4199                 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
4200
4201         /* Check if the MAC address is valid, if not get a random one */
4202         if (!is_valid_ether_addr(mac_addr_temp)) {
4203                 eth_hw_addr_random(netdev);
4204                 dev_warn(priv->dev, "using random MAC address %pM\n",
4205                          netdev->dev_addr);
4206         } else if (!ether_addr_equal(netdev->dev_addr, mac_addr_temp)) {
4207                 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
4208                 ether_addr_copy(netdev->perm_addr, mac_addr_temp);
4209         } else {
4210                 return 0;
4211         }
4212
4213         if (h->ae_algo->ops->set_mac_addr)
4214                 ret = h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true);
4215
4216         return ret;
4217 }
4218
4219 static int hns3_init_phy(struct net_device *netdev)
4220 {
4221         struct hnae3_handle *h = hns3_get_handle(netdev);
4222         int ret = 0;
4223
4224         if (h->ae_algo->ops->mac_connect_phy)
4225                 ret = h->ae_algo->ops->mac_connect_phy(h);
4226
4227         return ret;
4228 }
4229
4230 static void hns3_uninit_phy(struct net_device *netdev)
4231 {
4232         struct hnae3_handle *h = hns3_get_handle(netdev);
4233
4234         if (h->ae_algo->ops->mac_disconnect_phy)
4235                 h->ae_algo->ops->mac_disconnect_phy(h);
4236 }
4237
4238 static int hns3_client_start(struct hnae3_handle *handle)
4239 {
4240         if (!handle->ae_algo->ops->client_start)
4241                 return 0;
4242
4243         return handle->ae_algo->ops->client_start(handle);
4244 }
4245
4246 static void hns3_client_stop(struct hnae3_handle *handle)
4247 {
4248         if (!handle->ae_algo->ops->client_stop)
4249                 return;
4250
4251         handle->ae_algo->ops->client_stop(handle);
4252 }
4253
4254 static void hns3_info_show(struct hns3_nic_priv *priv)
4255 {
4256         struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;
4257
4258         dev_info(priv->dev, "MAC address: %pM\n", priv->netdev->dev_addr);
4259         dev_info(priv->dev, "Task queue pairs numbers: %u\n", kinfo->num_tqps);
4260         dev_info(priv->dev, "RSS size: %u\n", kinfo->rss_size);
4261         dev_info(priv->dev, "Allocated RSS size: %u\n", kinfo->req_rss_size);
4262         dev_info(priv->dev, "RX buffer length: %u\n", kinfo->rx_buf_len);
4263         dev_info(priv->dev, "Desc num per TX queue: %u\n", kinfo->num_tx_desc);
4264         dev_info(priv->dev, "Desc num per RX queue: %u\n", kinfo->num_rx_desc);
4265         dev_info(priv->dev, "Total number of enabled TCs: %u\n",
4266                  kinfo->tc_info.num_tc);
4267         dev_info(priv->dev, "Max mtu size: %u\n", priv->netdev->max_mtu);
4268 }
4269
4270 static int hns3_client_init(struct hnae3_handle *handle)
4271 {
4272         struct pci_dev *pdev = handle->pdev;
4273         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
4274         u16 alloc_tqps, max_rss_size;
4275         struct hns3_nic_priv *priv;
4276         struct net_device *netdev;
4277         int ret;
4278
4279         handle->ae_algo->ops->get_tqps_and_rss_info(handle, &alloc_tqps,
4280                                                     &max_rss_size);
4281         netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv), alloc_tqps);
4282         if (!netdev)
4283                 return -ENOMEM;
4284
4285         priv = netdev_priv(netdev);
4286         priv->dev = &pdev->dev;
4287         priv->netdev = netdev;
4288         priv->ae_handle = handle;
4289         priv->tx_timeout_count = 0;
4290         priv->max_non_tso_bd_num = ae_dev->dev_specs.max_non_tso_bd_num;
4291         set_bit(HNS3_NIC_STATE_DOWN, &priv->state);
4292
4293         handle->msg_enable = netif_msg_init(debug, DEFAULT_MSG_LEVEL);
4294
4295         handle->kinfo.netdev = netdev;
4296         handle->priv = (void *)priv;
4297
4298         hns3_init_mac_addr(netdev);
4299
4300         hns3_set_default_feature(netdev);
4301
4302         netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
4303         netdev->priv_flags |= IFF_UNICAST_FLT;
4304         netdev->netdev_ops = &hns3_nic_netdev_ops;
4305         SET_NETDEV_DEV(netdev, &pdev->dev);
4306         hns3_ethtool_set_ops(netdev);
4307
4308         /* Carrier off reporting is important to ethtool even BEFORE open */
4309         netif_carrier_off(netdev);
4310
4311         ret = hns3_get_ring_config(priv);
4312         if (ret) {
4313                 ret = -ENOMEM;
4314                 goto out_get_ring_cfg;
4315         }
4316
4317         hns3_nic_init_coal_cfg(priv);
4318
4319         ret = hns3_nic_alloc_vector_data(priv);
4320         if (ret) {
4321                 ret = -ENOMEM;
4322                 goto out_alloc_vector_data;
4323         }
4324
4325         ret = hns3_nic_init_vector_data(priv);
4326         if (ret) {
4327                 ret = -ENOMEM;
4328                 goto out_init_vector_data;
4329         }
4330
4331         ret = hns3_init_all_ring(priv);
4332         if (ret) {
4333                 ret = -ENOMEM;
4334                 goto out_init_ring;
4335         }
4336
4337         ret = hns3_init_phy(netdev);
4338         if (ret)
4339                 goto out_init_phy;
4340
4341         /* the device can work without cpu rmap, only aRFS needs it */
4342         ret = hns3_set_rx_cpu_rmap(netdev);
4343         if (ret)
4344                 dev_warn(priv->dev, "set rx cpu rmap fail, ret=%d\n", ret);
4345
4346         ret = hns3_nic_init_irq(priv);
4347         if (ret) {
4348                 dev_err(priv->dev, "init irq failed! ret=%d\n", ret);
4349                 hns3_free_rx_cpu_rmap(netdev);
4350                 goto out_init_irq_fail;
4351         }
4352
4353         ret = hns3_client_start(handle);
4354         if (ret) {
4355                 dev_err(priv->dev, "hns3_client_start fail! ret=%d\n", ret);
4356                 goto out_client_start;
4357         }
4358
4359         hns3_dcbnl_setup(handle);
4360
4361         hns3_dbg_init(handle);
4362
4363         netdev->max_mtu = HNS3_MAX_MTU(ae_dev->dev_specs.max_frm_size);
4364
4365         if (test_bit(HNAE3_DEV_SUPPORT_HW_TX_CSUM_B, ae_dev->caps))
4366                 set_bit(HNS3_NIC_STATE_HW_TX_CSUM_ENABLE, &priv->state);
4367
4368         set_bit(HNS3_NIC_STATE_INITED, &priv->state);
4369
4370         if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3)
4371                 set_bit(HNAE3_PFLAG_LIMIT_PROMISC, &handle->supported_pflags);
4372
4373         ret = register_netdev(netdev);
4374         if (ret) {
4375                 dev_err(priv->dev, "probe register netdev fail!\n");
4376                 goto out_reg_netdev_fail;
4377         }
4378
4379         if (netif_msg_drv(handle))
4380                 hns3_info_show(priv);
4381
4382         return ret;
4383
4384 out_reg_netdev_fail:
4385         hns3_dbg_uninit(handle);
4386 out_client_start:
4387         hns3_free_rx_cpu_rmap(netdev);
4388         hns3_nic_uninit_irq(priv);
4389 out_init_irq_fail:
4390         hns3_uninit_phy(netdev);
4391 out_init_phy:
4392         hns3_uninit_all_ring(priv);
4393 out_init_ring:
4394         hns3_nic_uninit_vector_data(priv);
4395 out_init_vector_data:
4396         hns3_nic_dealloc_vector_data(priv);
4397 out_alloc_vector_data:
4398         priv->ring = NULL;
4399 out_get_ring_cfg:
4400         priv->ae_handle = NULL;
4401         free_netdev(netdev);
4402         return ret;
4403 }
4404
4405 static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
4406 {
4407         struct net_device *netdev = handle->kinfo.netdev;
4408         struct hns3_nic_priv *priv = netdev_priv(netdev);
4409
4410         if (netdev->reg_state != NETREG_UNINITIALIZED)
4411                 unregister_netdev(netdev);
4412
4413         hns3_client_stop(handle);
4414
4415         hns3_uninit_phy(netdev);
4416
4417         if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
4418                 netdev_warn(netdev, "already uninitialized\n");
4419                 goto out_netdev_free;
4420         }
4421
4422         hns3_free_rx_cpu_rmap(netdev);
4423
4424         hns3_nic_uninit_irq(priv);
4425
4426         hns3_clear_all_ring(handle, true);
4427
4428         hns3_nic_uninit_vector_data(priv);
4429
4430         hns3_nic_dealloc_vector_data(priv);
4431
4432         hns3_uninit_all_ring(priv);
4433
4434         hns3_put_ring_config(priv);
4435
4436 out_netdev_free:
4437         hns3_dbg_uninit(handle);
4438         free_netdev(netdev);
4439 }
4440
4441 static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
4442 {
4443         struct net_device *netdev = handle->kinfo.netdev;
4444
4445         if (!netdev)
4446                 return;
4447
4448         if (linkup) {
4449                 netif_tx_wake_all_queues(netdev);
4450                 netif_carrier_on(netdev);
4451                 if (netif_msg_link(handle))
4452                         netdev_info(netdev, "link up\n");
4453         } else {
4454                 netif_carrier_off(netdev);
4455                 netif_tx_stop_all_queues(netdev);
4456                 if (netif_msg_link(handle))
4457                         netdev_info(netdev, "link down\n");
4458         }
4459 }
4460
4461 static void hns3_clear_tx_ring(struct hns3_enet_ring *ring)
4462 {
4463         while (ring->next_to_clean != ring->next_to_use) {
4464                 ring->desc[ring->next_to_clean].tx.bdtp_fe_sc_vld_ra_ri = 0;
4465                 hns3_free_buffer_detach(ring, ring->next_to_clean, 0);
4466                 ring_ptr_move_fw(ring, next_to_clean);
4467         }
4468
4469         ring->pending_buf = 0;
4470 }
4471
4472 static int hns3_clear_rx_ring(struct hns3_enet_ring *ring)
4473 {
4474         struct hns3_desc_cb res_cbs;
4475         int ret;
4476
4477         while (ring->next_to_use != ring->next_to_clean) {
4478                 /* When a buffer is not reused, it's memory has been
4479                  * freed in hns3_handle_rx_bd or will be freed by
4480                  * stack, so we need to replace the buffer here.
4481                  */
4482                 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
4483                         ret = hns3_alloc_and_map_buffer(ring, &res_cbs);
4484                         if (ret) {
4485                                 u64_stats_update_begin(&ring->syncp);
4486                                 ring->stats.sw_err_cnt++;
4487                                 u64_stats_update_end(&ring->syncp);
4488                                 /* if alloc new buffer fail, exit directly
4489                                  * and reclear in up flow.
4490                                  */
4491                                 netdev_warn(ring_to_netdev(ring),
4492                                             "reserve buffer map failed, ret = %d\n",
4493                                             ret);
4494                                 return ret;
4495                         }
4496                         hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
4497                 }
4498                 ring_ptr_move_fw(ring, next_to_use);
4499         }
4500
4501         /* Free the pending skb in rx ring */
4502         if (ring->skb) {
4503                 dev_kfree_skb_any(ring->skb);
4504                 ring->skb = NULL;
4505                 ring->pending_buf = 0;
4506         }
4507
4508         return 0;
4509 }
4510
4511 static void hns3_force_clear_rx_ring(struct hns3_enet_ring *ring)
4512 {
4513         while (ring->next_to_use != ring->next_to_clean) {
4514                 /* When a buffer is not reused, it's memory has been
4515                  * freed in hns3_handle_rx_bd or will be freed by
4516                  * stack, so only need to unmap the buffer here.
4517                  */
4518                 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
4519                         hns3_unmap_buffer(ring,
4520                                           &ring->desc_cb[ring->next_to_use]);
4521                         ring->desc_cb[ring->next_to_use].dma = 0;
4522                 }
4523
4524                 ring_ptr_move_fw(ring, next_to_use);
4525         }
4526 }
4527
4528 static void hns3_clear_all_ring(struct hnae3_handle *h, bool force)
4529 {
4530         struct net_device *ndev = h->kinfo.netdev;
4531         struct hns3_nic_priv *priv = netdev_priv(ndev);
4532         u32 i;
4533
4534         for (i = 0; i < h->kinfo.num_tqps; i++) {
4535                 struct hns3_enet_ring *ring;
4536
4537                 ring = &priv->ring[i];
4538                 hns3_clear_tx_ring(ring);
4539
4540                 ring = &priv->ring[i + h->kinfo.num_tqps];
4541                 /* Continue to clear other rings even if clearing some
4542                  * rings failed.
4543                  */
4544                 if (force)
4545                         hns3_force_clear_rx_ring(ring);
4546                 else
4547                         hns3_clear_rx_ring(ring);
4548         }
4549 }
4550
4551 int hns3_nic_reset_all_ring(struct hnae3_handle *h)
4552 {
4553         struct net_device *ndev = h->kinfo.netdev;
4554         struct hns3_nic_priv *priv = netdev_priv(ndev);
4555         struct hns3_enet_ring *rx_ring;
4556         int i, j;
4557         int ret;
4558
4559         ret = h->ae_algo->ops->reset_queue(h);
4560         if (ret)
4561                 return ret;
4562
4563         for (i = 0; i < h->kinfo.num_tqps; i++) {
4564                 hns3_init_ring_hw(&priv->ring[i]);
4565
4566                 /* We need to clear tx ring here because self test will
4567                  * use the ring and will not run down before up
4568                  */
4569                 hns3_clear_tx_ring(&priv->ring[i]);
4570                 priv->ring[i].next_to_clean = 0;
4571                 priv->ring[i].next_to_use = 0;
4572                 priv->ring[i].last_to_use = 0;
4573
4574                 rx_ring = &priv->ring[i + h->kinfo.num_tqps];
4575                 hns3_init_ring_hw(rx_ring);
4576                 ret = hns3_clear_rx_ring(rx_ring);
4577                 if (ret)
4578                         return ret;
4579
4580                 /* We can not know the hardware head and tail when this
4581                  * function is called in reset flow, so we reuse all desc.
4582                  */
4583                 for (j = 0; j < rx_ring->desc_num; j++)
4584                         hns3_reuse_buffer(rx_ring, j);
4585
4586                 rx_ring->next_to_clean = 0;
4587                 rx_ring->next_to_use = 0;
4588         }
4589
4590         hns3_init_tx_ring_tc(priv);
4591
4592         return 0;
4593 }
4594
4595 static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
4596 {
4597         struct hnae3_knic_private_info *kinfo = &handle->kinfo;
4598         struct net_device *ndev = kinfo->netdev;
4599         struct hns3_nic_priv *priv = netdev_priv(ndev);
4600
4601         if (test_and_set_bit(HNS3_NIC_STATE_RESETTING, &priv->state))
4602                 return 0;
4603
4604         if (!netif_running(ndev))
4605                 return 0;
4606
4607         return hns3_nic_net_stop(ndev);
4608 }
4609
4610 static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
4611 {
4612         struct hnae3_knic_private_info *kinfo = &handle->kinfo;
4613         struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev);
4614         int ret = 0;
4615
4616         if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
4617                 netdev_err(kinfo->netdev, "device is not initialized yet\n");
4618                 return -EFAULT;
4619         }
4620
4621         clear_bit(HNS3_NIC_STATE_RESETTING, &priv->state);
4622
4623         if (netif_running(kinfo->netdev)) {
4624                 ret = hns3_nic_net_open(kinfo->netdev);
4625                 if (ret) {
4626                         set_bit(HNS3_NIC_STATE_RESETTING, &priv->state);
4627                         netdev_err(kinfo->netdev,
4628                                    "net up fail, ret=%d!\n", ret);
4629                         return ret;
4630                 }
4631         }
4632
4633         return ret;
4634 }
4635
4636 static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
4637 {
4638         struct net_device *netdev = handle->kinfo.netdev;
4639         struct hns3_nic_priv *priv = netdev_priv(netdev);
4640         int ret;
4641
4642         /* Carrier off reporting is important to ethtool even BEFORE open */
4643         netif_carrier_off(netdev);
4644
4645         ret = hns3_get_ring_config(priv);
4646         if (ret)
4647                 return ret;
4648
4649         ret = hns3_nic_alloc_vector_data(priv);
4650         if (ret)
4651                 goto err_put_ring;
4652
4653         ret = hns3_nic_init_vector_data(priv);
4654         if (ret)
4655                 goto err_dealloc_vector;
4656
4657         ret = hns3_init_all_ring(priv);
4658         if (ret)
4659                 goto err_uninit_vector;
4660
4661         /* the device can work without cpu rmap, only aRFS needs it */
4662         ret = hns3_set_rx_cpu_rmap(netdev);
4663         if (ret)
4664                 dev_warn(priv->dev, "set rx cpu rmap fail, ret=%d\n", ret);
4665
4666         ret = hns3_nic_init_irq(priv);
4667         if (ret) {
4668                 dev_err(priv->dev, "init irq failed! ret=%d\n", ret);
4669                 hns3_free_rx_cpu_rmap(netdev);
4670                 goto err_init_irq_fail;
4671         }
4672
4673         if (!hns3_is_phys_func(handle->pdev))
4674                 hns3_init_mac_addr(netdev);
4675
4676         ret = hns3_client_start(handle);
4677         if (ret) {
4678                 dev_err(priv->dev, "hns3_client_start fail! ret=%d\n", ret);
4679                 goto err_client_start_fail;
4680         }
4681
4682         set_bit(HNS3_NIC_STATE_INITED, &priv->state);
4683
4684         return ret;
4685
4686 err_client_start_fail:
4687         hns3_free_rx_cpu_rmap(netdev);
4688         hns3_nic_uninit_irq(priv);
4689 err_init_irq_fail:
4690         hns3_uninit_all_ring(priv);
4691 err_uninit_vector:
4692         hns3_nic_uninit_vector_data(priv);
4693 err_dealloc_vector:
4694         hns3_nic_dealloc_vector_data(priv);
4695 err_put_ring:
4696         hns3_put_ring_config(priv);
4697
4698         return ret;
4699 }
4700
4701 static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
4702 {
4703         struct net_device *netdev = handle->kinfo.netdev;
4704         struct hns3_nic_priv *priv = netdev_priv(netdev);
4705
4706         if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
4707                 netdev_warn(netdev, "already uninitialized\n");
4708                 return 0;
4709         }
4710
4711         hns3_free_rx_cpu_rmap(netdev);
4712         hns3_nic_uninit_irq(priv);
4713         hns3_clear_all_ring(handle, true);
4714         hns3_reset_tx_queue(priv->ae_handle);
4715
4716         hns3_nic_uninit_vector_data(priv);
4717
4718         hns3_nic_dealloc_vector_data(priv);
4719
4720         hns3_uninit_all_ring(priv);
4721
4722         hns3_put_ring_config(priv);
4723
4724         return 0;
4725 }
4726
4727 static int hns3_reset_notify(struct hnae3_handle *handle,
4728                              enum hnae3_reset_notify_type type)
4729 {
4730         int ret = 0;
4731
4732         switch (type) {
4733         case HNAE3_UP_CLIENT:
4734                 ret = hns3_reset_notify_up_enet(handle);
4735                 break;
4736         case HNAE3_DOWN_CLIENT:
4737                 ret = hns3_reset_notify_down_enet(handle);
4738                 break;
4739         case HNAE3_INIT_CLIENT:
4740                 ret = hns3_reset_notify_init_enet(handle);
4741                 break;
4742         case HNAE3_UNINIT_CLIENT:
4743                 ret = hns3_reset_notify_uninit_enet(handle);
4744                 break;
4745         default:
4746                 break;
4747         }
4748
4749         return ret;
4750 }
4751
4752 static int hns3_change_channels(struct hnae3_handle *handle, u32 new_tqp_num,
4753                                 bool rxfh_configured)
4754 {
4755         int ret;
4756
4757         ret = handle->ae_algo->ops->set_channels(handle, new_tqp_num,
4758                                                  rxfh_configured);
4759         if (ret) {
4760                 dev_err(&handle->pdev->dev,
4761                         "Change tqp num(%u) fail.\n", new_tqp_num);
4762                 return ret;
4763         }
4764
4765         ret = hns3_reset_notify(handle, HNAE3_INIT_CLIENT);
4766         if (ret)
4767                 return ret;
4768
4769         ret =  hns3_reset_notify(handle, HNAE3_UP_CLIENT);
4770         if (ret)
4771                 hns3_reset_notify(handle, HNAE3_UNINIT_CLIENT);
4772
4773         return ret;
4774 }
4775
4776 int hns3_set_channels(struct net_device *netdev,
4777                       struct ethtool_channels *ch)
4778 {
4779         struct hnae3_handle *h = hns3_get_handle(netdev);
4780         struct hnae3_knic_private_info *kinfo = &h->kinfo;
4781         bool rxfh_configured = netif_is_rxfh_configured(netdev);
4782         u32 new_tqp_num = ch->combined_count;
4783         u16 org_tqp_num;
4784         int ret;
4785
4786         if (hns3_nic_resetting(netdev))
4787                 return -EBUSY;
4788
4789         if (ch->rx_count || ch->tx_count)
4790                 return -EINVAL;
4791
4792         if (kinfo->tc_info.mqprio_active) {
4793                 dev_err(&netdev->dev,
4794                         "it's not allowed to set channels via ethtool when MQPRIO mode is on\n");
4795                 return -EINVAL;
4796         }
4797
4798         if (new_tqp_num > hns3_get_max_available_channels(h) ||
4799             new_tqp_num < 1) {
4800                 dev_err(&netdev->dev,
4801                         "Change tqps fail, the tqp range is from 1 to %u",
4802                         hns3_get_max_available_channels(h));
4803                 return -EINVAL;
4804         }
4805
4806         if (kinfo->rss_size == new_tqp_num)
4807                 return 0;
4808
4809         netif_dbg(h, drv, netdev,
4810                   "set channels: tqp_num=%u, rxfh=%d\n",
4811                   new_tqp_num, rxfh_configured);
4812
4813         ret = hns3_reset_notify(h, HNAE3_DOWN_CLIENT);
4814         if (ret)
4815                 return ret;
4816
4817         ret = hns3_reset_notify(h, HNAE3_UNINIT_CLIENT);
4818         if (ret)
4819                 return ret;
4820
4821         org_tqp_num = h->kinfo.num_tqps;
4822         ret = hns3_change_channels(h, new_tqp_num, rxfh_configured);
4823         if (ret) {
4824                 int ret1;
4825
4826                 netdev_warn(netdev,
4827                             "Change channels fail, revert to old value\n");
4828                 ret1 = hns3_change_channels(h, org_tqp_num, rxfh_configured);
4829                 if (ret1) {
4830                         netdev_err(netdev,
4831                                    "revert to old channel fail\n");
4832                         return ret1;
4833                 }
4834
4835                 return ret;
4836         }
4837
4838         return 0;
4839 }
4840
4841 static const struct hns3_hw_error_info hns3_hw_err[] = {
4842         { .type = HNAE3_PPU_POISON_ERROR,
4843           .msg = "PPU poison" },
4844         { .type = HNAE3_CMDQ_ECC_ERROR,
4845           .msg = "IMP CMDQ error" },
4846         { .type = HNAE3_IMP_RD_POISON_ERROR,
4847           .msg = "IMP RD poison" },
4848         { .type = HNAE3_ROCEE_AXI_RESP_ERROR,
4849           .msg = "ROCEE AXI RESP error" },
4850 };
4851
4852 static void hns3_process_hw_error(struct hnae3_handle *handle,
4853                                   enum hnae3_hw_error_type type)
4854 {
4855         int i;
4856
4857         for (i = 0; i < ARRAY_SIZE(hns3_hw_err); i++) {
4858                 if (hns3_hw_err[i].type == type) {
4859                         dev_err(&handle->pdev->dev, "Detected %s!\n",
4860                                 hns3_hw_err[i].msg);
4861                         break;
4862                 }
4863         }
4864 }
4865
4866 static const struct hnae3_client_ops client_ops = {
4867         .init_instance = hns3_client_init,
4868         .uninit_instance = hns3_client_uninit,
4869         .link_status_change = hns3_link_status_change,
4870         .reset_notify = hns3_reset_notify,
4871         .process_hw_error = hns3_process_hw_error,
4872 };
4873
4874 /* hns3_init_module - Driver registration routine
4875  * hns3_init_module is the first routine called when the driver is
4876  * loaded. All it does is register with the PCI subsystem.
4877  */
4878 static int __init hns3_init_module(void)
4879 {
4880         int ret;
4881
4882         pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
4883         pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
4884
4885         client.type = HNAE3_CLIENT_KNIC;
4886         snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH, "%s",
4887                  hns3_driver_name);
4888
4889         client.ops = &client_ops;
4890
4891         INIT_LIST_HEAD(&client.node);
4892
4893         hns3_dbg_register_debugfs(hns3_driver_name);
4894
4895         ret = hnae3_register_client(&client);
4896         if (ret)
4897                 goto err_reg_client;
4898
4899         ret = pci_register_driver(&hns3_driver);
4900         if (ret)
4901                 goto err_reg_driver;
4902
4903         return ret;
4904
4905 err_reg_driver:
4906         hnae3_unregister_client(&client);
4907 err_reg_client:
4908         hns3_dbg_unregister_debugfs();
4909         return ret;
4910 }
4911 module_init(hns3_init_module);
4912
4913 /* hns3_exit_module - Driver exit cleanup routine
4914  * hns3_exit_module is called just before the driver is removed
4915  * from memory.
4916  */
4917 static void __exit hns3_exit_module(void)
4918 {
4919         pci_unregister_driver(&hns3_driver);
4920         hnae3_unregister_client(&client);
4921         hns3_dbg_unregister_debugfs();
4922 }
4923 module_exit(hns3_exit_module);
4924
4925 MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
4926 MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
4927 MODULE_LICENSE("GPL");
4928 MODULE_ALIAS("pci:hns-nic");