drivers/net/ethernet/mellanox/mlx5/core/en_main.c

   1 /*
   2  * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <net/tc_act/tc_gact.h>
  34 #include <net/pkt_cls.h>
  35 #include <linux/mlx5/fs.h>
  36 #include <net/vxlan.h>
  37 #include <net/geneve.h>
  38 #include <linux/bpf.h>
  39 #include <linux/if_bridge.h>
  40 #include <net/page_pool.h>
  41 #include <net/xdp_sock_drv.h>
  42 #include "eswitch.h"
  43 #include "en.h"
  44 #include "en/txrx.h"
  45 #include "en_tc.h"
  46 #include "en_rep.h"
  47 #include "en_accel/ipsec.h"
  48 #include "en_accel/en_accel.h"
  49 #include "en_accel/tls.h"
  50 #include "accel/ipsec.h"
  51 #include "accel/tls.h"
  52 #include "lib/vxlan.h"
  53 #include "lib/clock.h"
  54 #include "en/port.h"
  55 #include "en/xdp.h"
  56 #include "lib/eq.h"
  57 #include "en/monitor_stats.h"
  58 #include "en/health.h"
  59 #include "en/params.h"
  60 #include "en/xsk/pool.h"
  61 #include "en/xsk/setup.h"
  62 #include "en/xsk/rx.h"
  63 #include "en/xsk/tx.h"
  64 #include "en/hv_vhca_stats.h"
  65 #include "en/devlink.h"
  66 #include "lib/mlx5.h"
  67 #include "en/ptp.h"
  68 #include "qos.h"
  69 #include "en/trap.h"
  70
  71 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
  72 {
  73         bool striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) &&
  74                 MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
  75                 MLX5_CAP_ETH(mdev, reg_umr_sq);
  76         u16 max_wqe_sz_cap = MLX5_CAP_GEN(mdev, max_wqe_sz_sq);
  77         bool inline_umr = MLX5E_UMR_WQE_INLINE_SZ <= max_wqe_sz_cap;
  78
  79         if (!striding_rq_umr)
  80                 return false;
  81         if (!inline_umr) {
  82                 mlx5_core_warn(mdev, "Cannot support Striding RQ: UMR WQE size (%d) exceeds maximum supported (%d).\n",
  83                                (int)MLX5E_UMR_WQE_INLINE_SZ, max_wqe_sz_cap);
  84                 return false;
  85         }
  86         return true;
  87 }
  88
  89 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
  90                                struct mlx5e_params *params)
  91 {
  92         params->log_rq_mtu_frames = is_kdump_kernel() ?
  93                 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE :
  94                 MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
  95
  96         mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
  97                        params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
  98                        params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ?
  99                        BIT(mlx5e_mpwqe_get_log_rq_size(params, NULL)) :
 100                        BIT(params->log_rq_mtu_frames),
 101                        BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL)),
 102                        MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
 103 }
 104
 105 bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev,
 106                                 struct mlx5e_params *params)
 107 {
 108         if (!mlx5e_check_fragmented_striding_rq_cap(mdev))
 109                 return false;
 110
 111         if (MLX5_IPSEC_DEV(mdev))
 112                 return false;
 113
 114         if (params->xdp_prog) {
 115                 /* XSK params are not considered here. If striding RQ is in use,
 116                  * and an XSK is being opened, mlx5e_rx_mpwqe_is_linear_skb will
 117                  * be called with the known XSK params.
 118                  */
 119                 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL))
 120                         return false;
 121         }
 122
 123         return true;
 124 }
 125
 126 void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
 127 {
 128         params->rq_wq_type = mlx5e_striding_rq_possible(mdev, params) &&
 129                 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) ?
 130                 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
 131                 MLX5_WQ_TYPE_CYCLIC;
 132 }
 133
 134 void mlx5e_update_carrier(struct mlx5e_priv *priv)
 135 {
 136         struct mlx5_core_dev *mdev = priv->mdev;
 137         u8 port_state;
 138
 139         port_state = mlx5_query_vport_state(mdev,
 140                                             MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT,
 141                                             0);
 142
 143         if (port_state == VPORT_STATE_UP) {
 144                 netdev_info(priv->netdev, "Link up\n");
 145                 netif_carrier_on(priv->netdev);
 146         } else {
 147                 netdev_info(priv->netdev, "Link down\n");
 148                 netif_carrier_off(priv->netdev);
 149         }
 150 }
 151
 152 static void mlx5e_update_carrier_work(struct work_struct *work)
 153 {
 154         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 155                                                update_carrier_work);
 156
 157         mutex_lock(&priv->state_lock);
 158         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
 159                 if (priv->profile->update_carrier)
 160                         priv->profile->update_carrier(priv);
 161         mutex_unlock(&priv->state_lock);
 162 }
 163
 164 static void mlx5e_update_stats_work(struct work_struct *work)
 165 {
 166         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 167                                                update_stats_work);
 168
 169         mutex_lock(&priv->state_lock);
 170         priv->profile->update_stats(priv);
 171         mutex_unlock(&priv->state_lock);
 172 }
 173
 174 void mlx5e_queue_update_stats(struct mlx5e_priv *priv)
 175 {
 176         if (!priv->profile->update_stats)
 177                 return;
 178
 179         if (unlikely(test_bit(MLX5E_STATE_DESTROYING, &priv->state)))
 180                 return;
 181
 182         queue_work(priv->wq, &priv->update_stats_work);
 183 }
 184
 185 static int async_event(struct notifier_block *nb, unsigned long event, void *data)
 186 {
 187         struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb);
 188         struct mlx5_eqe   *eqe = data;
 189
 190         if (event != MLX5_EVENT_TYPE_PORT_CHANGE)
 191                 return NOTIFY_DONE;
 192
 193         switch (eqe->sub_type) {
 194         case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
 195         case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
 196                 queue_work(priv->wq, &priv->update_carrier_work);
 197                 break;
 198         default:
 199                 return NOTIFY_DONE;
 200         }
 201
 202         return NOTIFY_OK;
 203 }
 204
 205 static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
 206 {
 207         priv->events_nb.notifier_call = async_event;
 208         mlx5_notifier_register(priv->mdev, &priv->events_nb);
 209 }
 210
 211 static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
 212 {
 213         mlx5_notifier_unregister(priv->mdev, &priv->events_nb);
 214 }
 215
 216 static int blocking_event(struct notifier_block *nb, unsigned long event, void *data)
 217 {
 218         struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, blocking_events_nb);
 219         int err;
 220
 221         switch (event) {
 222         case MLX5_DRIVER_EVENT_TYPE_TRAP:
 223                 err = mlx5e_handle_trap_event(priv, data);
 224                 break;
 225         default:
 226                 netdev_warn(priv->netdev, "Sync event: Unknouwn event %ld\n", event);
 227                 err = -EINVAL;
 228         }
 229         return err;
 230 }
 231
 232 static void mlx5e_enable_blocking_events(struct mlx5e_priv *priv)
 233 {
 234         priv->blocking_events_nb.notifier_call = blocking_event;
 235         mlx5_blocking_notifier_register(priv->mdev, &priv->blocking_events_nb);
 236 }
 237
 238 static void mlx5e_disable_blocking_events(struct mlx5e_priv *priv)
 239 {
 240         mlx5_blocking_notifier_unregister(priv->mdev, &priv->blocking_events_nb);
 241 }
 242
 243 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
 244                                        struct mlx5e_icosq *sq,
 245                                        struct mlx5e_umr_wqe *wqe)
 246 {
 247         struct mlx5_wqe_ctrl_seg      *cseg = &wqe->ctrl;
 248         struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
 249         u8 ds_cnt = DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_DS);
 250
 251         cseg->qpn_ds    = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
 252                                       ds_cnt);
 253         cseg->umr_mkey  = rq->mkey_be;
 254
 255         ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE;
 256         ucseg->xlt_octowords =
 257                 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
 258         ucseg->mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);
 259 }
 260
 261 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq,
 262                                      struct mlx5e_channel *c)
 263 {
 264         int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
 265
 266         rq->mpwqe.info = kvzalloc_node(array_size(wq_sz,
 267                                                   sizeof(*rq->mpwqe.info)),
 268                                        GFP_KERNEL, cpu_to_node(c->cpu));
 269         if (!rq->mpwqe.info)
 270                 return -ENOMEM;
 271
 272         mlx5e_build_umr_wqe(rq, &c->icosq, &rq->mpwqe.umr_wqe);
 273
 274         return 0;
 275 }
 276
 277 static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
 278                                  u64 npages, u8 page_shift,
 279                                  struct mlx5_core_mkey *umr_mkey,
 280                                  dma_addr_t filler_addr)
 281 {
 282         struct mlx5_mtt *mtt;
 283         int inlen;
 284         void *mkc;
 285         u32 *in;
 286         int err;
 287         int i;
 288
 289         inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + sizeof(*mtt) * npages;
 290
 291         in = kvzalloc(inlen, GFP_KERNEL);
 292         if (!in)
 293                 return -ENOMEM;
 294
 295         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 296
 297         MLX5_SET(mkc, mkc, free, 1);
 298         MLX5_SET(mkc, mkc, umr_en, 1);
 299         MLX5_SET(mkc, mkc, lw, 1);
 300         MLX5_SET(mkc, mkc, lr, 1);
 301         MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
 302         mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
 303         MLX5_SET(mkc, mkc, qpn, 0xffffff);
 304         MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn);
 305         MLX5_SET64(mkc, mkc, len, npages << page_shift);
 306         MLX5_SET(mkc, mkc, translations_octword_size,
 307                  MLX5_MTT_OCTW(npages));
 308         MLX5_SET(mkc, mkc, log_page_size, page_shift);
 309         MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
 310                  MLX5_MTT_OCTW(npages));
 311
 312         /* Initialize the mkey with all MTTs pointing to a default
 313          * page (filler_addr). When the channels are activated, UMR
 314          * WQEs will redirect the RX WQEs to the actual memory from
 315          * the RQ's pool, while the gaps (wqe_overflow) remain mapped
 316          * to the default page.
 317          */
 318         mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
 319         for (i = 0 ; i < npages ; i++)
 320                 mtt[i].ptag = cpu_to_be64(filler_addr);
 321
 322         err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
 323
 324         kvfree(in);
 325         return err;
 326 }
 327
 328 static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq)
 329 {
 330         u64 num_mtts = MLX5E_REQUIRED_MTTS(mlx5_wq_ll_get_size(&rq->mpwqe.wq));
 331
 332         return mlx5e_create_umr_mkey(mdev, num_mtts, PAGE_SHIFT, &rq->umr_mkey,
 333                                      rq->wqe_overflow.addr);
 334 }
 335
 336 static inline u64 mlx5e_get_mpwqe_offset(struct mlx5e_rq *rq, u16 wqe_ix)
 337 {
 338         return (wqe_ix << MLX5E_LOG_ALIGNED_MPWQE_PPW) << PAGE_SHIFT;
 339 }
 340
 341 static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
 342 {
 343         struct mlx5e_wqe_frag_info next_frag = {};
 344         struct mlx5e_wqe_frag_info *prev = NULL;
 345         int i;
 346
 347         next_frag.di = &rq->wqe.di[0];
 348
 349         for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
 350                 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
 351                 struct mlx5e_wqe_frag_info *frag =
 352                         &rq->wqe.frags[i << rq->wqe.info.log_num_frags];
 353                 int f;
 354
 355                 for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) {
 356                         if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) {
 357                                 next_frag.di++;
 358                                 next_frag.offset = 0;
 359                                 if (prev)
 360                                         prev->last_in_page = true;
 361                         }
 362                         *frag = next_frag;
 363
 364                         /* prepare next */
 365                         next_frag.offset += frag_info[f].frag_stride;
 366                         prev = frag;
 367                 }
 368         }
 369
 370         if (prev)
 371                 prev->last_in_page = true;
 372 }
 373
 374 int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node)
 375 {
 376         int len = wq_sz << rq->wqe.info.log_num_frags;
 377
 378         rq->wqe.di = kvzalloc_node(array_size(len, sizeof(*rq->wqe.di)), GFP_KERNEL, node);
 379         if (!rq->wqe.di)
 380                 return -ENOMEM;
 381
 382         mlx5e_init_frags_partition(rq);
 383
 384         return 0;
 385 }
 386
 387 void mlx5e_free_di_list(struct mlx5e_rq *rq)
 388 {
 389         kvfree(rq->wqe.di);
 390 }
 391
 392 static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work)
 393 {
 394         struct mlx5e_rq *rq = container_of(recover_work, struct mlx5e_rq, recover_work);
 395
 396         mlx5e_reporter_rq_cqe_err(rq);
 397 }
 398
 399 static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
 400 {
 401         rq->wqe_overflow.page = alloc_page(GFP_KERNEL);
 402         if (!rq->wqe_overflow.page)
 403                 return -ENOMEM;
 404
 405         rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0,
 406                                              PAGE_SIZE, rq->buff.map_dir);
 407         if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) {
 408                 __free_page(rq->wqe_overflow.page);
 409                 return -ENOMEM;
 410         }
 411         return 0;
 412 }
 413
 414 static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
 415 {
 416          dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE,
 417                         rq->buff.map_dir);
 418          __free_page(rq->wqe_overflow.page);
 419 }
 420
 421 static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 422                           struct mlx5e_params *params,
 423                           struct mlx5e_xsk_param *xsk,
 424                           struct xsk_buff_pool *xsk_pool,
 425                           struct mlx5e_rq_param *rqp,
 426                           struct mlx5e_rq *rq)
 427 {
 428         struct page_pool_params pp_params = { 0 };
 429         struct mlx5_core_dev *mdev = c->mdev;
 430         void *rqc = rqp->rqc;
 431         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
 432         u32 rq_xdp_ix;
 433         u32 pool_size;
 434         int wq_sz;
 435         int err;
 436         int i;
 437
 438         rqp->wq.db_numa_node = cpu_to_node(c->cpu);
 439
 440         rq->wq_type = params->rq_wq_type;
 441         rq->pdev    = c->pdev;
 442         rq->netdev  = c->netdev;
 443         rq->priv    = c->priv;
 444         rq->tstamp  = c->tstamp;
 445         rq->clock   = &mdev->clock;
 446         rq->icosq   = &c->icosq;
 447         rq->ix      = c->ix;
 448         rq->mdev    = mdev;
 449         rq->hw_mtu  = MLX5E_SW2HW_MTU(params, params->sw_mtu);
 450         rq->xdpsq   = &c->rq_xdpsq;
 451         rq->xsk_pool = xsk_pool;
 452
 453         if (rq->xsk_pool)
 454                 rq->stats = &c->priv->channel_stats[c->ix].xskrq;
 455         else
 456                 rq->stats = &c->priv->channel_stats[c->ix].rq;
 457         INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work);
 458
 459         if (params->xdp_prog)
 460                 bpf_prog_inc(params->xdp_prog);
 461         RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog);
 462
 463         rq_xdp_ix = rq->ix;
 464         if (xsk)
 465                 rq_xdp_ix += params->num_channels * MLX5E_RQ_GROUP_XSK;
 466         err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix, 0);
 467         if (err < 0)
 468                 goto err_rq_xdp_prog;
 469
 470         rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
 471         rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk);
 472         pool_size = 1 << params->log_rq_mtu_frames;
 473
 474         switch (rq->wq_type) {
 475         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 476                 err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq,
 477                                         &rq->wq_ctrl);
 478                 if (err)
 479                         goto err_rq_xdp;
 480
 481                 err = mlx5e_alloc_mpwqe_rq_drop_page(rq);
 482                 if (err)
 483                         goto err_rq_wq_destroy;
 484
 485                 rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR];
 486
 487                 wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
 488
 489                 pool_size = MLX5_MPWRQ_PAGES_PER_WQE <<
 490                         mlx5e_mpwqe_get_log_rq_size(params, xsk);
 491
 492                 rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
 493                 rq->mpwqe.num_strides =
 494                         BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
 495
 496                 rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz);
 497
 498                 err = mlx5e_create_rq_umr_mkey(mdev, rq);
 499                 if (err)
 500                         goto err_rq_drop_page;
 501                 rq->mkey_be = cpu_to_be32(rq->umr_mkey.key);
 502
 503                 err = mlx5e_rq_alloc_mpwqe_info(rq, c);
 504                 if (err)
 505                         goto err_rq_mkey;
 506                 break;
 507         default: /* MLX5_WQ_TYPE_CYCLIC */
 508                 err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq,
 509                                          &rq->wq_ctrl);
 510                 if (err)
 511                         goto err_rq_xdp;
 512
 513                 rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR];
 514
 515                 wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
 516
 517                 rq->wqe.info = rqp->frags_info;
 518                 rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride;
 519
 520                 rq->wqe.frags =
 521                         kvzalloc_node(array_size(sizeof(*rq->wqe.frags),
 522                                         (wq_sz << rq->wqe.info.log_num_frags)),
 523                                       GFP_KERNEL, cpu_to_node(c->cpu));
 524                 if (!rq->wqe.frags) {
 525                         err = -ENOMEM;
 526                         goto err_rq_wq_destroy;
 527                 }
 528
 529                 err = mlx5e_init_di_list(rq, wq_sz, cpu_to_node(c->cpu));
 530                 if (err)
 531                         goto err_rq_frags;
 532
 533                 rq->mkey_be = c->mkey_be;
 534         }
 535
 536         err = mlx5e_rq_set_handlers(rq, params, xsk);
 537         if (err)
 538                 goto err_free_by_rq_type;
 539
 540         if (xsk) {
 541                 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
 542                                                  MEM_TYPE_XSK_BUFF_POOL, NULL);
 543                 xsk_pool_set_rxq_info(rq->xsk_pool, &rq->xdp_rxq);
 544         } else {
 545                 /* Create a page_pool and register it with rxq */
 546                 pp_params.order     = 0;
 547                 pp_params.flags     = 0; /* No-internal DMA mapping in page_pool */
 548                 pp_params.pool_size = pool_size;
 549                 pp_params.nid       = cpu_to_node(c->cpu);
 550                 pp_params.dev       = c->pdev;
 551                 pp_params.dma_dir   = rq->buff.map_dir;
 552
 553                 /* page_pool can be used even when there is no rq->xdp_prog,
 554                  * given page_pool does not handle DMA mapping there is no
 555                  * required state to clear. And page_pool gracefully handle
 556                  * elevated refcnt.
 557                  */
 558                 rq->page_pool = page_pool_create(&pp_params);
 559                 if (IS_ERR(rq->page_pool)) {
 560                         err = PTR_ERR(rq->page_pool);
 561                         rq->page_pool = NULL;
 562                         goto err_free_by_rq_type;
 563                 }
 564                 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
 565                                                  MEM_TYPE_PAGE_POOL, rq->page_pool);
 566         }
 567         if (err)
 568                 goto err_free_by_rq_type;
 569
 570         for (i = 0; i < wq_sz; i++) {
 571                 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
 572                         struct mlx5e_rx_wqe_ll *wqe =
 573                                 mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i);
 574                         u32 byte_count =
 575                                 rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz;
 576                         u64 dma_offset = mlx5e_get_mpwqe_offset(rq, i);
 577
 578                         wqe->data[0].addr = cpu_to_be64(dma_offset + rq->buff.headroom);
 579                         wqe->data[0].byte_count = cpu_to_be32(byte_count);
 580                         wqe->data[0].lkey = rq->mkey_be;
 581                 } else {
 582                         struct mlx5e_rx_wqe_cyc *wqe =
 583                                 mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i);
 584                         int f;
 585
 586                         for (f = 0; f < rq->wqe.info.num_frags; f++) {
 587                                 u32 frag_size = rq->wqe.info.arr[f].frag_size |
 588                                         MLX5_HW_START_PADDING;
 589
 590                                 wqe->data[f].byte_count = cpu_to_be32(frag_size);
 591                                 wqe->data[f].lkey = rq->mkey_be;
 592                         }
 593                         /* check if num_frags is not a pow of two */
 594                         if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) {
 595                                 wqe->data[f].byte_count = 0;
 596                                 wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
 597                                 wqe->data[f].addr = 0;
 598                         }
 599                 }
 600         }
 601
 602         INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work);
 603
 604         switch (params->rx_cq_moderation.cq_period_mode) {
 605         case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
 606                 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
 607                 break;
 608         case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
 609         default:
 610                 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 611         }
 612
 613         rq->page_cache.head = 0;
 614         rq->page_cache.tail = 0;
 615
 616         return 0;
 617
 618 err_free_by_rq_type:
 619         switch (rq->wq_type) {
 620         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 621                 kvfree(rq->mpwqe.info);
 622 err_rq_mkey:
 623                 mlx5_core_destroy_mkey(mdev, &rq->umr_mkey);
 624 err_rq_drop_page:
 625                 mlx5e_free_mpwqe_rq_drop_page(rq);
 626                 break;
 627         default: /* MLX5_WQ_TYPE_CYCLIC */
 628                 mlx5e_free_di_list(rq);
 629 err_rq_frags:
 630                 kvfree(rq->wqe.frags);
 631         }
 632 err_rq_wq_destroy:
 633         mlx5_wq_destroy(&rq->wq_ctrl);
 634 err_rq_xdp:
 635         xdp_rxq_info_unreg(&rq->xdp_rxq);
 636 err_rq_xdp_prog:
 637         if (params->xdp_prog)
 638                 bpf_prog_put(params->xdp_prog);
 639
 640         return err;
 641 }
 642
 643 static void mlx5e_free_rq(struct mlx5e_rq *rq)
 644 {
 645         struct bpf_prog *old_prog;
 646         int i;
 647
 648         old_prog = rcu_dereference_protected(rq->xdp_prog,
 649                                              lockdep_is_held(&rq->priv->state_lock));
 650         if (old_prog)
 651                 bpf_prog_put(old_prog);
 652
 653         switch (rq->wq_type) {
 654         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 655                 kvfree(rq->mpwqe.info);
 656                 mlx5_core_destroy_mkey(rq->mdev, &rq->umr_mkey);
 657                 mlx5e_free_mpwqe_rq_drop_page(rq);
 658                 break;
 659         default: /* MLX5_WQ_TYPE_CYCLIC */
 660                 kvfree(rq->wqe.frags);
 661                 mlx5e_free_di_list(rq);
 662         }
 663
 664         for (i = rq->page_cache.head; i != rq->page_cache.tail;
 665              i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
 666                 struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];
 667
 668                 /* With AF_XDP, page_cache is not used, so this loop is not
 669                  * entered, and it's safe to call mlx5e_page_release_dynamic
 670                  * directly.
 671                  */
 672                 mlx5e_page_release_dynamic(rq, dma_info, false);
 673         }
 674
 675         xdp_rxq_info_unreg(&rq->xdp_rxq);
 676         page_pool_destroy(rq->page_pool);
 677         mlx5_wq_destroy(&rq->wq_ctrl);
 678 }
 679
 680 int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
 681 {
 682         struct mlx5_core_dev *mdev = rq->mdev;
 683
 684         void *in;
 685         void *rqc;
 686         void *wq;
 687         int inlen;
 688         int err;
 689
 690         inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
 691                 sizeof(u64) * rq->wq_ctrl.buf.npages;
 692         in = kvzalloc(inlen, GFP_KERNEL);
 693         if (!in)
 694                 return -ENOMEM;
 695
 696         rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 697         wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 698
 699         memcpy(rqc, param->rqc, sizeof(param->rqc));
 700
 701         MLX5_SET(rqc,  rqc, cqn,                rq->cq.mcq.cqn);
 702         MLX5_SET(rqc,  rqc, state,              MLX5_RQC_STATE_RST);
 703         MLX5_SET(wq,   wq,  log_wq_pg_sz,       rq->wq_ctrl.buf.page_shift -
 704                                                 MLX5_ADAPTER_PAGE_SHIFT);
 705         MLX5_SET64(wq, wq,  dbr_addr,           rq->wq_ctrl.db.dma);
 706
 707         mlx5_fill_page_frag_array(&rq->wq_ctrl.buf,
 708                                   (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
 709
 710         err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
 711
 712         kvfree(in);
 713
 714         return err;
 715 }
 716
 717 int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state)
 718 {
 719         struct mlx5_core_dev *mdev = rq->mdev;
 720
 721         void *in;
 722         void *rqc;
 723         int inlen;
 724         int err;
 725
 726         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 727         in = kvzalloc(inlen, GFP_KERNEL);
 728         if (!in)
 729                 return -ENOMEM;
 730
 731         if (curr_state == MLX5_RQC_STATE_RST && next_state == MLX5_RQC_STATE_RDY)
 732                 mlx5e_rqwq_reset(rq);
 733
 734         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 735
 736         MLX5_SET(modify_rq_in, in, rq_state, curr_state);
 737         MLX5_SET(rqc, rqc, state, next_state);
 738
 739         err = mlx5_core_modify_rq(mdev, rq->rqn, in);
 740
 741         kvfree(in);
 742
 743         return err;
 744 }
 745
 746 static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq *rq, bool enable)
 747 {
 748         struct mlx5_core_dev *mdev = rq->mdev;
 749
 750         void *in;
 751         void *rqc;
 752         int inlen;
 753         int err;
 754
 755         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 756         in = kvzalloc(inlen, GFP_KERNEL);
 757         if (!in)
 758                 return -ENOMEM;
 759
 760         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 761
 762         MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
 763         MLX5_SET64(modify_rq_in, in, modify_bitmask,
 764                    MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS);
 765         MLX5_SET(rqc, rqc, scatter_fcs, enable);
 766         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
 767
 768         err = mlx5_core_modify_rq(mdev, rq->rqn, in);
 769
 770         kvfree(in);
 771
 772         return err;
 773 }
 774
 775 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
 776 {
 777         struct mlx5_core_dev *mdev = rq->mdev;
 778         void *in;
 779         void *rqc;
 780         int inlen;
 781         int err;
 782
 783         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 784         in = kvzalloc(inlen, GFP_KERNEL);
 785         if (!in)
 786                 return -ENOMEM;
 787
 788         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 789
 790         MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
 791         MLX5_SET64(modify_rq_in, in, modify_bitmask,
 792                    MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
 793         MLX5_SET(rqc, rqc, vsd, vsd);
 794         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
 795
 796         err = mlx5_core_modify_rq(mdev, rq->rqn, in);
 797
 798         kvfree(in);
 799
 800         return err;
 801 }
 802
 803 void mlx5e_destroy_rq(struct mlx5e_rq *rq)
 804 {
 805         mlx5_core_destroy_rq(rq->mdev, rq->rqn);
 806 }
 807
 808 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time)
 809 {
 810         unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time);
 811
 812         u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq));
 813
 814         do {
 815                 if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes)
 816                         return 0;
 817
 818                 msleep(20);
 819         } while (time_before(jiffies, exp_time));
 820
 821         netdev_warn(rq->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n",
 822                     rq->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes);
 823
 824         mlx5e_reporter_rx_timeout(rq);
 825         return -ETIMEDOUT;
 826 }
 827
 828 void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq)
 829 {
 830         struct mlx5_wq_ll *wq;
 831         u16 head;
 832         int i;
 833
 834         if (rq->wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
 835                 return;
 836
 837         wq = &rq->mpwqe.wq;
 838         head = wq->head;
 839
 840         /* Outstanding UMR WQEs (in progress) start at wq->head */
 841         for (i = 0; i < rq->mpwqe.umr_in_progress; i++) {
 842                 rq->dealloc_wqe(rq, head);
 843                 head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
 844         }
 845
 846         rq->mpwqe.actual_wq_head = wq->head;
 847         rq->mpwqe.umr_in_progress = 0;
 848         rq->mpwqe.umr_completed = 0;
 849 }
 850
 851 void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
 852 {
 853         __be16 wqe_ix_be;
 854         u16 wqe_ix;
 855
 856         if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
 857                 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
 858
 859                 mlx5e_free_rx_in_progress_descs(rq);
 860
 861                 while (!mlx5_wq_ll_is_empty(wq)) {
 862                         struct mlx5e_rx_wqe_ll *wqe;
 863
 864                         wqe_ix_be = *wq->tail_next;
 865                         wqe_ix    = be16_to_cpu(wqe_ix_be);
 866                         wqe       = mlx5_wq_ll_get_wqe(wq, wqe_ix);
 867                         rq->dealloc_wqe(rq, wqe_ix);
 868                         mlx5_wq_ll_pop(wq, wqe_ix_be,
 869                                        &wqe->next.next_wqe_index);
 870                 }
 871         } else {
 872                 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
 873
 874                 while (!mlx5_wq_cyc_is_empty(wq)) {
 875                         wqe_ix = mlx5_wq_cyc_get_tail(wq);
 876                         rq->dealloc_wqe(rq, wqe_ix);
 877                         mlx5_wq_cyc_pop(wq);
 878                 }
 879         }
 880
 881 }
 882
 883 int mlx5e_open_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
 884                   struct mlx5e_rq_param *param, struct mlx5e_xsk_param *xsk,
 885                   struct xsk_buff_pool *xsk_pool, struct mlx5e_rq *rq)
 886 {
 887         int err;
 888
 889         err = mlx5e_alloc_rq(c, params, xsk, xsk_pool, param, rq);
 890         if (err)
 891                 return err;
 892
 893         err = mlx5e_create_rq(rq, param);
 894         if (err)
 895                 goto err_free_rq;
 896
 897         err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
 898         if (err)
 899                 goto err_destroy_rq;
 900
 901         if (mlx5e_is_tls_on(c->priv) && !mlx5_accel_is_ktls_device(c->mdev))
 902                 __set_bit(MLX5E_RQ_STATE_FPGA_TLS, &c->rq.state); /* must be FPGA */
 903
 904         if (MLX5_CAP_ETH(c->mdev, cqe_checksum_full))
 905                 __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &c->rq.state);
 906
 907         if (params->rx_dim_enabled)
 908                 __set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
 909
 910         /* We disable csum_complete when XDP is enabled since
 911          * XDP programs might manipulate packets which will render
 912          * skb->checksum incorrect.
 913          */
 914         if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || c->xdp)
 915                 __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &c->rq.state);
 916
 917         /* For CQE compression on striding RQ, use stride index provided by
 918          * HW if capability is supported.
 919          */
 920         if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) &&
 921             MLX5_CAP_GEN(c->mdev, mini_cqe_resp_stride_index))
 922                 __set_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &c->rq.state);
 923
 924         return 0;
 925
 926 err_destroy_rq:
 927         mlx5e_destroy_rq(rq);
 928 err_free_rq:
 929         mlx5e_free_rq(rq);
 930
 931         return err;
 932 }
 933
 934 void mlx5e_activate_rq(struct mlx5e_rq *rq)
 935 {
 936         set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
 937         mlx5e_trigger_irq(rq->icosq);
 938 }
 939
 940 void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
 941 {
 942         clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
 943         synchronize_rcu(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */
 944 }
 945
 946 void mlx5e_close_rq(struct mlx5e_rq *rq)
 947 {
 948         cancel_work_sync(&rq->dim.work);
 949         cancel_work_sync(&rq->icosq->recover_work);
 950         cancel_work_sync(&rq->recover_work);
 951         mlx5e_destroy_rq(rq);
 952         mlx5e_free_rx_descs(rq);
 953         mlx5e_free_rq(rq);
 954 }
 955
 956 static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq)
 957 {
 958         kvfree(sq->db.xdpi_fifo.xi);
 959         kvfree(sq->db.wqe_info);
 960 }
 961
 962 static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa)
 963 {
 964         struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo;
 965         int wq_sz        = mlx5_wq_cyc_get_size(&sq->wq);
 966         int dsegs_per_wq = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
 967
 968         xdpi_fifo->xi = kvzalloc_node(sizeof(*xdpi_fifo->xi) * dsegs_per_wq,
 969                                       GFP_KERNEL, numa);
 970         if (!xdpi_fifo->xi)
 971                 return -ENOMEM;
 972
 973         xdpi_fifo->pc   = &sq->xdpi_fifo_pc;
 974         xdpi_fifo->cc   = &sq->xdpi_fifo_cc;
 975         xdpi_fifo->mask = dsegs_per_wq - 1;
 976
 977         return 0;
 978 }
 979
 980 static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa)
 981 {
 982         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
 983         int err;
 984
 985         sq->db.wqe_info = kvzalloc_node(sizeof(*sq->db.wqe_info) * wq_sz,
 986                                         GFP_KERNEL, numa);
 987         if (!sq->db.wqe_info)
 988                 return -ENOMEM;
 989
 990         err = mlx5e_alloc_xdpsq_fifo(sq, numa);
 991         if (err) {
 992                 mlx5e_free_xdpsq_db(sq);
 993                 return err;
 994         }
 995
 996         return 0;
 997 }
 998
 999 static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
1000                              struct mlx5e_params *params,
1001                              struct xsk_buff_pool *xsk_pool,
1002                              struct mlx5e_sq_param *param,
1003                              struct mlx5e_xdpsq *sq,
1004                              bool is_redirect)
1005 {
1006         void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
1007         struct mlx5_core_dev *mdev = c->mdev;
1008         struct mlx5_wq_cyc *wq = &sq->wq;
1009         int err;
1010
1011         sq->pdev      = c->pdev;
1012         sq->mkey_be   = c->mkey_be;
1013         sq->channel   = c;
1014         sq->uar_map   = mdev->mlx5e_res.bfreg.map;
1015         sq->min_inline_mode = params->tx_min_inline_mode;
1016         sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu);
1017         sq->xsk_pool  = xsk_pool;
1018
1019         sq->stats = sq->xsk_pool ?
1020                 &c->priv->channel_stats[c->ix].xsksq :
1021                 is_redirect ?
1022                         &c->priv->channel_stats[c->ix].xdpsq :
1023                         &c->priv->channel_stats[c->ix].rq_xdpsq;
1024
1025         param->wq.db_numa_node = cpu_to_node(c->cpu);
1026         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
1027         if (err)
1028                 return err;
1029         wq->db = &wq->db[MLX5_SND_DBR];
1030
1031         err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu));
1032         if (err)
1033                 goto err_sq_wq_destroy;
1034
1035         return 0;
1036
1037 err_sq_wq_destroy:
1038         mlx5_wq_destroy(&sq->wq_ctrl);
1039
1040         return err;
1041 }
1042
1043 static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq)
1044 {
1045         mlx5e_free_xdpsq_db(sq);
1046         mlx5_wq_destroy(&sq->wq_ctrl);
1047 }
1048
1049 static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq)
1050 {
1051         kvfree(sq->db.wqe_info);
1052 }
1053
1054 static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)
1055 {
1056         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1057         size_t size;
1058
1059         size = array_size(wq_sz, sizeof(*sq->db.wqe_info));
1060         sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa);
1061         if (!sq->db.wqe_info)
1062                 return -ENOMEM;
1063
1064         return 0;
1065 }
1066
1067 static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work)
1068 {
1069         struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq,
1070                                               recover_work);
1071
1072         mlx5e_reporter_icosq_cqe_err(sq);
1073 }
1074
1075 static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
1076                              struct mlx5e_sq_param *param,
1077                              struct mlx5e_icosq *sq)
1078 {
1079         void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
1080         struct mlx5_core_dev *mdev = c->mdev;
1081         struct mlx5_wq_cyc *wq = &sq->wq;
1082         int err;
1083
1084         sq->channel   = c;
1085         sq->uar_map   = mdev->mlx5e_res.bfreg.map;
1086
1087         param->wq.db_numa_node = cpu_to_node(c->cpu);
1088         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
1089         if (err)
1090                 return err;
1091         wq->db = &wq->db[MLX5_SND_DBR];
1092
1093         err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu));
1094         if (err)
1095                 goto err_sq_wq_destroy;
1096
1097         INIT_WORK(&sq->recover_work, mlx5e_icosq_err_cqe_work);
1098
1099         return 0;
1100
1101 err_sq_wq_destroy:
1102         mlx5_wq_destroy(&sq->wq_ctrl);
1103
1104         return err;
1105 }
1106
1107 static void mlx5e_free_icosq(struct mlx5e_icosq *sq)
1108 {
1109         mlx5e_free_icosq_db(sq);
1110         mlx5_wq_destroy(&sq->wq_ctrl);
1111 }
1112
1113 void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq)
1114 {
1115         kvfree(sq->db.wqe_info);
1116         kvfree(sq->db.skb_fifo.fifo);
1117         kvfree(sq->db.dma_fifo);
1118 }
1119
1120 int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa)
1121 {
1122         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1123         int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
1124
1125         sq->db.dma_fifo = kvzalloc_node(array_size(df_sz,
1126                                                    sizeof(*sq->db.dma_fifo)),
1127                                         GFP_KERNEL, numa);
1128         sq->db.skb_fifo.fifo = kvzalloc_node(array_size(df_sz,
1129                                                         sizeof(*sq->db.skb_fifo.fifo)),
1130                                         GFP_KERNEL, numa);
1131         sq->db.wqe_info = kvzalloc_node(array_size(wq_sz,
1132                                                    sizeof(*sq->db.wqe_info)),
1133                                         GFP_KERNEL, numa);
1134         if (!sq->db.dma_fifo || !sq->db.skb_fifo.fifo || !sq->db.wqe_info) {
1135                 mlx5e_free_txqsq_db(sq);
1136                 return -ENOMEM;
1137         }
1138
1139         sq->dma_fifo_mask = df_sz - 1;
1140
1141         sq->db.skb_fifo.pc   = &sq->skb_fifo_pc;
1142         sq->db.skb_fifo.cc   = &sq->skb_fifo_cc;
1143         sq->db.skb_fifo.mask = df_sz - 1;
1144
1145         return 0;
1146 }
1147
1148 static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
1149                              int txq_ix,
1150                              struct mlx5e_params *params,
1151                              struct mlx5e_sq_param *param,
1152                              struct mlx5e_txqsq *sq,
1153                              int tc)
1154 {
1155         void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
1156         struct mlx5_core_dev *mdev = c->mdev;
1157         struct mlx5_wq_cyc *wq = &sq->wq;
1158         int err;
1159
1160         sq->pdev      = c->pdev;
1161         sq->tstamp    = c->tstamp;
1162         sq->clock     = &mdev->clock;
1163         sq->mkey_be   = c->mkey_be;
1164         sq->netdev    = c->netdev;
1165         sq->mdev      = c->mdev;
1166         sq->priv      = c->priv;
1167         sq->ch_ix     = c->ix;
1168         sq->txq_ix    = txq_ix;
1169         sq->uar_map   = mdev->mlx5e_res.bfreg.map;
1170         sq->min_inline_mode = params->tx_min_inline_mode;
1171         sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu);
1172         INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work);
1173         if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert))
1174                 set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state);
1175         if (MLX5_IPSEC_DEV(c->priv->mdev))
1176                 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state);
1177         if (mlx5_accel_is_tls_device(c->priv->mdev))
1178                 set_bit(MLX5E_SQ_STATE_TLS, &sq->state);
1179         if (param->is_mpw)
1180                 set_bit(MLX5E_SQ_STATE_MPWQE, &sq->state);
1181         sq->stop_room = param->stop_room;
1182
1183         param->wq.db_numa_node = cpu_to_node(c->cpu);
1184         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
1185         if (err)
1186                 return err;
1187         wq->db    = &wq->db[MLX5_SND_DBR];
1188
1189         err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu));
1190         if (err)
1191                 goto err_sq_wq_destroy;
1192
1193         INIT_WORK(&sq->dim.work, mlx5e_tx_dim_work);
1194         sq->dim.mode = params->tx_cq_moderation.cq_period_mode;
1195
1196         return 0;
1197
1198 err_sq_wq_destroy:
1199         mlx5_wq_destroy(&sq->wq_ctrl);
1200
1201         return err;
1202 }
1203
1204 void mlx5e_free_txqsq(struct mlx5e_txqsq *sq)
1205 {
1206         mlx5e_free_txqsq_db(sq);
1207         mlx5_wq_destroy(&sq->wq_ctrl);
1208 }
1209
1210 static int mlx5e_create_sq(struct mlx5_core_dev *mdev,
1211                            struct mlx5e_sq_param *param,
1212                            struct mlx5e_create_sq_param *csp,
1213                            u32 *sqn)
1214 {
1215         void *in;
1216         void *sqc;
1217         void *wq;
1218         int inlen;
1219         int err;
1220
1221         inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1222                 sizeof(u64) * csp->wq_ctrl->buf.npages;
1223         in = kvzalloc(inlen, GFP_KERNEL);
1224         if (!in)
1225                 return -ENOMEM;
1226
1227         sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1228         wq = MLX5_ADDR_OF(sqc, sqc, wq);
1229
1230         memcpy(sqc, param->sqc, sizeof(param->sqc));
1231         MLX5_SET(sqc,  sqc, tis_lst_sz, csp->tis_lst_sz);
1232         MLX5_SET(sqc,  sqc, tis_num_0, csp->tisn);
1233         MLX5_SET(sqc,  sqc, cqn, csp->cqn);
1234         MLX5_SET(sqc,  sqc, ts_cqe_to_dest_cqn, csp->ts_cqe_to_dest_cqn);
1235
1236         if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT)
1237                 MLX5_SET(sqc,  sqc, min_wqe_inline_mode, csp->min_inline_mode);
1238
1239         MLX5_SET(sqc,  sqc, state, MLX5_SQC_STATE_RST);
1240         MLX5_SET(sqc,  sqc, flush_in_error_en, 1);
1241
1242         MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
1243         MLX5_SET(wq,   wq, uar_page,      mdev->mlx5e_res.bfreg.index);
1244         MLX5_SET(wq,   wq, log_wq_pg_sz,  csp->wq_ctrl->buf.page_shift -
1245                                           MLX5_ADAPTER_PAGE_SHIFT);
1246         MLX5_SET64(wq, wq, dbr_addr,      csp->wq_ctrl->db.dma);
1247
1248         mlx5_fill_page_frag_array(&csp->wq_ctrl->buf,
1249                                   (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
1250
1251         err = mlx5_core_create_sq(mdev, in, inlen, sqn);
1252
1253         kvfree(in);
1254
1255         return err;
1256 }
1257
1258 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
1259                     struct mlx5e_modify_sq_param *p)
1260 {
1261         u64 bitmask = 0;
1262         void *in;
1263         void *sqc;
1264         int inlen;
1265         int err;
1266
1267         inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1268         in = kvzalloc(inlen, GFP_KERNEL);
1269         if (!in)
1270                 return -ENOMEM;
1271
1272         sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1273
1274         MLX5_SET(modify_sq_in, in, sq_state, p->curr_state);
1275         MLX5_SET(sqc, sqc, state, p->next_state);
1276         if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) {
1277                 bitmask |= 1;
1278                 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index);
1279         }
1280         if (p->qos_update && p->next_state == MLX5_SQC_STATE_RDY) {
1281                 bitmask |= 1 << 2;
1282                 MLX5_SET(sqc, sqc, qos_queue_group_id, p->qos_queue_group_id);
1283         }
1284         MLX5_SET64(modify_sq_in, in, modify_bitmask, bitmask);
1285
1286         err = mlx5_core_modify_sq(mdev, sqn, in);
1287
1288         kvfree(in);
1289
1290         return err;
1291 }
1292
1293 static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn)
1294 {
1295         mlx5_core_destroy_sq(mdev, sqn);
1296 }
1297
1298 int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,
1299                         struct mlx5e_sq_param *param,
1300                         struct mlx5e_create_sq_param *csp,
1301                         u16 qos_queue_group_id,
1302                         u32 *sqn)
1303 {
1304         struct mlx5e_modify_sq_param msp = {0};
1305         int err;
1306
1307         err = mlx5e_create_sq(mdev, param, csp, sqn);
1308         if (err)
1309                 return err;
1310
1311         msp.curr_state = MLX5_SQC_STATE_RST;
1312         msp.next_state = MLX5_SQC_STATE_RDY;
1313         if (qos_queue_group_id) {
1314                 msp.qos_update = true;
1315                 msp.qos_queue_group_id = qos_queue_group_id;
1316         }
1317         err = mlx5e_modify_sq(mdev, *sqn, &msp);
1318         if (err)
1319                 mlx5e_destroy_sq(mdev, *sqn);
1320
1321         return err;
1322 }
1323
1324 static int mlx5e_set_sq_maxrate(struct net_device *dev,
1325                                 struct mlx5e_txqsq *sq, u32 rate);
1326
1327 int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix,
1328                      struct mlx5e_params *params, struct mlx5e_sq_param *param,
1329                      struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id, u16 qos_qid)
1330 {
1331         struct mlx5e_create_sq_param csp = {};
1332         u32 tx_rate;
1333         int err;
1334
1335         err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc);
1336         if (err)
1337                 return err;
1338
1339         if (qos_queue_group_id)
1340                 sq->stats = c->priv->htb.qos_sq_stats[qos_qid];
1341         else
1342                 sq->stats = &c->priv->channel_stats[c->ix].sq[tc];
1343
1344         csp.tisn            = tisn;
1345         csp.tis_lst_sz      = 1;
1346         csp.cqn             = sq->cq.mcq.cqn;
1347         csp.wq_ctrl         = &sq->wq_ctrl;
1348         csp.min_inline_mode = sq->min_inline_mode;
1349         err = mlx5e_create_sq_rdy(c->mdev, param, &csp, qos_queue_group_id, &sq->sqn);
1350         if (err)
1351                 goto err_free_txqsq;
1352
1353         tx_rate = c->priv->tx_rates[sq->txq_ix];
1354         if (tx_rate)
1355                 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate);
1356
1357         if (params->tx_dim_enabled)
1358                 sq->state |= BIT(MLX5E_SQ_STATE_AM);
1359
1360         return 0;
1361
1362 err_free_txqsq:
1363         mlx5e_free_txqsq(sq);
1364
1365         return err;
1366 }
1367
1368 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq)
1369 {
1370         sq->txq = netdev_get_tx_queue(sq->netdev, sq->txq_ix);
1371         set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1372         netdev_tx_reset_queue(sq->txq);
1373         netif_tx_start_queue(sq->txq);
1374 }
1375
1376 void mlx5e_tx_disable_queue(struct netdev_queue *txq)
1377 {
1378         __netif_tx_lock_bh(txq);
1379         netif_tx_stop_queue(txq);
1380         __netif_tx_unlock_bh(txq);
1381 }
1382
1383 void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
1384 {
1385         struct mlx5_wq_cyc *wq = &sq->wq;
1386
1387         clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1388         synchronize_rcu(); /* Sync with NAPI to prevent netif_tx_wake_queue. */
1389
1390         mlx5e_tx_disable_queue(sq->txq);
1391
1392         /* last doorbell out, godspeed .. */
1393         if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) {
1394                 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
1395                 struct mlx5e_tx_wqe *nop;
1396
1397                 sq->db.wqe_info[pi] = (struct mlx5e_tx_wqe_info) {
1398                         .num_wqebbs = 1,
1399                 };
1400
1401                 nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
1402                 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl);
1403         }
1404 }
1405
1406 void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
1407 {
1408         struct mlx5_core_dev *mdev = sq->mdev;
1409         struct mlx5_rate_limit rl = {0};
1410
1411         cancel_work_sync(&sq->dim.work);
1412         cancel_work_sync(&sq->recover_work);
1413         mlx5e_destroy_sq(mdev, sq->sqn);
1414         if (sq->rate_limit) {
1415                 rl.rate = sq->rate_limit;
1416                 mlx5_rl_remove_rate(mdev, &rl);
1417         }
1418         mlx5e_free_txqsq_descs(sq);
1419         mlx5e_free_txqsq(sq);
1420 }
1421
1422 void mlx5e_tx_err_cqe_work(struct work_struct *recover_work)
1423 {
1424         struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq,
1425                                               recover_work);
1426
1427         mlx5e_reporter_tx_err_cqe(sq);
1428 }
1429
1430 int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params,
1431                      struct mlx5e_sq_param *param, struct mlx5e_icosq *sq)
1432 {
1433         struct mlx5e_create_sq_param csp = {};
1434         int err;
1435
1436         err = mlx5e_alloc_icosq(c, param, sq);
1437         if (err)
1438                 return err;
1439
1440         csp.cqn             = sq->cq.mcq.cqn;
1441         csp.wq_ctrl         = &sq->wq_ctrl;
1442         csp.min_inline_mode = params->tx_min_inline_mode;
1443         err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn);
1444         if (err)
1445                 goto err_free_icosq;
1446
1447         return 0;
1448
1449 err_free_icosq:
1450         mlx5e_free_icosq(sq);
1451
1452         return err;
1453 }
1454
1455 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq)
1456 {
1457         set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
1458 }
1459
1460 void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq)
1461 {
1462         clear_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
1463         synchronize_rcu(); /* Sync with NAPI. */
1464 }
1465
1466 void mlx5e_close_icosq(struct mlx5e_icosq *sq)
1467 {
1468         struct mlx5e_channel *c = sq->channel;
1469
1470         mlx5e_destroy_sq(c->mdev, sq->sqn);
1471         mlx5e_free_icosq_descs(sq);
1472         mlx5e_free_icosq(sq);
1473 }
1474
1475 int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
1476                      struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool,
1477                      struct mlx5e_xdpsq *sq, bool is_redirect)
1478 {
1479         struct mlx5e_create_sq_param csp = {};
1480         int err;
1481
1482         err = mlx5e_alloc_xdpsq(c, params, xsk_pool, param, sq, is_redirect);
1483         if (err)
1484                 return err;
1485
1486         csp.tis_lst_sz      = 1;
1487         csp.tisn            = c->priv->tisn[c->lag_port][0]; /* tc = 0 */
1488         csp.cqn             = sq->cq.mcq.cqn;
1489         csp.wq_ctrl         = &sq->wq_ctrl;
1490         csp.min_inline_mode = sq->min_inline_mode;
1491         set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1492         err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn);
1493         if (err)
1494                 goto err_free_xdpsq;
1495
1496         mlx5e_set_xmit_fp(sq, param->is_mpw);
1497
1498         if (!param->is_mpw) {
1499                 unsigned int ds_cnt = MLX5E_XDP_TX_DS_COUNT;
1500                 unsigned int inline_hdr_sz = 0;
1501                 int i;
1502
1503                 if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
1504                         inline_hdr_sz = MLX5E_XDP_MIN_INLINE;
1505                         ds_cnt++;
1506                 }
1507
1508                 /* Pre initialize fixed WQE fields */
1509                 for (i = 0; i < mlx5_wq_cyc_get_size(&sq->wq); i++) {
1510                         struct mlx5e_tx_wqe      *wqe  = mlx5_wq_cyc_get_wqe(&sq->wq, i);
1511                         struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
1512                         struct mlx5_wqe_eth_seg  *eseg = &wqe->eth;
1513                         struct mlx5_wqe_data_seg *dseg;
1514
1515                         sq->db.wqe_info[i] = (struct mlx5e_xdp_wqe_info) {
1516                                 .num_wqebbs = 1,
1517                                 .num_pkts   = 1,
1518                         };
1519
1520                         cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
1521                         eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz);
1522
1523                         dseg = (struct mlx5_wqe_data_seg *)cseg + (ds_cnt - 1);
1524                         dseg->lkey = sq->mkey_be;
1525                 }
1526         }
1527
1528         return 0;
1529
1530 err_free_xdpsq:
1531         clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1532         mlx5e_free_xdpsq(sq);
1533
1534         return err;
1535 }
1536
1537 void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq)
1538 {
1539         struct mlx5e_channel *c = sq->channel;
1540
1541         clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1542         synchronize_rcu(); /* Sync with NAPI. */
1543
1544         mlx5e_destroy_sq(c->mdev, sq->sqn);
1545         mlx5e_free_xdpsq_descs(sq);
1546         mlx5e_free_xdpsq(sq);
1547 }
1548
1549 static int mlx5e_alloc_cq_common(struct mlx5e_priv *priv,
1550                                  struct mlx5e_cq_param *param,
1551                                  struct mlx5e_cq *cq)
1552 {
1553         struct mlx5_core_dev *mdev = priv->mdev;
1554         struct mlx5_core_cq *mcq = &cq->mcq;
1555         int eqn_not_used;
1556         unsigned int irqn;
1557         int err;
1558         u32 i;
1559
1560         err = mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
1561         if (err)
1562                 return err;
1563
1564         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1565                                &cq->wq_ctrl);
1566         if (err)
1567                 return err;
1568
1569         mcq->cqe_sz     = 64;
1570         mcq->set_ci_db  = cq->wq_ctrl.db.db;
1571         mcq->arm_db     = cq->wq_ctrl.db.db + 1;
1572         *mcq->set_ci_db = 0;
1573         *mcq->arm_db    = 0;
1574         mcq->vector     = param->eq_ix;
1575         mcq->comp       = mlx5e_completion_event;
1576         mcq->event      = mlx5e_cq_error_event;
1577         mcq->irqn       = irqn;
1578
1579         for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1580                 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1581
1582                 cqe->op_own = 0xf1;
1583         }
1584
1585         cq->mdev = mdev;
1586         cq->netdev = priv->netdev;
1587         cq->priv = priv;
1588
1589         return 0;
1590 }
1591
1592 static int mlx5e_alloc_cq(struct mlx5e_priv *priv,
1593                           struct mlx5e_cq_param *param,
1594                           struct mlx5e_create_cq_param *ccp,
1595                           struct mlx5e_cq *cq)
1596 {
1597         int err;
1598
1599         param->wq.buf_numa_node = ccp->node;
1600         param->wq.db_numa_node  = ccp->node;
1601         param->eq_ix            = ccp->ix;
1602
1603         err = mlx5e_alloc_cq_common(priv, param, cq);
1604
1605         cq->napi     = ccp->napi;
1606         cq->ch_stats = ccp->ch_stats;
1607
1608         return err;
1609 }
1610
1611 static void mlx5e_free_cq(struct mlx5e_cq *cq)
1612 {
1613         mlx5_wq_destroy(&cq->wq_ctrl);
1614 }
1615
1616 static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
1617 {
1618         u32 out[MLX5_ST_SZ_DW(create_cq_out)];
1619         struct mlx5_core_dev *mdev = cq->mdev;
1620         struct mlx5_core_cq *mcq = &cq->mcq;
1621
1622         void *in;
1623         void *cqc;
1624         int inlen;
1625         unsigned int irqn_not_used;
1626         int eqn;
1627         int err;
1628
1629         err = mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
1630         if (err)
1631                 return err;
1632
1633         inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1634                 sizeof(u64) * cq->wq_ctrl.buf.npages;
1635         in = kvzalloc(inlen, GFP_KERNEL);
1636         if (!in)
1637                 return -ENOMEM;
1638
1639         cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1640
1641         memcpy(cqc, param->cqc, sizeof(param->cqc));
1642
1643         mlx5_fill_page_frag_array(&cq->wq_ctrl.buf,
1644                                   (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
1645
1646         MLX5_SET(cqc,   cqc, cq_period_mode, param->cq_period_mode);
1647         MLX5_SET(cqc,   cqc, c_eqn,         eqn);
1648         MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.uar->index);
1649         MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
1650                                             MLX5_ADAPTER_PAGE_SHIFT);
1651         MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
1652
1653         err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out));
1654
1655         kvfree(in);
1656
1657         if (err)
1658                 return err;
1659
1660         mlx5e_cq_arm(cq);
1661
1662         return 0;
1663 }
1664
1665 static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
1666 {
1667         mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
1668 }
1669
1670 int mlx5e_open_cq(struct mlx5e_priv *priv, struct dim_cq_moder moder,
1671                   struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp,
1672                   struct mlx5e_cq *cq)
1673 {
1674         struct mlx5_core_dev *mdev = priv->mdev;
1675         int err;
1676
1677         err = mlx5e_alloc_cq(priv, param, ccp, cq);
1678         if (err)
1679                 return err;
1680
1681         err = mlx5e_create_cq(cq, param);
1682         if (err)
1683                 goto err_free_cq;
1684
1685         if (MLX5_CAP_GEN(mdev, cq_moderation))
1686                 mlx5_core_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts);
1687         return 0;
1688
1689 err_free_cq:
1690         mlx5e_free_cq(cq);
1691
1692         return err;
1693 }
1694
1695 void mlx5e_close_cq(struct mlx5e_cq *cq)
1696 {
1697         mlx5e_destroy_cq(cq);
1698         mlx5e_free_cq(cq);
1699 }
1700
1701 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
1702                              struct mlx5e_params *params,
1703                              struct mlx5e_create_cq_param *ccp,
1704                              struct mlx5e_channel_param *cparam)
1705 {
1706         int err;
1707         int tc;
1708
1709         for (tc = 0; tc < c->num_tc; tc++) {
1710                 err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->txq_sq.cqp,
1711                                     ccp, &c->sq[tc].cq);
1712                 if (err)
1713                         goto err_close_tx_cqs;
1714         }
1715
1716         return 0;
1717
1718 err_close_tx_cqs:
1719         for (tc--; tc >= 0; tc--)
1720                 mlx5e_close_cq(&c->sq[tc].cq);
1721
1722         return err;
1723 }
1724
1725 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
1726 {
1727         int tc;
1728
1729         for (tc = 0; tc < c->num_tc; tc++)
1730                 mlx5e_close_cq(&c->sq[tc].cq);
1731 }
1732
1733 static int mlx5e_open_sqs(struct mlx5e_channel *c,
1734                           struct mlx5e_params *params,
1735                           struct mlx5e_channel_param *cparam)
1736 {
1737         int err, tc;
1738
1739         for (tc = 0; tc < params->num_tc; tc++) {
1740                 int txq_ix = c->ix + tc * params->num_channels;
1741
1742                 err = mlx5e_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix,
1743                                        params, &cparam->txq_sq, &c->sq[tc], tc, 0, 0);
1744                 if (err)
1745                         goto err_close_sqs;
1746         }
1747
1748         return 0;
1749
1750 err_close_sqs:
1751         for (tc--; tc >= 0; tc--)
1752                 mlx5e_close_txqsq(&c->sq[tc]);
1753
1754         return err;
1755 }
1756
1757 static void mlx5e_close_sqs(struct mlx5e_channel *c)
1758 {
1759         int tc;
1760
1761         for (tc = 0; tc < c->num_tc; tc++)
1762                 mlx5e_close_txqsq(&c->sq[tc]);
1763 }
1764
1765 static int mlx5e_set_sq_maxrate(struct net_device *dev,
1766                                 struct mlx5e_txqsq *sq, u32 rate)
1767 {
1768         struct mlx5e_priv *priv = netdev_priv(dev);
1769         struct mlx5_core_dev *mdev = priv->mdev;
1770         struct mlx5e_modify_sq_param msp = {0};
1771         struct mlx5_rate_limit rl = {0};
1772         u16 rl_index = 0;
1773         int err;
1774
1775         if (rate == sq->rate_limit)
1776                 /* nothing to do */
1777                 return 0;
1778
1779         if (sq->rate_limit) {
1780                 rl.rate = sq->rate_limit;
1781                 /* remove current rl index to free space to next ones */
1782                 mlx5_rl_remove_rate(mdev, &rl);
1783         }
1784
1785         sq->rate_limit = 0;
1786
1787         if (rate) {
1788                 rl.rate = rate;
1789                 err = mlx5_rl_add_rate(mdev, &rl_index, &rl);
1790                 if (err) {
1791                         netdev_err(dev, "Failed configuring rate %u: %d\n",
1792                                    rate, err);
1793                         return err;
1794                 }
1795         }
1796
1797         msp.curr_state = MLX5_SQC_STATE_RDY;
1798         msp.next_state = MLX5_SQC_STATE_RDY;
1799         msp.rl_index   = rl_index;
1800         msp.rl_update  = true;
1801         err = mlx5e_modify_sq(mdev, sq->sqn, &msp);
1802         if (err) {
1803                 netdev_err(dev, "Failed configuring rate %u: %d\n",
1804                            rate, err);
1805                 /* remove the rate from the table */
1806                 if (rate)
1807                         mlx5_rl_remove_rate(mdev, &rl);
1808                 return err;
1809         }
1810
1811         sq->rate_limit = rate;
1812         return 0;
1813 }
1814
1815 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
1816 {
1817         struct mlx5e_priv *priv = netdev_priv(dev);
1818         struct mlx5_core_dev *mdev = priv->mdev;
1819         struct mlx5e_txqsq *sq = priv->txq2sq[index];
1820         int err = 0;
1821
1822         if (!mlx5_rl_is_supported(mdev)) {
1823                 netdev_err(dev, "Rate limiting is not supported on this device\n");
1824                 return -EINVAL;
1825         }
1826
1827         /* rate is given in Mb/sec, HW config is in Kb/sec */
1828         rate = rate << 10;
1829
1830         /* Check whether rate in valid range, 0 is always valid */
1831         if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
1832                 netdev_err(dev, "TX rate %u, is not in range\n", rate);
1833                 return -ERANGE;
1834         }
1835
1836         mutex_lock(&priv->state_lock);
1837         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
1838                 err = mlx5e_set_sq_maxrate(dev, sq, rate);
1839         if (!err)
1840                 priv->tx_rates[index] = rate;
1841         mutex_unlock(&priv->state_lock);
1842
1843         return err;
1844 }
1845
1846 void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e_channel *c)
1847 {
1848         *ccp = (struct mlx5e_create_cq_param) {
1849                 .napi = &c->napi,
1850                 .ch_stats = c->stats,
1851                 .node = cpu_to_node(c->cpu),
1852                 .ix = c->ix,
1853         };
1854 }
1855
1856 static int mlx5e_open_queues(struct mlx5e_channel *c,
1857                              struct mlx5e_params *params,
1858                              struct mlx5e_channel_param *cparam)
1859 {
1860         struct dim_cq_moder icocq_moder = {0, 0};
1861         struct mlx5e_create_cq_param ccp;
1862         int err;
1863
1864         mlx5e_build_create_cq_param(&ccp, c);
1865
1866         err = mlx5e_open_cq(c->priv, icocq_moder, &cparam->icosq.cqp, &ccp,
1867                             &c->async_icosq.cq);
1868         if (err)
1869                 return err;
1870
1871         err = mlx5e_open_cq(c->priv, icocq_moder, &cparam->async_icosq.cqp, &ccp,
1872                             &c->icosq.cq);
1873         if (err)
1874                 goto err_close_async_icosq_cq;
1875
1876         err = mlx5e_open_tx_cqs(c, params, &ccp, cparam);
1877         if (err)
1878                 goto err_close_icosq_cq;
1879
1880         err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp,
1881                             &c->xdpsq.cq);
1882         if (err)
1883                 goto err_close_tx_cqs;
1884
1885         err = mlx5e_open_cq(c->priv, params->rx_cq_moderation, &cparam->rq.cqp, &ccp,
1886                             &c->rq.cq);
1887         if (err)
1888                 goto err_close_xdp_tx_cqs;
1889
1890         err = c->xdp ? mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp,
1891                                      &ccp, &c->rq_xdpsq.cq) : 0;
1892         if (err)
1893                 goto err_close_rx_cq;
1894
1895         spin_lock_init(&c->async_icosq_lock);
1896
1897         err = mlx5e_open_icosq(c, params, &cparam->async_icosq, &c->async_icosq);
1898         if (err)
1899                 goto err_close_xdpsq_cq;
1900
1901         err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq);
1902         if (err)
1903                 goto err_close_async_icosq;
1904
1905         err = mlx5e_open_sqs(c, params, cparam);
1906         if (err)
1907                 goto err_close_icosq;
1908
1909         if (c->xdp) {
1910                 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL,
1911                                        &c->rq_xdpsq, false);
1912                 if (err)
1913                         goto err_close_sqs;
1914         }
1915
1916         err = mlx5e_open_rq(c, params, &cparam->rq, NULL, NULL, &c->rq);
1917         if (err)
1918                 goto err_close_xdp_sq;
1919
1920         err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, &c->xdpsq, true);
1921         if (err)
1922                 goto err_close_rq;
1923
1924         return 0;
1925
1926 err_close_rq:
1927         mlx5e_close_rq(&c->rq);
1928
1929 err_close_xdp_sq:
1930         if (c->xdp)
1931                 mlx5e_close_xdpsq(&c->rq_xdpsq);
1932
1933 err_close_sqs:
1934         mlx5e_close_sqs(c);
1935
1936 err_close_icosq:
1937         mlx5e_close_icosq(&c->icosq);
1938
1939 err_close_async_icosq:
1940         mlx5e_close_icosq(&c->async_icosq);
1941
1942 err_close_xdpsq_cq:
1943         if (c->xdp)
1944                 mlx5e_close_cq(&c->rq_xdpsq.cq);
1945
1946 err_close_rx_cq:
1947         mlx5e_close_cq(&c->rq.cq);
1948
1949 err_close_xdp_tx_cqs:
1950         mlx5e_close_cq(&c->xdpsq.cq);
1951
1952 err_close_tx_cqs:
1953         mlx5e_close_tx_cqs(c);
1954
1955 err_close_icosq_cq:
1956         mlx5e_close_cq(&c->icosq.cq);
1957
1958 err_close_async_icosq_cq:
1959         mlx5e_close_cq(&c->async_icosq.cq);
1960
1961         return err;
1962 }
1963
1964 static void mlx5e_close_queues(struct mlx5e_channel *c)
1965 {
1966         mlx5e_close_xdpsq(&c->xdpsq);
1967         mlx5e_close_rq(&c->rq);
1968         if (c->xdp)
1969                 mlx5e_close_xdpsq(&c->rq_xdpsq);
1970         mlx5e_close_sqs(c);
1971         mlx5e_close_icosq(&c->icosq);
1972         mlx5e_close_icosq(&c->async_icosq);
1973         if (c->xdp)
1974                 mlx5e_close_cq(&c->rq_xdpsq.cq);
1975         mlx5e_close_cq(&c->rq.cq);
1976         mlx5e_close_cq(&c->xdpsq.cq);
1977         mlx5e_close_tx_cqs(c);
1978         mlx5e_close_cq(&c->icosq.cq);
1979         mlx5e_close_cq(&c->async_icosq.cq);
1980 }
1981
1982 static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix)
1983 {
1984         u16 port_aff_bias = mlx5_core_is_pf(mdev) ? 0 : MLX5_CAP_GEN(mdev, vhca_id);
1985
1986         return (ix + port_aff_bias) % mlx5e_get_num_lag_ports(mdev);
1987 }
1988
1989 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
1990                               struct mlx5e_params *params,
1991                               struct mlx5e_channel_param *cparam,
1992                               struct xsk_buff_pool *xsk_pool,
1993                               struct mlx5e_channel **cp)
1994 {
1995         int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(priv->mdev, ix));
1996         struct net_device *netdev = priv->netdev;
1997         struct mlx5e_xsk_param xsk;
1998         struct mlx5e_channel *c;
1999         unsigned int irq;
2000         int err;
2001         int eqn;
2002
2003         err = mlx5_vector2eqn(priv->mdev, ix, &eqn, &irq);
2004         if (err)
2005                 return err;
2006
2007         c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
2008         if (!c)
2009                 return -ENOMEM;
2010
2011         c->priv     = priv;
2012         c->mdev     = priv->mdev;
2013         c->tstamp   = &priv->tstamp;
2014         c->ix       = ix;
2015         c->cpu      = cpu;
2016         c->pdev     = mlx5_core_dma_dev(priv->mdev);
2017         c->netdev   = priv->netdev;
2018         c->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);
2019         c->num_tc   = params->num_tc;
2020         c->xdp      = !!params->xdp_prog;
2021         c->stats    = &priv->channel_stats[ix].ch;
2022         c->aff_mask = irq_get_effective_affinity_mask(irq);
2023         c->lag_port = mlx5e_enumerate_lag_port(priv->mdev, ix);
2024
2025         netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
2026
2027         err = mlx5e_open_queues(c, params, cparam);
2028         if (unlikely(err))
2029                 goto err_napi_del;
2030
2031         if (xsk_pool) {
2032                 mlx5e_build_xsk_param(xsk_pool, &xsk);
2033                 err = mlx5e_open_xsk(priv, params, &xsk, xsk_pool, c);
2034                 if (unlikely(err))
2035                         goto err_close_queues;
2036         }
2037
2038         *cp = c;
2039
2040         return 0;
2041
2042 err_close_queues:
2043         mlx5e_close_queues(c);
2044
2045 err_napi_del:
2046         netif_napi_del(&c->napi);
2047
2048         kvfree(c);
2049
2050         return err;
2051 }
2052
2053 static void mlx5e_activate_channel(struct mlx5e_channel *c)
2054 {
2055         int tc;
2056
2057         napi_enable(&c->napi);
2058
2059         for (tc = 0; tc < c->num_tc; tc++)
2060                 mlx5e_activate_txqsq(&c->sq[tc]);
2061         mlx5e_activate_icosq(&c->icosq);
2062         mlx5e_activate_icosq(&c->async_icosq);
2063         mlx5e_activate_rq(&c->rq);
2064
2065         if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2066                 mlx5e_activate_xsk(c);
2067 }
2068
2069 static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
2070 {
2071         int tc;
2072
2073         if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2074                 mlx5e_deactivate_xsk(c);
2075
2076         mlx5e_deactivate_rq(&c->rq);
2077         mlx5e_deactivate_icosq(&c->async_icosq);
2078         mlx5e_deactivate_icosq(&c->icosq);
2079         for (tc = 0; tc < c->num_tc; tc++)
2080                 mlx5e_deactivate_txqsq(&c->sq[tc]);
2081         mlx5e_qos_deactivate_queues(c);
2082
2083         napi_disable(&c->napi);
2084 }
2085
2086 static void mlx5e_close_channel(struct mlx5e_channel *c)
2087 {
2088         if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2089                 mlx5e_close_xsk(c);
2090         mlx5e_close_queues(c);
2091         mlx5e_qos_close_queues(c);
2092         netif_napi_del(&c->napi);
2093
2094         kvfree(c);
2095 }
2096
2097 #define DEFAULT_FRAG_SIZE (2048)
2098
2099 static void mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
2100                                       struct mlx5e_params *params,
2101                                       struct mlx5e_xsk_param *xsk,
2102                                       struct mlx5e_rq_frags_info *info)
2103 {
2104         u32 byte_count = MLX5E_SW2HW_MTU(params, params->sw_mtu);
2105         int frag_size_max = DEFAULT_FRAG_SIZE;
2106         u32 buf_size = 0;
2107         int i;
2108
2109         if (MLX5_IPSEC_DEV(mdev))
2110                 byte_count += MLX5E_METADATA_ETHER_LEN;
2111
2112         if (mlx5e_rx_is_linear_skb(params, xsk)) {
2113                 int frag_stride;
2114
2115                 frag_stride = mlx5e_rx_get_linear_frag_sz(params, xsk);
2116                 frag_stride = roundup_pow_of_two(frag_stride);
2117
2118                 info->arr[0].frag_size = byte_count;
2119                 info->arr[0].frag_stride = frag_stride;
2120                 info->num_frags = 1;
2121                 info->wqe_bulk = PAGE_SIZE / frag_stride;
2122                 goto out;
2123         }
2124
2125         if (byte_count > PAGE_SIZE +
2126             (MLX5E_MAX_RX_FRAGS - 1) * frag_size_max)
2127                 frag_size_max = PAGE_SIZE;
2128
2129         i = 0;
2130         while (buf_size < byte_count) {
2131                 int frag_size = byte_count - buf_size;
2132
2133                 if (i < MLX5E_MAX_RX_FRAGS - 1)
2134                         frag_size = min(frag_size, frag_size_max);
2135
2136                 info->arr[i].frag_size = frag_size;
2137                 info->arr[i].frag_stride = roundup_pow_of_two(frag_size);
2138
2139                 buf_size += frag_size;
2140                 i++;
2141         }
2142         info->num_frags = i;
2143         /* number of different wqes sharing a page */
2144         info->wqe_bulk = 1 + (info->num_frags % 2);
2145
2146 out:
2147         info->wqe_bulk = max_t(u8, info->wqe_bulk, 8);
2148         info->log_num_frags = order_base_2(info->num_frags);
2149 }
2150
2151 static inline u8 mlx5e_get_rqwq_log_stride(u8 wq_type, int ndsegs)
2152 {
2153         int sz = sizeof(struct mlx5_wqe_data_seg) * ndsegs;
2154
2155         switch (wq_type) {
2156         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
2157                 sz += sizeof(struct mlx5e_rx_wqe_ll);
2158                 break;
2159         default: /* MLX5_WQ_TYPE_CYCLIC */
2160                 sz += sizeof(struct mlx5e_rx_wqe_cyc);
2161         }
2162
2163         return order_base_2(sz);
2164 }
2165
2166 static u8 mlx5e_get_rq_log_wq_sz(void *rqc)
2167 {
2168         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2169
2170         return MLX5_GET(wq, wq, log_wq_sz);
2171 }
2172
2173 void mlx5e_build_rq_param(struct mlx5e_priv *priv,
2174                           struct mlx5e_params *params,
2175                           struct mlx5e_xsk_param *xsk,
2176                           struct mlx5e_rq_param *param)
2177 {
2178         struct mlx5_core_dev *mdev = priv->mdev;
2179         void *rqc = param->rqc;
2180         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2181         int ndsegs = 1;
2182
2183         switch (params->rq_wq_type) {
2184         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
2185                 MLX5_SET(wq, wq, log_wqe_num_of_strides,
2186                          mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk) -
2187                          MLX5_MPWQE_LOG_NUM_STRIDES_BASE);
2188                 MLX5_SET(wq, wq, log_wqe_stride_size,
2189                          mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk) -
2190                          MLX5_MPWQE_LOG_STRIDE_SZ_BASE);
2191                 MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params, xsk));
2192                 break;
2193         default: /* MLX5_WQ_TYPE_CYCLIC */
2194                 MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames);
2195                 mlx5e_build_rq_frags_info(mdev, params, xsk, &param->frags_info);
2196                 ndsegs = param->frags_info.num_frags;
2197         }
2198
2199         MLX5_SET(wq, wq, wq_type,          params->rq_wq_type);
2200         MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
2201         MLX5_SET(wq, wq, log_wq_stride,
2202                  mlx5e_get_rqwq_log_stride(params->rq_wq_type, ndsegs));
2203         MLX5_SET(wq, wq, pd,               mdev->mlx5e_res.pdn);
2204         MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter);
2205         MLX5_SET(rqc, rqc, vsd,            params->vlan_strip_disable);
2206         MLX5_SET(rqc, rqc, scatter_fcs,    params->scatter_fcs_en);
2207
2208         param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
2209         mlx5e_build_rx_cq_param(priv, params, xsk, &param->cqp);
2210 }
2211
2212 static void mlx5e_build_drop_rq_param(struct mlx5e_priv *priv,
2213                                       struct mlx5e_rq_param *param)
2214 {
2215         struct mlx5_core_dev *mdev = priv->mdev;
2216         void *rqc = param->rqc;
2217         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2218
2219         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
2220         MLX5_SET(wq, wq, log_wq_stride,
2221                  mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1));
2222         MLX5_SET(rqc, rqc, counter_set_id, priv->drop_rq_q_counter);
2223
2224         param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
2225 }
2226
2227 void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,
2228                                  struct mlx5e_sq_param *param)
2229 {
2230         void *sqc = param->sqc;
2231         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2232
2233         MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
2234         MLX5_SET(wq, wq, pd,            priv->mdev->mlx5e_res.pdn);
2235
2236         param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(priv->mdev));
2237 }
2238
2239 void mlx5e_build_sq_param(struct mlx5e_priv *priv, struct mlx5e_params *params,
2240                           struct mlx5e_sq_param *param)
2241 {
2242         void *sqc = param->sqc;
2243         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2244         bool allow_swp;
2245
2246         allow_swp = mlx5_geneve_tx_allowed(priv->mdev) ||
2247                     !!MLX5_IPSEC_DEV(priv->mdev);
2248         mlx5e_build_sq_param_common(priv, param);
2249         MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
2250         MLX5_SET(sqc, sqc, allow_swp, allow_swp);
2251         param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE);
2252         param->stop_room = mlx5e_calc_sq_stop_room(priv->mdev, params);
2253         mlx5e_build_tx_cq_param(priv, params, &param->cqp);
2254 }
2255
2256 static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
2257                                         struct mlx5e_cq_param *param)
2258 {
2259         void *cqc = param->cqc;
2260
2261         MLX5_SET(cqc, cqc, uar_page, priv->mdev->priv.uar->index);
2262         if (MLX5_CAP_GEN(priv->mdev, cqe_128_always) && cache_line_size() >= 128)
2263                 MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD);
2264 }
2265
2266 void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
2267                              struct mlx5e_params *params,
2268                              struct mlx5e_xsk_param *xsk,
2269                              struct mlx5e_cq_param *param)
2270 {
2271         struct mlx5_core_dev *mdev = priv->mdev;
2272         bool hw_stridx = false;
2273         void *cqc = param->cqc;
2274         u8 log_cq_size;
2275
2276         switch (params->rq_wq_type) {
2277         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
2278                 log_cq_size = mlx5e_mpwqe_get_log_rq_size(params, xsk) +
2279                         mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
2280                 hw_stridx = MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index);
2281                 break;
2282         default: /* MLX5_WQ_TYPE_CYCLIC */
2283                 log_cq_size = params->log_rq_mtu_frames;
2284         }
2285
2286         MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
2287         if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
2288                 MLX5_SET(cqc, cqc, mini_cqe_res_format, hw_stridx ?
2289                          MLX5_CQE_FORMAT_CSUM_STRIDX : MLX5_CQE_FORMAT_CSUM);
2290                 MLX5_SET(cqc, cqc, cqe_comp_en, 1);
2291         }
2292
2293         mlx5e_build_common_cq_param(priv, param);
2294         param->cq_period_mode = params->rx_cq_moderation.cq_period_mode;
2295 }
2296
2297 void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
2298                              struct mlx5e_params *params,
2299                              struct mlx5e_cq_param *param)
2300 {
2301         void *cqc = param->cqc;
2302
2303         MLX5_SET(cqc, cqc, log_cq_size, params->log_sq_size);
2304
2305         mlx5e_build_common_cq_param(priv, param);
2306         param->cq_period_mode = params->tx_cq_moderation.cq_period_mode;
2307 }
2308
2309 void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
2310                               u8 log_wq_size,
2311                               struct mlx5e_cq_param *param)
2312 {
2313         void *cqc = param->cqc;
2314
2315         MLX5_SET(cqc, cqc, log_cq_size, log_wq_size);
2316
2317         mlx5e_build_common_cq_param(priv, param);
2318
2319         param->cq_period_mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
2320 }
2321
2322 void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
2323                              u8 log_wq_size,
2324                              struct mlx5e_sq_param *param)
2325 {
2326         void *sqc = param->sqc;
2327         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2328
2329         mlx5e_build_sq_param_common(priv, param);
2330
2331         MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
2332         MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq));
2333         mlx5e_build_ico_cq_param(priv, log_wq_size, &param->cqp);
2334 }
2335
2336 void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv,
2337                              struct mlx5e_params *params,
2338                              struct mlx5e_sq_param *param)
2339 {
2340         void *sqc = param->sqc;
2341         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2342
2343         mlx5e_build_sq_param_common(priv, param);
2344         MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
2345         param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE);
2346         mlx5e_build_tx_cq_param(priv, params, &param->cqp);
2347 }
2348
2349 static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5e_params *params,
2350                                       struct mlx5e_rq_param *rqp)
2351 {
2352         switch (params->rq_wq_type) {
2353         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
2354                 return order_base_2(MLX5E_UMR_WQEBBS) +
2355                         mlx5e_get_rq_log_wq_sz(rqp->rqc);
2356         default: /* MLX5_WQ_TYPE_CYCLIC */
2357                 return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
2358         }
2359 }
2360
2361 static u8 mlx5e_build_async_icosq_log_wq_sz(struct net_device *netdev)
2362 {
2363         if (netdev->hw_features & NETIF_F_HW_TLS_RX)
2364                 return MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
2365
2366         return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
2367 }
2368
2369 static void mlx5e_build_channel_param(struct mlx5e_priv *priv,
2370                                       struct mlx5e_params *params,
2371                                       struct mlx5e_channel_param *cparam)
2372 {
2373         u8 icosq_log_wq_sz, async_icosq_log_wq_sz;
2374
2375         mlx5e_build_rq_param(priv, params, NULL, &cparam->rq);
2376
2377         icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(params, &cparam->rq);
2378         async_icosq_log_wq_sz = mlx5e_build_async_icosq_log_wq_sz(priv->netdev);
2379
2380         mlx5e_build_sq_param(priv, params, &cparam->txq_sq);
2381         mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq);
2382         mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq);
2383         mlx5e_build_icosq_param(priv, async_icosq_log_wq_sz, &cparam->async_icosq);
2384 }
2385
2386 int mlx5e_open_channels(struct mlx5e_priv *priv,
2387                         struct mlx5e_channels *chs)
2388 {
2389         struct mlx5e_channel_param *cparam;
2390         int err = -ENOMEM;
2391         int i;
2392
2393         chs->num = chs->params.num_channels;
2394
2395         chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL);
2396         cparam = kvzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
2397         if (!chs->c || !cparam)
2398                 goto err_free;
2399
2400         mlx5e_build_channel_param(priv, &chs->params, cparam);
2401         for (i = 0; i < chs->num; i++) {
2402                 struct xsk_buff_pool *xsk_pool = NULL;
2403
2404                 if (chs->params.xdp_prog)
2405                         xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i);
2406
2407                 err = mlx5e_open_channel(priv, i, &chs->params, cparam, xsk_pool, &chs->c[i]);
2408                 if (err)
2409                         goto err_close_channels;
2410         }
2411
2412         if (MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS)) {
2413                 err = mlx5e_port_ptp_open(priv, &chs->params, chs->c[0]->lag_port,
2414                                           &chs->port_ptp);
2415                 if (err)
2416                         goto err_close_channels;
2417         }
2418
2419         err = mlx5e_qos_open_queues(priv, chs);
2420         if (err)
2421                 goto err_close_ptp;
2422
2423         mlx5e_health_channels_update(priv);
2424         kvfree(cparam);
2425         return 0;
2426
2427 err_close_ptp:
2428         if (chs->port_ptp)
2429                 mlx5e_port_ptp_close(chs->port_ptp);
2430
2431 err_close_channels:
2432         for (i--; i >= 0; i--)
2433                 mlx5e_close_channel(chs->c[i]);
2434
2435 err_free:
2436         kfree(chs->c);
2437         kvfree(cparam);
2438         chs->num = 0;
2439         return err;
2440 }
2441
2442 static void mlx5e_activate_channels(struct mlx5e_channels *chs)
2443 {
2444         int i;
2445
2446         for (i = 0; i < chs->num; i++)
2447                 mlx5e_activate_channel(chs->c[i]);
2448
2449         if (chs->port_ptp)
2450                 mlx5e_ptp_activate_channel(chs->port_ptp);
2451 }
2452
2453 #define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */
2454
2455 static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
2456 {
2457         int err = 0;
2458         int i;
2459
2460         for (i = 0; i < chs->num; i++) {
2461                 int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT;
2462
2463                 err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq, timeout);
2464
2465                 /* Don't wait on the XSK RQ, because the newer xdpsock sample
2466                  * doesn't provide any Fill Ring entries at the setup stage.
2467                  */
2468         }
2469
2470         return err ? -ETIMEDOUT : 0;
2471 }
2472
2473 static void mlx5e_deactivate_channels(struct mlx5e_channels *chs)
2474 {
2475         int i;
2476
2477         if (chs->port_ptp)
2478                 mlx5e_ptp_deactivate_channel(chs->port_ptp);
2479
2480         for (i = 0; i < chs->num; i++)
2481                 mlx5e_deactivate_channel(chs->c[i]);
2482 }
2483
2484 void mlx5e_close_channels(struct mlx5e_channels *chs)
2485 {
2486         int i;
2487
2488         if (chs->port_ptp)
2489                 mlx5e_port_ptp_close(chs->port_ptp);
2490
2491         for (i = 0; i < chs->num; i++)
2492                 mlx5e_close_channel(chs->c[i]);
2493
2494         kfree(chs->c);
2495         chs->num = 0;
2496 }
2497
2498 static int
2499 mlx5e_create_rqt(struct mlx5e_priv *priv, int sz, struct mlx5e_rqt *rqt)
2500 {
2501         struct mlx5_core_dev *mdev = priv->mdev;
2502         void *rqtc;
2503         int inlen;
2504         int err;
2505         u32 *in;
2506         int i;
2507
2508         inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
2509         in = kvzalloc(inlen, GFP_KERNEL);
2510         if (!in)
2511                 return -ENOMEM;
2512
2513         rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
2514
2515         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2516         MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
2517
2518         for (i = 0; i < sz; i++)
2519                 MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn);
2520
2521         err = mlx5_core_create_rqt(mdev, in, inlen, &rqt->rqtn);
2522         if (!err)
2523                 rqt->enabled = true;
2524
2525         kvfree(in);
2526         return err;
2527 }
2528
2529 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt)
2530 {
2531         rqt->enabled = false;
2532         mlx5_core_destroy_rqt(priv->mdev, rqt->rqtn);
2533 }
2534
2535 int mlx5e_create_indirect_rqt(struct mlx5e_priv *priv)
2536 {
2537         struct mlx5e_rqt *rqt = &priv->indir_rqt;
2538         int err;
2539
2540         err = mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, rqt);
2541         if (err)
2542                 mlx5_core_warn(priv->mdev, "create indirect rqts failed, %d\n", err);
2543         return err;
2544 }
2545
2546 int mlx5e_create_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs)
2547 {
2548         int err;
2549         int ix;
2550
2551         for (ix = 0; ix < priv->max_nch; ix++) {
2552                 err = mlx5e_create_rqt(priv, 1 /*size */, &tirs[ix].rqt);
2553                 if (unlikely(err))
2554                         goto err_destroy_rqts;
2555         }
2556
2557         return 0;
2558
2559 err_destroy_rqts:
2560         mlx5_core_warn(priv->mdev, "create rqts failed, %d\n", err);
2561         for (ix--; ix >= 0; ix--)
2562                 mlx5e_destroy_rqt(priv, &tirs[ix].rqt);
2563
2564         return err;
2565 }
2566
2567 void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs)
2568 {
2569         int i;
2570
2571         for (i = 0; i < priv->max_nch; i++)
2572                 mlx5e_destroy_rqt(priv, &tirs[i].rqt);
2573 }
2574
2575 static int mlx5e_rx_hash_fn(int hfunc)
2576 {
2577         return (hfunc == ETH_RSS_HASH_TOP) ?
2578                MLX5_RX_HASH_FN_TOEPLITZ :
2579                MLX5_RX_HASH_FN_INVERTED_XOR8;
2580 }
2581
2582 int mlx5e_bits_invert(unsigned long a, int size)
2583 {
2584         int inv = 0;
2585         int i;
2586
2587         for (i = 0; i < size; i++)
2588                 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
2589
2590         return inv;
2591 }
2592
2593 static void mlx5e_fill_rqt_rqns(struct mlx5e_priv *priv, int sz,
2594                                 struct mlx5e_redirect_rqt_param rrp, void *rqtc)
2595 {
2596         int i;
2597
2598         for (i = 0; i < sz; i++) {
2599                 u32 rqn;
2600
2601                 if (rrp.is_rss) {
2602                         int ix = i;
2603
2604                         if (rrp.rss.hfunc == ETH_RSS_HASH_XOR)
2605                                 ix = mlx5e_bits_invert(i, ilog2(sz));
2606
2607                         ix = priv->rss_params.indirection_rqt[ix];
2608                         rqn = rrp.rss.channels->c[ix]->rq.rqn;
2609                 } else {
2610                         rqn = rrp.rqn;
2611                 }
2612                 MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
2613         }
2614 }
2615
2616 int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz,
2617                        struct mlx5e_redirect_rqt_param rrp)
2618 {
2619         struct mlx5_core_dev *mdev = priv->mdev;
2620         void *rqtc;
2621         int inlen;
2622         u32 *in;
2623         int err;
2624
2625         inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
2626         in = kvzalloc(inlen, GFP_KERNEL);
2627         if (!in)
2628                 return -ENOMEM;
2629
2630         rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
2631
2632         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2633         MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
2634         mlx5e_fill_rqt_rqns(priv, sz, rrp, rqtc);
2635         err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen);
2636
2637         kvfree(in);
2638         return err;
2639 }
2640
2641 static u32 mlx5e_get_direct_rqn(struct mlx5e_priv *priv, int ix,
2642                                 struct mlx5e_redirect_rqt_param rrp)
2643 {
2644         if (!rrp.is_rss)
2645                 return rrp.rqn;
2646
2647         if (ix >= rrp.rss.channels->num)
2648                 return priv->drop_rq.rqn;
2649
2650         return rrp.rss.channels->c[ix]->rq.rqn;
2651 }
2652
2653 static void mlx5e_redirect_rqts(struct mlx5e_priv *priv,
2654                                 struct mlx5e_redirect_rqt_param rrp)
2655 {
2656         u32 rqtn;
2657         int ix;
2658
2659         if (priv->indir_rqt.enabled) {
2660                 /* RSS RQ table */
2661                 rqtn = priv->indir_rqt.rqtn;
2662                 mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, rrp);
2663         }
2664
2665         for (ix = 0; ix < priv->max_nch; ix++) {
2666                 struct mlx5e_redirect_rqt_param direct_rrp = {
2667                         .is_rss = false,
2668                         {
2669                                 .rqn    = mlx5e_get_direct_rqn(priv, ix, rrp)
2670                         },
2671                 };
2672
2673                 /* Direct RQ Tables */
2674                 if (!priv->direct_tir[ix].rqt.enabled)
2675                         continue;
2676
2677                 rqtn = priv->direct_tir[ix].rqt.rqtn;
2678                 mlx5e_redirect_rqt(priv, rqtn, 1, direct_rrp);
2679         }
2680 }
2681
2682 static void mlx5e_redirect_rqts_to_channels(struct mlx5e_priv *priv,
2683                                             struct mlx5e_channels *chs)
2684 {
2685         struct mlx5e_redirect_rqt_param rrp = {
2686                 .is_rss        = true,
2687                 {
2688                         .rss = {
2689                                 .channels  = chs,
2690                                 .hfunc     = priv->rss_params.hfunc,
2691                         }
2692                 },
2693         };
2694
2695         mlx5e_redirect_rqts(priv, rrp);
2696 }
2697
2698 static void mlx5e_redirect_rqts_to_drop(struct mlx5e_priv *priv)
2699 {
2700         struct mlx5e_redirect_rqt_param drop_rrp = {
2701                 .is_rss = false,
2702                 {
2703                         .rqn = priv->drop_rq.rqn,
2704                 },
2705         };
2706
2707         mlx5e_redirect_rqts(priv, drop_rrp);
2708 }
2709
2710 static const struct mlx5e_tirc_config tirc_default_config[MLX5E_NUM_INDIR_TIRS] = {
2711         [MLX5E_TT_IPV4_TCP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
2712                                 .l4_prot_type = MLX5_L4_PROT_TYPE_TCP,
2713                                 .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
2714         },
2715         [MLX5E_TT_IPV6_TCP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
2716                                 .l4_prot_type = MLX5_L4_PROT_TYPE_TCP,
2717                                 .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
2718         },
2719         [MLX5E_TT_IPV4_UDP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
2720                                 .l4_prot_type = MLX5_L4_PROT_TYPE_UDP,
2721                                 .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
2722         },
2723         [MLX5E_TT_IPV6_UDP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
2724                                 .l4_prot_type = MLX5_L4_PROT_TYPE_UDP,
2725                                 .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
2726         },
2727         [MLX5E_TT_IPV4_IPSEC_AH] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
2728                                      .l4_prot_type = 0,
2729                                      .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
2730         },
2731         [MLX5E_TT_IPV6_IPSEC_AH] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
2732                                      .l4_prot_type = 0,
2733                                      .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
2734         },
2735         [MLX5E_TT_IPV4_IPSEC_ESP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
2736                                       .l4_prot_type = 0,
2737                                       .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
2738         },
2739         [MLX5E_TT_IPV6_IPSEC_ESP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
2740                                       .l4_prot_type = 0,
2741                                       .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
2742         },
2743         [MLX5E_TT_IPV4] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
2744                             .l4_prot_type = 0,
2745                             .rx_hash_fields = MLX5_HASH_IP,
2746         },
2747         [MLX5E_TT_IPV6] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
2748                             .l4_prot_type = 0,
2749                             .rx_hash_fields = MLX5_HASH_IP,
2750         },
2751 };
2752
2753 struct mlx5e_tirc_config mlx5e_tirc_get_default_config(enum mlx5e_traffic_types tt)
2754 {
2755         return tirc_default_config[tt];
2756 }
2757
2758 static void mlx5e_build_tir_ctx_lro(struct mlx5e_params *params, void *tirc)
2759 {
2760         if (!params->lro_en)
2761                 return;
2762
2763 #define ROUGH_MAX_L2_L3_HDR_SZ 256
2764
2765         MLX5_SET(tirc, tirc, lro_enable_mask,
2766                  MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
2767                  MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
2768         MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
2769                  (MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ - ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
2770         MLX5_SET(tirc, tirc, lro_timeout_period_usecs, params->lro_timeout);
2771 }
2772
2773 void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_rss_params *rss_params,
2774                                     const struct mlx5e_tirc_config *ttconfig,
2775                                     void *tirc, bool inner)
2776 {
2777         void *hfso = inner ? MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner) :
2778                              MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2779
2780         MLX5_SET(tirc, tirc, rx_hash_fn, mlx5e_rx_hash_fn(rss_params->hfunc));
2781         if (rss_params->hfunc == ETH_RSS_HASH_TOP) {
2782                 void *rss_key = MLX5_ADDR_OF(tirc, tirc,
2783                                              rx_hash_toeplitz_key);
2784                 size_t len = MLX5_FLD_SZ_BYTES(tirc,
2785                                                rx_hash_toeplitz_key);
2786
2787                 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
2788                 memcpy(rss_key, rss_params->toeplitz_hash_key, len);
2789         }
2790         MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2791                  ttconfig->l3_prot_type);
2792         MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2793                  ttconfig->l4_prot_type);
2794         MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2795                  ttconfig->rx_hash_fields);
2796 }
2797
2798 static void mlx5e_update_rx_hash_fields(struct mlx5e_tirc_config *ttconfig,
2799                                         enum mlx5e_traffic_types tt,
2800                                         u32 rx_hash_fields)
2801 {
2802         *ttconfig                = tirc_default_config[tt];
2803         ttconfig->rx_hash_fields = rx_hash_fields;
2804 }
2805
2806 void mlx5e_modify_tirs_hash(struct mlx5e_priv *priv, void *in)
2807 {
2808         void *tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx);
2809         struct mlx5e_rss_params *rss = &priv->rss_params;
2810         struct mlx5_core_dev *mdev = priv->mdev;
2811         int ctxlen = MLX5_ST_SZ_BYTES(tirc);
2812         struct mlx5e_tirc_config ttconfig;
2813         int tt;
2814
2815         MLX5_SET(modify_tir_in, in, bitmask.hash, 1);
2816
2817         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2818                 memset(tirc, 0, ctxlen);
2819                 mlx5e_update_rx_hash_fields(&ttconfig, tt,
2820                                             rss->rx_hash_fields[tt]);
2821                 mlx5e_build_indir_tir_ctx_hash(rss, &ttconfig, tirc, false);
2822                 mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in);
2823         }
2824
2825         /* Verify inner tirs resources allocated */
2826         if (!priv->inner_indir_tir[0].tirn)
2827                 return;
2828
2829         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2830                 memset(tirc, 0, ctxlen);
2831                 mlx5e_update_rx_hash_fields(&ttconfig, tt,
2832                                             rss->rx_hash_fields[tt]);
2833                 mlx5e_build_indir_tir_ctx_hash(rss, &ttconfig, tirc, true);
2834                 mlx5_core_modify_tir(mdev, priv->inner_indir_tir[tt].tirn, in);
2835         }
2836 }
2837
2838 static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv)
2839 {
2840         struct mlx5_core_dev *mdev = priv->mdev;
2841
2842         void *in;
2843         void *tirc;
2844         int inlen;
2845         int err;
2846         int tt;
2847         int ix;
2848
2849         inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
2850         in = kvzalloc(inlen, GFP_KERNEL);
2851         if (!in)
2852                 return -ENOMEM;
2853
2854         MLX5_SET(modify_tir_in, in, bitmask.lro, 1);
2855         tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx);
2856
2857         mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
2858
2859         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2860                 err = mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in);
2861                 if (err)
2862                         goto free_in;
2863         }
2864
2865         for (ix = 0; ix < priv->max_nch; ix++) {
2866                 err = mlx5_core_modify_tir(mdev, priv->direct_tir[ix].tirn, in);
2867                 if (err)
2868                         goto free_in;
2869         }
2870
2871 free_in:
2872         kvfree(in);
2873
2874         return err;
2875 }
2876
2877 static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_lro);
2878
2879 static int mlx5e_set_mtu(struct mlx5_core_dev *mdev,
2880                          struct mlx5e_params *params, u16 mtu)
2881 {
2882         u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu);
2883         int err;
2884
2885         err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
2886         if (err)
2887                 return err;
2888
2889         /* Update vport context MTU */
2890         mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
2891         return 0;
2892 }
2893
2894 static void mlx5e_query_mtu(struct mlx5_core_dev *mdev,
2895                             struct mlx5e_params *params, u16 *mtu)
2896 {
2897         u16 hw_mtu = 0;
2898         int err;
2899
2900         err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
2901         if (err || !hw_mtu) /* fallback to port oper mtu */
2902                 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
2903
2904         *mtu = MLX5E_HW2SW_MTU(params, hw_mtu);
2905 }
2906
2907 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv)
2908 {
2909         struct mlx5e_params *params = &priv->channels.params;
2910         struct net_device *netdev = priv->netdev;
2911         struct mlx5_core_dev *mdev = priv->mdev;
2912         u16 mtu;
2913         int err;
2914
2915         err = mlx5e_set_mtu(mdev, params, params->sw_mtu);
2916         if (err)
2917                 return err;
2918
2919         mlx5e_query_mtu(mdev, params, &mtu);
2920         if (mtu != params->sw_mtu)
2921                 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
2922                             __func__, mtu, params->sw_mtu);
2923
2924         params->sw_mtu = mtu;
2925         return 0;
2926 }
2927
2928 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_set_dev_port_mtu);
2929
2930 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv)
2931 {
2932         struct mlx5e_params *params = &priv->channels.params;
2933         struct net_device *netdev   = priv->netdev;
2934         struct mlx5_core_dev *mdev  = priv->mdev;
2935         u16 max_mtu;
2936
2937         /* MTU range: 68 - hw-specific max */
2938         netdev->min_mtu = ETH_MIN_MTU;
2939
2940         mlx5_query_port_max_mtu(mdev, &max_mtu, 1);
2941         netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu),
2942                                 ETH_MAX_MTU);
2943 }
2944
2945 static void mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc)
2946 {
2947         int tc;
2948
2949         netdev_reset_tc(netdev);
2950
2951         if (ntc == 1)
2952                 return;
2953
2954         netdev_set_num_tc(netdev, ntc);
2955
2956         /* Map netdev TCs to offset 0
2957          * We have our own UP to TXQ mapping for QoS
2958          */
2959         for (tc = 0; tc < ntc; tc++)
2960                 netdev_set_tc_queue(netdev, tc, nch, 0);
2961 }
2962
2963 int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv)
2964 {
2965         int qos_queues, nch, ntc, num_txqs, err;
2966
2967         qos_queues = mlx5e_qos_cur_leaf_nodes(priv);
2968
2969         nch = priv->channels.params.num_channels;
2970         ntc = priv->channels.params.num_tc;
2971         num_txqs = nch * ntc + qos_queues;
2972         if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS))
2973                 num_txqs += ntc;
2974
2975         mlx5e_dbg(DRV, priv, "Setting num_txqs %d\n", num_txqs);
2976         err = netif_set_real_num_tx_queues(priv->netdev, num_txqs);
2977         if (err)
2978                 netdev_warn(priv->netdev, "netif_set_real_num_tx_queues failed, %d\n", err);
2979
2980         return err;
2981 }
2982
2983 static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv)
2984 {
2985         struct net_device *netdev = priv->netdev;
2986         int old_num_txqs, old_ntc;
2987         int num_rxqs, nch, ntc;
2988         int err;
2989
2990         old_num_txqs = netdev->real_num_tx_queues;
2991         old_ntc = netdev->num_tc;
2992
2993         nch = priv->channels.params.num_channels;
2994         ntc = priv->channels.params.num_tc;
2995         num_rxqs = nch * priv->profile->rq_groups;
2996
2997         mlx5e_netdev_set_tcs(netdev, nch, ntc);
2998
2999         err = mlx5e_update_tx_netdev_queues(priv);
3000         if (err)
3001                 goto err_tcs;
3002         err = netif_set_real_num_rx_queues(netdev, num_rxqs);
3003         if (err) {
3004                 netdev_warn(netdev, "netif_set_real_num_rx_queues failed, %d\n", err);
3005                 goto err_txqs;
3006         }
3007
3008         return 0;
3009
3010 err_txqs:
3011         /* netif_set_real_num_rx_queues could fail only when nch increased. Only
3012          * one of nch and ntc is changed in this function. That means, the call
3013          * to netif_set_real_num_tx_queues below should not fail, because it
3014          * decreases the number of TX queues.
3015          */
3016         WARN_ON_ONCE(netif_set_real_num_tx_queues(netdev, old_num_txqs));
3017
3018 err_tcs:
3019         mlx5e_netdev_set_tcs(netdev, old_num_txqs / old_ntc, old_ntc);
3020         return err;
3021 }
3022
3023 static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv,
3024                                            struct mlx5e_params *params)
3025 {
3026         struct mlx5_core_dev *mdev = priv->mdev;
3027         int num_comp_vectors, ix, irq;
3028
3029         num_comp_vectors = mlx5_comp_vectors_count(mdev);
3030
3031         for (ix = 0; ix < params->num_channels; ix++) {
3032                 cpumask_clear(priv->scratchpad.cpumask);
3033
3034                 for (irq = ix; irq < num_comp_vectors; irq += params->num_channels) {
3035                         int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(mdev, irq));
3036
3037                         cpumask_set_cpu(cpu, priv->scratchpad.cpumask);
3038                 }
3039
3040                 netif_set_xps_queue(priv->netdev, priv->scratchpad.cpumask, ix);
3041         }
3042 }
3043
3044 int mlx5e_num_channels_changed(struct mlx5e_priv *priv)
3045 {
3046         u16 count = priv->channels.params.num_channels;
3047         int err;
3048
3049         err = mlx5e_update_netdev_queues(priv);
3050         if (err)
3051                 return err;
3052
3053         mlx5e_set_default_xps_cpumasks(priv, &priv->channels.params);
3054
3055         if (!netif_is_rxfh_configured(priv->netdev))
3056                 mlx5e_build_default_indir_rqt(priv->rss_params.indirection_rqt,
3057                                               MLX5E_INDIR_RQT_SIZE, count);
3058
3059         return 0;
3060 }
3061
3062 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_num_channels_changed);
3063
3064 static void mlx5e_build_txq_maps(struct mlx5e_priv *priv)
3065 {
3066         int i, ch, tc, num_tc;
3067
3068         ch = priv->channels.num;
3069         num_tc = priv->channels.params.num_tc;
3070
3071         for (i = 0; i < ch; i++) {
3072                 for (tc = 0; tc < num_tc; tc++) {
3073                         struct mlx5e_channel *c = priv->channels.c[i];
3074                         struct mlx5e_txqsq *sq = &c->sq[tc];
3075
3076                         priv->txq2sq[sq->txq_ix] = sq;
3077                         priv->channel_tc2realtxq[i][tc] = i + tc * ch;
3078                 }
3079         }
3080
3081         if (!priv->channels.port_ptp)
3082                 return;
3083
3084         for (tc = 0; tc < num_tc; tc++) {
3085                 struct mlx5e_port_ptp *c = priv->channels.port_ptp;
3086                 struct mlx5e_txqsq *sq = &c->ptpsq[tc].txqsq;
3087
3088                 priv->txq2sq[sq->txq_ix] = sq;
3089                 priv->port_ptp_tc2realtxq[tc] = priv->num_tc_x_num_ch + tc;
3090         }
3091 }
3092
3093 static void mlx5e_update_num_tc_x_num_ch(struct mlx5e_priv *priv)
3094 {
3095         /* Sync with mlx5e_select_queue. */
3096         WRITE_ONCE(priv->num_tc_x_num_ch,
3097                    priv->channels.params.num_tc * priv->channels.num);
3098 }
3099
3100 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
3101 {
3102         mlx5e_update_num_tc_x_num_ch(priv);
3103         mlx5e_build_txq_maps(priv);
3104         mlx5e_activate_channels(&priv->channels);
3105         mlx5e_qos_activate_queues(priv);
3106         mlx5e_xdp_tx_enable(priv);
3107         netif_tx_start_all_queues(priv->netdev);
3108
3109         if (mlx5e_is_vport_rep(priv))
3110                 mlx5e_add_sqs_fwd_rules(priv);
3111
3112         mlx5e_wait_channels_min_rx_wqes(&priv->channels);
3113         mlx5e_redirect_rqts_to_channels(priv, &priv->channels);
3114
3115         mlx5e_xsk_redirect_rqts_to_channels(priv, &priv->channels);
3116 }
3117
3118 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
3119 {
3120         mlx5e_xsk_redirect_rqts_to_drop(priv, &priv->channels);
3121
3122         mlx5e_redirect_rqts_to_drop(priv);
3123
3124         if (mlx5e_is_vport_rep(priv))
3125                 mlx5e_remove_sqs_fwd_rules(priv);
3126
3127         /* FIXME: This is a W/A only for tx timeout watch dog false alarm when
3128          * polling for inactive tx queues.
3129          */
3130         netif_tx_stop_all_queues(priv->netdev);
3131         netif_tx_disable(priv->netdev);
3132         mlx5e_xdp_tx_disable(priv);
3133         mlx5e_deactivate_channels(&priv->channels);
3134 }
3135
3136 static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
3137                                       struct mlx5e_channels *new_chs,
3138                                       mlx5e_fp_preactivate preactivate,
3139                                       void *context)
3140 {
3141         struct net_device *netdev = priv->netdev;
3142         struct mlx5e_channels old_chs;
3143         int carrier_ok;
3144         int err = 0;
3145
3146         carrier_ok = netif_carrier_ok(netdev);
3147         netif_carrier_off(netdev);
3148
3149         mlx5e_deactivate_priv_channels(priv);
3150
3151         old_chs = priv->channels;
3152         priv->channels = *new_chs;
3153
3154         /* New channels are ready to roll, call the preactivate hook if needed
3155          * to modify HW settings or update kernel parameters.
3156          */
3157         if (preactivate) {
3158                 err = preactivate(priv, context);
3159                 if (err) {
3160                         priv->channels = old_chs;
3161                         goto out;
3162                 }
3163         }
3164
3165         mlx5e_close_channels(&old_chs);
3166         priv->profile->update_rx(priv);
3167
3168 out:
3169         mlx5e_activate_priv_channels(priv);
3170
3171         /* return carrier back if needed */
3172         if (carrier_ok)
3173                 netif_carrier_on(netdev);
3174
3175         return err;
3176 }
3177
3178 int mlx5e_safe_switch_channels(struct mlx5e_priv *priv,
3179                                struct mlx5e_channels *new_chs,
3180                                mlx5e_fp_preactivate preactivate,
3181                                void *context)
3182 {
3183         int err;
3184
3185         err = mlx5e_open_channels(priv, new_chs);
3186         if (err)
3187                 return err;
3188
3189         err = mlx5e_switch_priv_channels(priv, new_chs, preactivate, context);
3190         if (err)
3191                 goto err_close;
3192
3193         return 0;
3194
3195 err_close:
3196         mlx5e_close_channels(new_chs);
3197
3198         return err;
3199 }
3200
3201 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv)
3202 {
3203         struct mlx5e_channels new_channels = {};
3204
3205         new_channels.params = priv->channels.params;
3206         return mlx5e_safe_switch_channels(priv, &new_channels, NULL, NULL);
3207 }
3208
3209 void mlx5e_timestamp_init(struct mlx5e_priv *priv)
3210 {
3211         priv->tstamp.tx_type   = HWTSTAMP_TX_OFF;
3212         priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE;
3213 }
3214
3215 static void mlx5e_modify_admin_state(struct mlx5_core_dev *mdev,
3216                                      enum mlx5_port_status state)
3217 {
3218         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3219         int vport_admin_state;
3220
3221         mlx5_set_port_admin_status(mdev, state);
3222
3223         if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS ||
3224             !MLX5_CAP_GEN(mdev, uplink_follow))
3225                 return;
3226
3227         if (state == MLX5_PORT_UP)
3228                 vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO;
3229         else
3230                 vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3231
3232         mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state);
3233 }
3234
3235 int mlx5e_open_locked(struct net_device *netdev)
3236 {
3237         struct mlx5e_priv *priv = netdev_priv(netdev);
3238         int err;
3239
3240         set_bit(MLX5E_STATE_OPENED, &priv->state);
3241
3242         err = mlx5e_open_channels(priv, &priv->channels);
3243         if (err)
3244                 goto err_clear_state_opened_flag;
3245
3246         priv->profile->update_rx(priv);
3247         mlx5e_activate_priv_channels(priv);
3248         mlx5e_apply_traps(priv, true);
3249         if (priv->profile->update_carrier)
3250                 priv->profile->update_carrier(priv);
3251
3252         mlx5e_queue_update_stats(priv);
3253         return 0;
3254
3255 err_clear_state_opened_flag:
3256         clear_bit(MLX5E_STATE_OPENED, &priv->state);
3257         return err;
3258 }
3259
3260 int mlx5e_open(struct net_device *netdev)
3261 {
3262         struct mlx5e_priv *priv = netdev_priv(netdev);
3263         int err;
3264
3265         mutex_lock(&priv->state_lock);
3266         err = mlx5e_open_locked(netdev);
3267         if (!err)
3268                 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_UP);
3269         mutex_unlock(&priv->state_lock);
3270
3271         return err;
3272 }
3273
3274 int mlx5e_close_locked(struct net_device *netdev)
3275 {
3276         struct mlx5e_priv *priv = netdev_priv(netdev);
3277
3278         /* May already be CLOSED in case a previous configuration operation
3279          * (e.g RX/TX queue size change) that involves close&open failed.
3280          */
3281         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
3282                 return 0;
3283
3284         mlx5e_apply_traps(priv, false);
3285         clear_bit(MLX5E_STATE_OPENED, &priv->state);
3286
3287         netif_carrier_off(priv->netdev);
3288         mlx5e_deactivate_priv_channels(priv);
3289         mlx5e_close_channels(&priv->channels);
3290
3291         return 0;
3292 }
3293
3294 int mlx5e_close(struct net_device *netdev)
3295 {
3296         struct mlx5e_priv *priv = netdev_priv(netdev);
3297         int err;
3298
3299         if (!netif_device_present(netdev))
3300                 return -ENODEV;
3301
3302         mutex_lock(&priv->state_lock);
3303         mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_DOWN);
3304         err = mlx5e_close_locked(netdev);
3305         mutex_unlock(&priv->state_lock);
3306
3307         return err;
3308 }
3309
3310 static void mlx5e_free_drop_rq(struct mlx5e_rq *rq)
3311 {
3312         mlx5_wq_destroy(&rq->wq_ctrl);
3313 }
3314
3315 static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
3316                                struct mlx5e_rq *rq,
3317                                struct mlx5e_rq_param *param)
3318 {
3319         void *rqc = param->rqc;
3320         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
3321         int err;
3322
3323         param->wq.db_numa_node = param->wq.buf_numa_node;
3324
3325         err = mlx5_wq_cyc_create(mdev, &param->wq, rqc_wq, &rq->wqe.wq,
3326                                  &rq->wq_ctrl);
3327         if (err)
3328                 return err;
3329
3330         /* Mark as unused given "Drop-RQ" packets never reach XDP */
3331         xdp_rxq_info_unused(&rq->xdp_rxq);
3332
3333         rq->mdev = mdev;
3334
3335         return 0;
3336 }
3337
3338 static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv,
3339                                struct mlx5e_cq *cq,
3340                                struct mlx5e_cq_param *param)
3341 {
3342         struct mlx5_core_dev *mdev = priv->mdev;
3343
3344         param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
3345         param->wq.db_numa_node  = dev_to_node(mlx5_core_dma_dev(mdev));
3346
3347         return mlx5e_alloc_cq_common(priv, param, cq);
3348 }
3349
3350 int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
3351                        struct mlx5e_rq *drop_rq)
3352 {
3353         struct mlx5_core_dev *mdev = priv->mdev;
3354         struct mlx5e_cq_param cq_param = {};
3355         struct mlx5e_rq_param rq_param = {};
3356         struct mlx5e_cq *cq = &drop_rq->cq;
3357         int err;
3358
3359         mlx5e_build_drop_rq_param(priv, &rq_param);
3360
3361         err = mlx5e_alloc_drop_cq(priv, cq, &cq_param);
3362         if (err)
3363                 return err;
3364
3365         err = mlx5e_create_cq(cq, &cq_param);
3366         if (err)
3367                 goto err_free_cq;
3368
3369         err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param);
3370         if (err)
3371                 goto err_destroy_cq;
3372
3373         err = mlx5e_create_rq(drop_rq, &rq_param);
3374         if (err)
3375                 goto err_free_rq;
3376
3377         err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
3378         if (err)
3379                 mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err);
3380
3381         return 0;
3382
3383 err_free_rq:
3384         mlx5e_free_drop_rq(drop_rq);
3385
3386 err_destroy_cq:
3387         mlx5e_destroy_cq(cq);
3388
3389 err_free_cq:
3390         mlx5e_free_cq(cq);
3391
3392         return err;
3393 }
3394
3395 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq)
3396 {
3397         mlx5e_destroy_rq(drop_rq);
3398         mlx5e_free_drop_rq(drop_rq);
3399         mlx5e_destroy_cq(&drop_rq->cq);
3400         mlx5e_free_cq(&drop_rq->cq);
3401 }
3402
3403 int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn)
3404 {
3405         void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
3406
3407         MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.td.tdn);
3408
3409         if (MLX5_GET(tisc, tisc, tls_en))
3410                 MLX5_SET(tisc, tisc, pd, mdev->mlx5e_res.pdn);
3411
3412         if (mlx5_lag_is_lacp_owner(mdev))
3413                 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
3414
3415         return mlx5_core_create_tis(mdev, in, tisn);
3416 }
3417
3418 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn)
3419 {
3420         mlx5_core_destroy_tis(mdev, tisn);
3421 }
3422
3423 void mlx5e_destroy_tises(struct mlx5e_priv *priv)
3424 {
3425         int tc, i;
3426
3427         for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++)
3428                 for (tc = 0; tc < priv->profile->max_tc; tc++)
3429                         mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]);
3430 }
3431
3432 static bool mlx5e_lag_should_assign_affinity(struct mlx5_core_dev *mdev)
3433 {
3434         return MLX5_CAP_GEN(mdev, lag_tx_port_affinity) && mlx5e_get_num_lag_ports(mdev) > 1;
3435 }
3436
3437 int mlx5e_create_tises(struct mlx5e_priv *priv)
3438 {
3439         int tc, i;
3440         int err;
3441
3442         for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) {
3443                 for (tc = 0; tc < priv->profile->max_tc; tc++) {
3444                         u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
3445                         void *tisc;
3446
3447                         tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
3448
3449                         MLX5_SET(tisc, tisc, prio, tc << 1);
3450
3451                         if (mlx5e_lag_should_assign_affinity(priv->mdev))
3452                                 MLX5_SET(tisc, tisc, lag_tx_port_affinity, i + 1);
3453
3454                         err = mlx5e_create_tis(priv->mdev, in, &priv->tisn[i][tc]);
3455                         if (err)
3456                                 goto err_close_tises;
3457                 }
3458         }
3459
3460         return 0;
3461
3462 err_close_tises:
3463         for (; i >= 0; i--) {
3464                 for (tc--; tc >= 0; tc--)
3465                         mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]);
3466                 tc = priv->profile->max_tc;
3467         }
3468
3469         return err;
3470 }
3471
3472 static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
3473 {
3474         mlx5e_destroy_tises(priv);
3475 }
3476
3477 static void mlx5e_build_indir_tir_ctx_common(struct mlx5e_priv *priv,
3478                                              u32 rqtn, u32 *tirc)
3479 {
3480         MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
3481         MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
3482         MLX5_SET(tirc, tirc, indirect_table, rqtn);
3483         MLX5_SET(tirc, tirc, tunneled_offload_en,
3484                  priv->channels.params.tunneled_offload_en);
3485
3486         mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
3487 }
3488
3489 static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv,
3490                                       enum mlx5e_traffic_types tt,
3491                                       u32 *tirc)
3492 {
3493         mlx5e_build_indir_tir_ctx_common(priv, priv->indir_rqt.rqtn, tirc);
3494         mlx5e_build_indir_tir_ctx_hash(&priv->rss_params,
3495                                        &tirc_default_config[tt], tirc, false);
3496 }
3497
3498 static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 rqtn, u32 *tirc)
3499 {
3500         mlx5e_build_indir_tir_ctx_common(priv, rqtn, tirc);
3501         MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
3502 }
3503
3504 static void mlx5e_build_inner_indir_tir_ctx(struct mlx5e_priv *priv,
3505                                             enum mlx5e_traffic_types tt,
3506                                             u32 *tirc)
3507 {
3508         mlx5e_build_indir_tir_ctx_common(priv, priv->indir_rqt.rqtn, tirc);
3509         mlx5e_build_indir_tir_ctx_hash(&priv->rss_params,
3510                                        &tirc_default_config[tt], tirc, true);
3511 }
3512
3513 int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc)
3514 {
3515         struct mlx5e_tir *tir;
3516         void *tirc;
3517         int inlen;
3518         int i = 0;
3519         int err;
3520         u32 *in;
3521         int tt;
3522
3523         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
3524         in = kvzalloc(inlen, GFP_KERNEL);
3525         if (!in)
3526                 return -ENOMEM;
3527
3528         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
3529                 memset(in, 0, inlen);
3530                 tir = &priv->indir_tir[tt];
3531                 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
3532                 mlx5e_build_indir_tir_ctx(priv, tt, tirc);
3533                 err = mlx5e_create_tir(priv->mdev, tir, in);
3534                 if (err) {
3535                         mlx5_core_warn(priv->mdev, "create indirect tirs failed, %d\n", err);
3536                         goto err_destroy_inner_tirs;
3537                 }
3538         }
3539
3540         if (!inner_ttc || !mlx5e_tunnel_inner_ft_supported(priv->mdev))
3541                 goto out;
3542
3543         for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) {
3544                 memset(in, 0, inlen);
3545                 tir = &priv->inner_indir_tir[i];
3546                 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
3547                 mlx5e_build_inner_indir_tir_ctx(priv, i, tirc);
3548                 err = mlx5e_create_tir(priv->mdev, tir, in);
3549                 if (err) {
3550                         mlx5_core_warn(priv->mdev, "create inner indirect tirs failed, %d\n", err);
3551                         goto err_destroy_inner_tirs;
3552                 }
3553         }
3554
3555 out:
3556         kvfree(in);
3557
3558         return 0;
3559
3560 err_destroy_inner_tirs:
3561         for (i--; i >= 0; i--)
3562                 mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]);
3563
3564         for (tt--; tt >= 0; tt--)
3565                 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[tt]);
3566
3567         kvfree(in);
3568
3569         return err;
3570 }
3571
3572 int mlx5e_create_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs)
3573 {
3574         struct mlx5e_tir *tir;
3575         void *tirc;
3576         int inlen;
3577         int err = 0;
3578         u32 *in;
3579         int ix;
3580
3581         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
3582         in = kvzalloc(inlen, GFP_KERNEL);
3583         if (!in)
3584                 return -ENOMEM;
3585
3586         for (ix = 0; ix < priv->max_nch; ix++) {
3587                 memset(in, 0, inlen);
3588                 tir = &tirs[ix];
3589                 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
3590                 mlx5e_build_direct_tir_ctx(priv, tir->rqt.rqtn, tirc);
3591                 err = mlx5e_create_tir(priv->mdev, tir, in);
3592                 if (unlikely(err))
3593                         goto err_destroy_ch_tirs;
3594         }
3595
3596         goto out;
3597
3598 err_destroy_ch_tirs:
3599         mlx5_core_warn(priv->mdev, "create tirs failed, %d\n", err);
3600         for (ix--; ix >= 0; ix--)
3601                 mlx5e_destroy_tir(priv->mdev, &tirs[ix]);
3602
3603 out:
3604         kvfree(in);
3605
3606         return err;
3607 }
3608
3609 void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv)
3610 {
3611         int i;
3612
3613         for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
3614                 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[i]);
3615
3616         /* Verify inner tirs resources allocated */
3617         if (!priv->inner_indir_tir[0].tirn)
3618                 return;
3619
3620         for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
3621                 mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]);
3622 }
3623
3624 void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs)
3625 {
3626         int i;
3627
3628         for (i = 0; i < priv->max_nch; i++)
3629                 mlx5e_destroy_tir(priv->mdev, &tirs[i]);
3630 }
3631
3632 static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels *chs, bool enable)
3633 {
3634         int err = 0;
3635         int i;
3636
3637         for (i = 0; i < chs->num; i++) {
3638                 err = mlx5e_modify_rq_scatter_fcs(&chs->c[i]->rq, enable);
3639                 if (err)
3640                         return err;
3641         }
3642
3643         return 0;
3644 }
3645
3646 static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd)
3647 {
3648         int err = 0;
3649         int i;
3650
3651         for (i = 0; i < chs->num; i++) {
3652                 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd);
3653                 if (err)
3654                         return err;
3655         }
3656
3657         return 0;
3658 }
3659
3660 static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv,
3661                                  struct tc_mqprio_qopt *mqprio)
3662 {
3663         struct mlx5e_channels new_channels = {};
3664         u8 tc = mqprio->num_tc;
3665         int err = 0;
3666
3667         mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
3668
3669         if (tc && tc != MLX5E_MAX_NUM_TC)
3670                 return -EINVAL;
3671
3672         mutex_lock(&priv->state_lock);
3673
3674         /* MQPRIO is another toplevel qdisc that can't be attached
3675          * simultaneously with the offloaded HTB.
3676          */
3677         if (WARN_ON(priv->htb.maj_id)) {
3678                 err = -EINVAL;
3679                 goto out;
3680         }
3681
3682         new_channels.params = priv->channels.params;
3683         new_channels.params.num_tc = tc ? tc : 1;
3684
3685         if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
3686                 struct mlx5e_params old_params;
3687
3688                 old_params = priv->channels.params;
3689                 priv->channels.params = new_channels.params;
3690                 err = mlx5e_num_channels_changed(priv);
3691                 if (err)
3692                         priv->channels.params = old_params;
3693
3694                 goto out;
3695         }
3696
3697         err = mlx5e_safe_switch_channels(priv, &new_channels,
3698                                          mlx5e_num_channels_changed_ctx, NULL);
3699
3700 out:
3701         priv->max_opened_tc = max_t(u8, priv->max_opened_tc,
3702                                     priv->channels.params.num_tc);
3703         mutex_unlock(&priv->state_lock);
3704         return err;
3705 }
3706
3707 static int mlx5e_setup_tc_htb(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb)
3708 {
3709         int res;
3710
3711         switch (htb->command) {
3712         case TC_HTB_CREATE:
3713                 return mlx5e_htb_root_add(priv, htb->parent_classid, htb->classid,
3714                                           htb->extack);
3715         case TC_HTB_DESTROY:
3716                 return mlx5e_htb_root_del(priv);
3717         case TC_HTB_LEAF_ALLOC_QUEUE:
3718                 res = mlx5e_htb_leaf_alloc_queue(priv, htb->classid, htb->parent_classid,
3719                                                  htb->rate, htb->ceil, htb->extack);
3720                 if (res < 0)
3721                         return res;
3722                 htb->qid = res;
3723                 return 0;
3724         case TC_HTB_LEAF_TO_INNER:
3725                 return mlx5e_htb_leaf_to_inner(priv, htb->parent_classid, htb->classid,
3726                                                htb->rate, htb->ceil, htb->extack);
3727         case TC_HTB_LEAF_DEL:
3728                 return mlx5e_htb_leaf_del(priv, htb->classid, &htb->moved_qid, &htb->qid,
3729                                           htb->extack);
3730         case TC_HTB_LEAF_DEL_LAST:
3731         case TC_HTB_LEAF_DEL_LAST_FORCE:
3732                 return mlx5e_htb_leaf_del_last(priv, htb->classid,
3733                                                htb->command == TC_HTB_LEAF_DEL_LAST_FORCE,
3734                                                htb->extack);
3735         case TC_HTB_NODE_MODIFY:
3736                 return mlx5e_htb_node_modify(priv, htb->classid, htb->rate, htb->ceil,
3737                                              htb->extack);
3738         case TC_HTB_LEAF_QUERY_QUEUE:
3739                 res = mlx5e_get_txq_by_classid(priv, htb->classid);
3740                 if (res < 0)
3741                         return res;
3742                 htb->qid = res;
3743                 return 0;
3744         default:
3745                 return -EOPNOTSUPP;
3746         }
3747 }
3748
3749 static LIST_HEAD(mlx5e_block_cb_list);
3750
3751 static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
3752                           void *type_data)
3753 {
3754         struct mlx5e_priv *priv = netdev_priv(dev);
3755         int err;
3756
3757         switch (type) {
3758         case TC_SETUP_BLOCK: {
3759                 struct flow_block_offload *f = type_data;
3760
3761                 f->unlocked_driver_cb = true;
3762                 return flow_block_cb_setup_simple(type_data,
3763                                                   &mlx5e_block_cb_list,
3764                                                   mlx5e_setup_tc_block_cb,
3765                                                   priv, priv, true);
3766         }
3767         case TC_SETUP_QDISC_MQPRIO:
3768                 return mlx5e_setup_tc_mqprio(priv, type_data);
3769         case TC_SETUP_QDISC_HTB:
3770                 mutex_lock(&priv->state_lock);
3771                 err = mlx5e_setup_tc_htb(priv, type_data);
3772                 mutex_unlock(&priv->state_lock);
3773                 return err;
3774         default:
3775                 return -EOPNOTSUPP;
3776         }
3777 }
3778
3779 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s)
3780 {
3781         int i;
3782
3783         for (i = 0; i < priv->max_nch; i++) {
3784                 struct mlx5e_channel_stats *channel_stats = &priv->channel_stats[i];
3785                 struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq;
3786                 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;
3787                 int j;
3788
3789                 s->rx_packets   += rq_stats->packets + xskrq_stats->packets;
3790                 s->rx_bytes     += rq_stats->bytes + xskrq_stats->bytes;
3791                 s->multicast    += rq_stats->mcast_packets + xskrq_stats->mcast_packets;
3792
3793                 for (j = 0; j < priv->max_opened_tc; j++) {
3794                         struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];
3795
3796                         s->tx_packets    += sq_stats->packets;
3797                         s->tx_bytes      += sq_stats->bytes;
3798                         s->tx_dropped    += sq_stats->dropped;
3799                 }
3800         }
3801 }
3802
3803 void
3804 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
3805 {
3806         struct mlx5e_priv *priv = netdev_priv(dev);
3807         struct mlx5e_pport_stats *pstats = &priv->stats.pport;
3808
3809         /* In switchdev mode, monitor counters doesn't monitor
3810          * rx/tx stats of 802_3. The update stats mechanism
3811          * should keep the 802_3 layout counters updated
3812          */
3813         if (!mlx5e_monitor_counter_supported(priv) ||
3814             mlx5e_is_uplink_rep(priv)) {
3815                 /* update HW stats in background for next time */
3816                 mlx5e_queue_update_stats(priv);
3817         }
3818
3819         if (mlx5e_is_uplink_rep(priv)) {
3820                 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok);
3821                 stats->rx_bytes   = PPORT_802_3_GET(pstats, a_octets_received_ok);
3822                 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok);
3823                 stats->tx_bytes   = PPORT_802_3_GET(pstats, a_octets_transmitted_ok);
3824         } else {
3825                 mlx5e_fold_sw_stats64(priv, stats);
3826         }
3827
3828         stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
3829
3830         stats->rx_length_errors =
3831                 PPORT_802_3_GET(pstats, a_in_range_length_errors) +
3832                 PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
3833                 PPORT_802_3_GET(pstats, a_frame_too_long_errors);
3834         stats->rx_crc_errors =
3835                 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
3836         stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
3837         stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
3838         stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
3839                            stats->rx_frame_errors;
3840         stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
3841 }
3842
3843 static void mlx5e_set_rx_mode(struct net_device *dev)
3844 {
3845         struct mlx5e_priv *priv = netdev_priv(dev);
3846
3847         queue_work(priv->wq, &priv->set_rx_mode_work);
3848 }
3849
3850 static int mlx5e_set_mac(struct net_device *netdev, void *addr)
3851 {
3852         struct mlx5e_priv *priv = netdev_priv(netdev);
3853         struct sockaddr *saddr = addr;
3854
3855         if (!is_valid_ether_addr(saddr->sa_data))
3856                 return -EADDRNOTAVAIL;
3857
3858         netif_addr_lock_bh(netdev);
3859         ether_addr_copy(netdev->dev_addr, saddr->sa_data);
3860         netif_addr_unlock_bh(netdev);
3861
3862         queue_work(priv->wq, &priv->set_rx_mode_work);
3863
3864         return 0;
3865 }
3866
3867 #define MLX5E_SET_FEATURE(features, feature, enable)    \
3868         do {                                            \
3869                 if (enable)                             \
3870                         *features |= feature;           \
3871                 else                                    \
3872                         *features &= ~feature;          \
3873         } while (0)
3874
3875 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
3876
3877 static int set_feature_lro(struct net_device *netdev, bool enable)
3878 {
3879         struct mlx5e_priv *priv = netdev_priv(netdev);
3880         struct mlx5_core_dev *mdev = priv->mdev;
3881         struct mlx5e_channels new_channels = {};
3882         struct mlx5e_params *cur_params;
3883         int err = 0;
3884         bool reset;
3885
3886         mutex_lock(&priv->state_lock);
3887
3888         if (enable && priv->xsk.refcnt) {
3889                 netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n",
3890                             priv->xsk.refcnt);
3891                 err = -EINVAL;
3892                 goto out;
3893         }
3894
3895         cur_params = &priv->channels.params;
3896         if (enable && !MLX5E_GET_PFLAG(cur_params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
3897                 netdev_warn(netdev, "can't set LRO with legacy RQ\n");
3898                 err = -EINVAL;
3899                 goto out;
3900         }
3901
3902         reset = test_bit(MLX5E_STATE_OPENED, &priv->state);
3903
3904         new_channels.params = *cur_params;
3905         new_channels.params.lro_en = enable;
3906
3907         if (cur_params->rq_wq_type != MLX5_WQ_TYPE_CYCLIC) {
3908                 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, cur_params, NULL) ==
3909                     mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_channels.params, NULL))
3910                         reset = false;
3911         }
3912
3913         if (!reset) {
3914                 struct mlx5e_params old_params;
3915
3916                 old_params = *cur_params;
3917                 *cur_params = new_channels.params;
3918                 err = mlx5e_modify_tirs_lro(priv);
3919                 if (err)
3920                         *cur_params = old_params;
3921                 goto out;
3922         }
3923
3924         err = mlx5e_safe_switch_channels(priv, &new_channels,
3925                                          mlx5e_modify_tirs_lro_ctx, NULL);
3926 out:
3927         mutex_unlock(&priv->state_lock);
3928         return err;
3929 }
3930
3931 static int set_feature_cvlan_filter(struct net_device *netdev, bool enable)
3932 {
3933         struct mlx5e_priv *priv = netdev_priv(netdev);
3934
3935         if (enable)
3936                 mlx5e_enable_cvlan_filter(priv);
3937         else
3938                 mlx5e_disable_cvlan_filter(priv);
3939
3940         return 0;
3941 }
3942
3943 static int set_feature_hw_tc(struct net_device *netdev, bool enable)
3944 {
3945         struct mlx5e_priv *priv = netdev_priv(netdev);
3946
3947 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
3948         if (!enable && mlx5e_tc_num_filters(priv, MLX5_TC_FLAG(NIC_OFFLOAD))) {
3949                 netdev_err(netdev,
3950                            "Active offloaded tc filters, can't turn hw_tc_offload off\n");
3951                 return -EINVAL;
3952         }
3953 #endif
3954
3955         if (!enable && priv->htb.maj_id) {
3956                 netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n");
3957                 return -EINVAL;
3958         }
3959
3960         return 0;
3961 }
3962
3963 static int set_feature_rx_all(struct net_device *netdev, bool enable)
3964 {
3965         struct mlx5e_priv *priv = netdev_priv(netdev);
3966         struct mlx5_core_dev *mdev = priv->mdev;
3967
3968         return mlx5_set_port_fcs(mdev, !enable);
3969 }
3970
3971 static int set_feature_rx_fcs(struct net_device *netdev, bool enable)
3972 {
3973         struct mlx5e_priv *priv = netdev_priv(netdev);
3974         int err;
3975
3976         mutex_lock(&priv->state_lock);
3977
3978         priv->channels.params.scatter_fcs_en = enable;
3979         err = mlx5e_modify_channels_scatter_fcs(&priv->channels, enable);
3980         if (err)
3981                 priv->channels.params.scatter_fcs_en = !enable;
3982
3983         mutex_unlock(&priv->state_lock);
3984
3985         return err;
3986 }
3987
3988 static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
3989 {
3990         struct mlx5e_priv *priv = netdev_priv(netdev);
3991         int err = 0;
3992
3993         mutex_lock(&priv->state_lock);
3994
3995         priv->channels.params.vlan_strip_disable = !enable;
3996         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
3997                 goto unlock;
3998
3999         err = mlx5e_modify_channels_vsd(&priv->channels, !enable);
4000         if (err)
4001                 priv->channels.params.vlan_strip_disable = enable;
4002
4003 unlock:
4004         mutex_unlock(&priv->state_lock);
4005
4006         return err;
4007 }
4008
4009 #ifdef CONFIG_MLX5_EN_ARFS
4010 static int set_feature_arfs(struct net_device *netdev, bool enable)
4011 {
4012         struct mlx5e_priv *priv = netdev_priv(netdev);
4013         int err;
4014
4015         if (enable)
4016                 err = mlx5e_arfs_enable(priv);
4017         else
4018                 err = mlx5e_arfs_disable(priv);
4019
4020         return err;
4021 }
4022 #endif
4023
4024 static int mlx5e_handle_feature(struct net_device *netdev,
4025                                 netdev_features_t *features,
4026                                 netdev_features_t wanted_features,
4027                                 netdev_features_t feature,
4028                                 mlx5e_feature_handler feature_handler)
4029 {
4030         netdev_features_t changes = wanted_features ^ netdev->features;
4031         bool enable = !!(wanted_features & feature);
4032         int err;
4033
4034         if (!(changes & feature))
4035                 return 0;
4036
4037         err = feature_handler(netdev, enable);
4038         if (err) {
4039                 netdev_err(netdev, "%s feature %pNF failed, err %d\n",
4040                            enable ? "Enable" : "Disable", &feature, err);
4041                 return err;
4042         }
4043
4044         MLX5E_SET_FEATURE(features, feature, enable);
4045         return 0;
4046 }
4047
4048 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features)
4049 {
4050         netdev_features_t oper_features = netdev->features;
4051         int err = 0;
4052
4053 #define MLX5E_HANDLE_FEATURE(feature, handler) \
4054         mlx5e_handle_feature(netdev, &oper_features, features, feature, handler)
4055
4056         err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro);
4057         err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER,
4058                                     set_feature_cvlan_filter);
4059         err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_hw_tc);
4060         err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all);
4061         err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs);
4062         err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan);
4063 #ifdef CONFIG_MLX5_EN_ARFS
4064         err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs);
4065 #endif
4066         err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TLS_RX, mlx5e_ktls_set_feature_rx);
4067
4068         if (err) {
4069                 netdev->features = oper_features;
4070                 return -EINVAL;
4071         }
4072
4073         return 0;
4074 }
4075
4076 static netdev_features_t mlx5e_fix_features(struct net_device *netdev,
4077                                             netdev_features_t features)
4078 {
4079         struct mlx5e_priv *priv = netdev_priv(netdev);
4080         struct mlx5e_params *params;
4081
4082         mutex_lock(&priv->state_lock);
4083         params = &priv->channels.params;
4084         if (!bitmap_empty(priv->fs.vlan.active_svlans, VLAN_N_VID)) {
4085                 /* HW strips the outer C-tag header, this is a problem
4086                  * for S-tag traffic.
4087                  */
4088                 features &= ~NETIF_F_HW_VLAN_CTAG_RX;
4089                 if (!params->vlan_strip_disable)
4090                         netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n");
4091         }
4092
4093         if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
4094                 if (features & NETIF_F_LRO) {
4095                         netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n");
4096                         features &= ~NETIF_F_LRO;
4097                 }
4098         }
4099
4100         if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
4101                 features &= ~NETIF_F_RXHASH;
4102                 if (netdev->features & NETIF_F_RXHASH)
4103                         netdev_warn(netdev, "Disabling rxhash, not supported when CQE compress is active\n");
4104         }
4105
4106         mutex_unlock(&priv->state_lock);
4107
4108         return features;
4109 }
4110
4111 static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
4112                                    struct mlx5e_channels *chs,
4113                                    struct mlx5e_params *new_params,
4114                                    struct mlx5_core_dev *mdev)
4115 {
4116         u16 ix;
4117
4118         for (ix = 0; ix < chs->params.num_channels; ix++) {
4119                 struct xsk_buff_pool *xsk_pool =
4120                         mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix);
4121                 struct mlx5e_xsk_param xsk;
4122
4123                 if (!xsk_pool)
4124                         continue;
4125
4126                 mlx5e_build_xsk_param(xsk_pool, &xsk);
4127
4128                 if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev)) {
4129                         u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk);
4130                         int max_mtu_frame, max_mtu_page, max_mtu;
4131
4132                         /* Two criteria must be met:
4133                          * 1. HW MTU + all headrooms <= XSK frame size.
4134                          * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE.
4135                          */
4136                         max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr);
4137                         max_mtu_page = mlx5e_xdp_max_mtu(new_params, &xsk);
4138                         max_mtu = min(max_mtu_frame, max_mtu_page);
4139
4140                         netdev_err(netdev, "MTU %d is too big for an XSK running on channel %u. Try MTU <= %d\n",
4141                                    new_params->sw_mtu, ix, max_mtu);
4142                         return false;
4143                 }
4144         }
4145
4146         return true;
4147 }
4148
4149 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
4150                      mlx5e_fp_preactivate preactivate)
4151 {
4152         struct mlx5e_priv *priv = netdev_priv(netdev);
4153         struct mlx5e_channels new_channels = {};
4154         struct mlx5e_params *params;
4155         int err = 0;
4156         bool reset;
4157
4158         mutex_lock(&priv->state_lock);
4159
4160         params = &priv->channels.params;
4161
4162         reset = !params->lro_en;
4163         reset = reset && test_bit(MLX5E_STATE_OPENED, &priv->state);
4164
4165         new_channels.params = *params;
4166         new_channels.params.sw_mtu = new_mtu;
4167         err = mlx5e_validate_params(priv, &new_channels.params);
4168         if (err)
4169                 goto out;
4170
4171         if (params->xdp_prog &&
4172             !mlx5e_rx_is_linear_skb(&new_channels.params, NULL)) {
4173                 netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n",
4174                            new_mtu, mlx5e_xdp_max_mtu(params, NULL));
4175                 err = -EINVAL;
4176                 goto out;
4177         }
4178
4179         if (priv->xsk.refcnt &&
4180             !mlx5e_xsk_validate_mtu(netdev, &priv->channels,
4181                                     &new_channels.params, priv->mdev)) {
4182                 err = -EINVAL;
4183                 goto out;
4184         }
4185
4186         if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
4187                 bool is_linear = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev,
4188                                                               &new_channels.params,
4189                                                               NULL);
4190                 u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params, NULL);
4191                 u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_channels.params, NULL);
4192
4193                 /* If XSK is active, XSK RQs are linear. */
4194                 is_linear |= priv->xsk.refcnt;
4195
4196                 /* Always reset in linear mode - hw_mtu is used in data path. */
4197                 reset = reset && (is_linear || (ppw_old != ppw_new));
4198         }
4199
4200         if (!reset) {
4201                 unsigned int old_mtu = params->sw_mtu;
4202
4203                 params->sw_mtu = new_mtu;
4204                 if (preactivate) {
4205                         err = preactivate(priv, NULL);
4206                         if (err) {
4207                                 params->sw_mtu = old_mtu;
4208                                 goto out;
4209                         }
4210                 }
4211                 netdev->mtu = params->sw_mtu;
4212                 goto out;
4213         }
4214
4215         err = mlx5e_safe_switch_channels(priv, &new_channels, preactivate, NULL);
4216         if (err)
4217                 goto out;
4218
4219         netdev->mtu = new_channels.params.sw_mtu;
4220
4221 out:
4222         mutex_unlock(&priv->state_lock);
4223         return err;
4224 }
4225
4226 static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu)
4227 {
4228         return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu_ctx);
4229 }
4230
4231 int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr)
4232 {
4233         struct hwtstamp_config config;
4234         int err;
4235
4236         if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) ||
4237             (mlx5_clock_get_ptp_index(priv->mdev) == -1))
4238                 return -EOPNOTSUPP;
4239
4240         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
4241                 return -EFAULT;
4242
4243         /* TX HW timestamp */
4244         switch (config.tx_type) {
4245         case HWTSTAMP_TX_OFF:
4246         case HWTSTAMP_TX_ON:
4247                 break;
4248         default:
4249                 return -ERANGE;
4250         }
4251
4252         mutex_lock(&priv->state_lock);
4253         /* RX HW timestamp */
4254         switch (config.rx_filter) {
4255         case HWTSTAMP_FILTER_NONE:
4256                 /* Reset CQE compression to Admin default */
4257                 mlx5e_modify_rx_cqe_compression_locked(priv, priv->channels.params.rx_cqe_compress_def);
4258                 break;
4259         case HWTSTAMP_FILTER_ALL:
4260         case HWTSTAMP_FILTER_SOME:
4261         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
4262         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
4263         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
4264         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
4265         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
4266         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
4267         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
4268         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
4269         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
4270         case HWTSTAMP_FILTER_PTP_V2_EVENT:
4271         case HWTSTAMP_FILTER_PTP_V2_SYNC:
4272         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
4273         case HWTSTAMP_FILTER_NTP_ALL:
4274                 /* Disable CQE compression */
4275                 if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS))
4276                         netdev_warn(priv->netdev, "Disabling RX cqe compression\n");
4277                 err = mlx5e_modify_rx_cqe_compression_locked(priv, false);
4278                 if (err) {
4279                         netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err);
4280                         mutex_unlock(&priv->state_lock);
4281                         return err;
4282                 }
4283                 config.rx_filter = HWTSTAMP_FILTER_ALL;
4284                 break;
4285         default:
4286                 mutex_unlock(&priv->state_lock);
4287                 return -ERANGE;
4288         }
4289
4290         memcpy(&priv->tstamp, &config, sizeof(config));
4291         mutex_unlock(&priv->state_lock);
4292
4293         /* might need to fix some features */
4294         netdev_update_features(priv->netdev);
4295
4296         return copy_to_user(ifr->ifr_data, &config,
4297                             sizeof(config)) ? -EFAULT : 0;
4298 }
4299
4300 int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr)
4301 {
4302         struct hwtstamp_config *cfg = &priv->tstamp;
4303
4304         if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz))
4305                 return -EOPNOTSUPP;
4306
4307         return copy_to_user(ifr->ifr_data, cfg, sizeof(*cfg)) ? -EFAULT : 0;
4308 }
4309
4310 static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
4311 {
4312         struct mlx5e_priv *priv = netdev_priv(dev);
4313
4314         switch (cmd) {
4315         case SIOCSHWTSTAMP:
4316                 return mlx5e_hwstamp_set(priv, ifr);
4317         case SIOCGHWTSTAMP:
4318                 return mlx5e_hwstamp_get(priv, ifr);
4319         default:
4320                 return -EOPNOTSUPP;
4321         }
4322 }
4323
4324 #ifdef CONFIG_MLX5_ESWITCH
4325 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
4326 {
4327         struct mlx5e_priv *priv = netdev_priv(dev);
4328         struct mlx5_core_dev *mdev = priv->mdev;
4329
4330         return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
4331 }
4332
4333 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
4334                              __be16 vlan_proto)
4335 {
4336         struct mlx5e_priv *priv = netdev_priv(dev);
4337         struct mlx5_core_dev *mdev = priv->mdev;
4338
4339         if (vlan_proto != htons(ETH_P_8021Q))
4340                 return -EPROTONOSUPPORT;
4341
4342         return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
4343                                            vlan, qos);
4344 }
4345
4346 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
4347 {
4348         struct mlx5e_priv *priv = netdev_priv(dev);
4349         struct mlx5_core_dev *mdev = priv->mdev;
4350
4351         return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting);
4352 }
4353
4354 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
4355 {
4356         struct mlx5e_priv *priv = netdev_priv(dev);
4357         struct mlx5_core_dev *mdev = priv->mdev;
4358
4359         return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting);
4360 }
4361
4362 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
4363                       int max_tx_rate)
4364 {
4365         struct mlx5e_priv *priv = netdev_priv(dev);
4366         struct mlx5_core_dev *mdev = priv->mdev;
4367
4368         return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1,
4369                                            max_tx_rate, min_tx_rate);
4370 }
4371
4372 static int mlx5_vport_link2ifla(u8 esw_link)
4373 {
4374         switch (esw_link) {
4375         case MLX5_VPORT_ADMIN_STATE_DOWN:
4376                 return IFLA_VF_LINK_STATE_DISABLE;
4377         case MLX5_VPORT_ADMIN_STATE_UP:
4378                 return IFLA_VF_LINK_STATE_ENABLE;
4379         }
4380         return IFLA_VF_LINK_STATE_AUTO;
4381 }
4382
4383 static int mlx5_ifla_link2vport(u8 ifla_link)
4384 {
4385         switch (ifla_link) {
4386         case IFLA_VF_LINK_STATE_DISABLE:
4387                 return MLX5_VPORT_ADMIN_STATE_DOWN;
4388         case IFLA_VF_LINK_STATE_ENABLE:
4389                 return MLX5_VPORT_ADMIN_STATE_UP;
4390         }
4391         return MLX5_VPORT_ADMIN_STATE_AUTO;
4392 }
4393
4394 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
4395                                    int link_state)
4396 {
4397         struct mlx5e_priv *priv = netdev_priv(dev);
4398         struct mlx5_core_dev *mdev = priv->mdev;
4399
4400         return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1,
4401                                             mlx5_ifla_link2vport(link_state));
4402 }
4403
4404 int mlx5e_get_vf_config(struct net_device *dev,
4405                         int vf, struct ifla_vf_info *ivi)
4406 {
4407         struct mlx5e_priv *priv = netdev_priv(dev);
4408         struct mlx5_core_dev *mdev = priv->mdev;
4409         int err;
4410
4411         err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi);
4412         if (err)
4413                 return err;
4414         ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate);
4415         return 0;
4416 }
4417
4418 int mlx5e_get_vf_stats(struct net_device *dev,
4419                        int vf, struct ifla_vf_stats *vf_stats)
4420 {
4421         struct mlx5e_priv *priv = netdev_priv(dev);
4422         struct mlx5_core_dev *mdev = priv->mdev;
4423
4424         return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
4425                                             vf_stats);
4426 }
4427 #endif
4428
4429 static bool mlx5e_tunnel_proto_supported_tx(struct mlx5_core_dev *mdev, u8 proto_type)
4430 {
4431         switch (proto_type) {
4432         case IPPROTO_GRE:
4433                 return MLX5_CAP_ETH(mdev, tunnel_stateless_gre);
4434         case IPPROTO_IPIP:
4435         case IPPROTO_IPV6:
4436                 return (MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip) ||
4437                         MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip_tx));
4438         default:
4439                 return false;
4440         }
4441 }
4442
4443 static bool mlx5e_gre_tunnel_inner_proto_offload_supported(struct mlx5_core_dev *mdev,
4444                                                            struct sk_buff *skb)
4445 {
4446         switch (skb->inner_protocol) {
4447         case htons(ETH_P_IP):
4448         case htons(ETH_P_IPV6):
4449         case htons(ETH_P_TEB):
4450                 return true;
4451         case htons(ETH_P_MPLS_UC):
4452         case htons(ETH_P_MPLS_MC):
4453                 return MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre);
4454         }
4455         return false;
4456 }
4457
4458 static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv,
4459                                                      struct sk_buff *skb,
4460                                                      netdev_features_t features)
4461 {
4462         unsigned int offset = 0;
4463         struct udphdr *udph;
4464         u8 proto;
4465         u16 port;
4466
4467         switch (vlan_get_protocol(skb)) {
4468         case htons(ETH_P_IP):
4469                 proto = ip_hdr(skb)->protocol;
4470                 break;
4471         case htons(ETH_P_IPV6):
4472                 proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL);
4473                 break;
4474         default:
4475                 goto out;
4476         }
4477
4478         switch (proto) {
4479         case IPPROTO_GRE:
4480                 if (mlx5e_gre_tunnel_inner_proto_offload_supported(priv->mdev, skb))
4481                         return features;
4482                 break;
4483         case IPPROTO_IPIP:
4484         case IPPROTO_IPV6:
4485                 if (mlx5e_tunnel_proto_supported_tx(priv->mdev, IPPROTO_IPIP))
4486                         return features;
4487                 break;
4488         case IPPROTO_UDP:
4489                 udph = udp_hdr(skb);
4490                 port = be16_to_cpu(udph->dest);
4491
4492                 /* Verify if UDP port is being offloaded by HW */
4493                 if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port))
4494                         return features;
4495
4496 #if IS_ENABLED(CONFIG_GENEVE)
4497                 /* Support Geneve offload for default UDP port */
4498                 if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev))
4499                         return features;
4500 #endif
4501         }
4502
4503 out:
4504         /* Disable CSUM and GSO if the udp dport is not offloaded by HW */
4505         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
4506 }
4507
4508 netdev_features_t mlx5e_features_check(struct sk_buff *skb,
4509                                        struct net_device *netdev,
4510                                        netdev_features_t features)
4511 {
4512         struct mlx5e_priv *priv = netdev_priv(netdev);
4513
4514         features = vlan_features_check(skb, features);
4515         features = vxlan_features_check(skb, features);
4516
4517         if (mlx5e_ipsec_feature_check(skb, netdev, features))
4518                 return features;
4519
4520         /* Validate if the tunneled packet is being offloaded by HW */
4521         if (skb->encapsulation &&
4522             (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
4523                 return mlx5e_tunnel_features_check(priv, skb, features);
4524
4525         return features;
4526 }
4527
4528 static void mlx5e_tx_timeout_work(struct work_struct *work)
4529 {
4530         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
4531                                                tx_timeout_work);
4532         struct net_device *netdev = priv->netdev;
4533         int i;
4534
4535         rtnl_lock();
4536         mutex_lock(&priv->state_lock);
4537
4538         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
4539                 goto unlock;
4540
4541         for (i = 0; i < netdev->real_num_tx_queues; i++) {
4542                 struct netdev_queue *dev_queue =
4543                         netdev_get_tx_queue(netdev, i);
4544                 struct mlx5e_txqsq *sq = priv->txq2sq[i];
4545
4546                 if (!netif_xmit_stopped(dev_queue))
4547                         continue;
4548
4549                 if (mlx5e_reporter_tx_timeout(sq))
4550                 /* break if tried to reopened channels */
4551                         break;
4552         }
4553
4554 unlock:
4555         mutex_unlock(&priv->state_lock);
4556         rtnl_unlock();
4557 }
4558
4559 static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue)
4560 {
4561         struct mlx5e_priv *priv = netdev_priv(dev);
4562
4563         netdev_err(dev, "TX timeout detected\n");
4564         queue_work(priv->wq, &priv->tx_timeout_work);
4565 }
4566
4567 static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog)
4568 {
4569         struct net_device *netdev = priv->netdev;
4570         struct mlx5e_channels new_channels = {};
4571
4572         if (priv->channels.params.lro_en) {
4573                 netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n");
4574                 return -EINVAL;
4575         }
4576
4577         if (MLX5_IPSEC_DEV(priv->mdev)) {
4578                 netdev_warn(netdev, "can't set XDP with IPSec offload\n");
4579                 return -EINVAL;
4580         }
4581
4582         new_channels.params = priv->channels.params;
4583         new_channels.params.xdp_prog = prog;
4584
4585         /* No XSK params: AF_XDP can't be enabled yet at the point of setting
4586          * the XDP program.
4587          */
4588         if (!mlx5e_rx_is_linear_skb(&new_channels.params, NULL)) {
4589                 netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n",
4590                             new_channels.params.sw_mtu,
4591                             mlx5e_xdp_max_mtu(&new_channels.params, NULL));
4592                 return -EINVAL;
4593         }
4594
4595         return 0;
4596 }
4597
4598 static void mlx5e_rq_replace_xdp_prog(struct mlx5e_rq *rq, struct bpf_prog *prog)
4599 {
4600         struct bpf_prog *old_prog;
4601
4602         old_prog = rcu_replace_pointer(rq->xdp_prog, prog,
4603                                        lockdep_is_held(&rq->priv->state_lock));
4604         if (old_prog)
4605                 bpf_prog_put(old_prog);
4606 }
4607
4608 static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
4609 {
4610         struct mlx5e_priv *priv = netdev_priv(netdev);
4611         struct bpf_prog *old_prog;
4612         bool reset, was_opened;
4613         int err = 0;
4614         int i;
4615
4616         mutex_lock(&priv->state_lock);
4617
4618         if (prog) {
4619                 err = mlx5e_xdp_allowed(priv, prog);
4620                 if (err)
4621                         goto unlock;
4622         }
4623
4624         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
4625         /* no need for full reset when exchanging programs */
4626         reset = (!priv->channels.params.xdp_prog || !prog);
4627
4628         if (was_opened && !reset)
4629                 /* num_channels is invariant here, so we can take the
4630                  * batched reference right upfront.
4631                  */
4632                 bpf_prog_add(prog, priv->channels.num);
4633
4634         if (was_opened && reset) {
4635                 struct mlx5e_channels new_channels = {};
4636
4637                 new_channels.params = priv->channels.params;
4638                 new_channels.params.xdp_prog = prog;
4639                 mlx5e_set_rq_type(priv->mdev, &new_channels.params);
4640                 old_prog = priv->channels.params.xdp_prog;
4641
4642                 err = mlx5e_safe_switch_channels(priv, &new_channels, NULL, NULL);
4643                 if (err)
4644                         goto unlock;
4645         } else {
4646                 /* exchange programs, extra prog reference we got from caller
4647                  * as long as we don't fail from this point onwards.
4648                  */
4649                 old_prog = xchg(&priv->channels.params.xdp_prog, prog);
4650         }
4651
4652         if (old_prog)
4653                 bpf_prog_put(old_prog);
4654
4655         if (!was_opened && reset) /* change RQ type according to priv->xdp_prog */
4656                 mlx5e_set_rq_type(priv->mdev, &priv->channels.params);
4657
4658         if (!was_opened || reset)
4659                 goto unlock;
4660
4661         /* exchanging programs w/o reset, we update ref counts on behalf
4662          * of the channels RQs here.
4663          */
4664         for (i = 0; i < priv->channels.num; i++) {
4665                 struct mlx5e_channel *c = priv->channels.c[i];
4666
4667                 mlx5e_rq_replace_xdp_prog(&c->rq, prog);
4668                 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
4669                         mlx5e_rq_replace_xdp_prog(&c->xskrq, prog);
4670         }
4671
4672 unlock:
4673         mutex_unlock(&priv->state_lock);
4674         return err;
4675 }
4676
4677 static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp)
4678 {
4679         switch (xdp->command) {
4680         case XDP_SETUP_PROG:
4681                 return mlx5e_xdp_set(dev, xdp->prog);
4682         case XDP_SETUP_XSK_POOL:
4683                 return mlx5e_xsk_setup_pool(dev, xdp->xsk.pool,
4684                                             xdp->xsk.queue_id);
4685         default:
4686                 return -EINVAL;
4687         }
4688 }
4689
4690 #ifdef CONFIG_MLX5_ESWITCH
4691 static int mlx5e_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4692                                 struct net_device *dev, u32 filter_mask,
4693                                 int nlflags)
4694 {
4695         struct mlx5e_priv *priv = netdev_priv(dev);
4696         struct mlx5_core_dev *mdev = priv->mdev;
4697         u8 mode, setting;
4698         int err;
4699
4700         err = mlx5_eswitch_get_vepa(mdev->priv.eswitch, &setting);
4701         if (err)
4702                 return err;
4703         mode = setting ? BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB;
4704         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4705                                        mode,
4706                                        0, 0, nlflags, filter_mask, NULL);
4707 }
4708
4709 static int mlx5e_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
4710                                 u16 flags, struct netlink_ext_ack *extack)
4711 {
4712         struct mlx5e_priv *priv = netdev_priv(dev);
4713         struct mlx5_core_dev *mdev = priv->mdev;
4714         struct nlattr *attr, *br_spec;
4715         u16 mode = BRIDGE_MODE_UNDEF;
4716         u8 setting;
4717         int rem;
4718
4719         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4720         if (!br_spec)
4721                 return -EINVAL;
4722
4723         nla_for_each_nested(attr, br_spec, rem) {
4724                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4725                         continue;
4726
4727                 if (nla_len(attr) < sizeof(mode))
4728                         return -EINVAL;
4729
4730                 mode = nla_get_u16(attr);
4731                 if (mode > BRIDGE_MODE_VEPA)
4732                         return -EINVAL;
4733
4734                 break;
4735         }
4736
4737         if (mode == BRIDGE_MODE_UNDEF)
4738                 return -EINVAL;
4739
4740         setting = (mode == BRIDGE_MODE_VEPA) ?  1 : 0;
4741         return mlx5_eswitch_set_vepa(mdev->priv.eswitch, setting);
4742 }
4743 #endif
4744
4745 const struct net_device_ops mlx5e_netdev_ops = {
4746         .ndo_open                = mlx5e_open,
4747         .ndo_stop                = mlx5e_close,
4748         .ndo_start_xmit          = mlx5e_xmit,
4749         .ndo_setup_tc            = mlx5e_setup_tc,
4750         .ndo_select_queue        = mlx5e_select_queue,
4751         .ndo_get_stats64         = mlx5e_get_stats,
4752         .ndo_set_rx_mode         = mlx5e_set_rx_mode,
4753         .ndo_set_mac_address     = mlx5e_set_mac,
4754         .ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
4755         .ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
4756         .ndo_set_features        = mlx5e_set_features,
4757         .ndo_fix_features        = mlx5e_fix_features,
4758         .ndo_change_mtu          = mlx5e_change_nic_mtu,
4759         .ndo_do_ioctl            = mlx5e_ioctl,
4760         .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
4761         .ndo_features_check      = mlx5e_features_check,
4762         .ndo_tx_timeout          = mlx5e_tx_timeout,
4763         .ndo_bpf                 = mlx5e_xdp,
4764         .ndo_xdp_xmit            = mlx5e_xdp_xmit,
4765         .ndo_xsk_wakeup          = mlx5e_xsk_wakeup,
4766 #ifdef CONFIG_MLX5_EN_ARFS
4767         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
4768 #endif
4769 #ifdef CONFIG_MLX5_ESWITCH
4770         .ndo_bridge_setlink      = mlx5e_bridge_setlink,
4771         .ndo_bridge_getlink      = mlx5e_bridge_getlink,
4772
4773         /* SRIOV E-Switch NDOs */
4774         .ndo_set_vf_mac          = mlx5e_set_vf_mac,
4775         .ndo_set_vf_vlan         = mlx5e_set_vf_vlan,
4776         .ndo_set_vf_spoofchk     = mlx5e_set_vf_spoofchk,
4777         .ndo_set_vf_trust        = mlx5e_set_vf_trust,
4778         .ndo_set_vf_rate         = mlx5e_set_vf_rate,
4779         .ndo_get_vf_config       = mlx5e_get_vf_config,
4780         .ndo_set_vf_link_state   = mlx5e_set_vf_link_state,
4781         .ndo_get_vf_stats        = mlx5e_get_vf_stats,
4782 #endif
4783         .ndo_get_devlink_port    = mlx5e_get_devlink_port,
4784 };
4785
4786 void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len,
4787                                    int num_channels)
4788 {
4789         int i;
4790
4791         for (i = 0; i < len; i++)
4792                 indirection_rqt[i] = i % num_channels;
4793 }
4794
4795 static bool slow_pci_heuristic(struct mlx5_core_dev *mdev)
4796 {
4797         u32 link_speed = 0;
4798         u32 pci_bw = 0;
4799
4800         mlx5e_port_max_linkspeed(mdev, &link_speed);
4801         pci_bw = pcie_bandwidth_available(mdev->pdev, NULL, NULL, NULL);
4802         mlx5_core_dbg_once(mdev, "Max link speed = %d, PCI BW = %d\n",
4803                            link_speed, pci_bw);
4804
4805 #define MLX5E_SLOW_PCI_RATIO (2)
4806
4807         return link_speed && pci_bw &&
4808                 link_speed > MLX5E_SLOW_PCI_RATIO * pci_bw;
4809 }
4810
4811 static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode)
4812 {
4813         struct dim_cq_moder moder;
4814
4815         moder.cq_period_mode = cq_period_mode;
4816         moder.pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
4817         moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
4818         if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
4819                 moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE;
4820
4821         return moder;
4822 }
4823
4824 static struct dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode)
4825 {
4826         struct dim_cq_moder moder;
4827
4828         moder.cq_period_mode = cq_period_mode;
4829         moder.pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
4830         moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
4831         if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
4832                 moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
4833
4834         return moder;
4835 }
4836
4837 static u8 mlx5_to_net_dim_cq_period_mode(u8 cq_period_mode)
4838 {
4839         return cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE ?
4840                 DIM_CQ_PERIOD_MODE_START_FROM_CQE :
4841                 DIM_CQ_PERIOD_MODE_START_FROM_EQE;
4842 }
4843
4844 void mlx5e_reset_tx_moderation(struct mlx5e_params *params, u8 cq_period_mode)
4845 {
4846         if (params->tx_dim_enabled) {
4847                 u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode);
4848
4849                 params->tx_cq_moderation = net_dim_get_def_tx_moderation(dim_period_mode);
4850         } else {
4851                 params->tx_cq_moderation = mlx5e_get_def_tx_moderation(cq_period_mode);
4852         }
4853 }
4854
4855 void mlx5e_reset_rx_moderation(struct mlx5e_params *params, u8 cq_period_mode)
4856 {
4857         if (params->rx_dim_enabled) {
4858                 u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode);
4859
4860                 params->rx_cq_moderation = net_dim_get_def_rx_moderation(dim_period_mode);
4861         } else {
4862                 params->rx_cq_moderation = mlx5e_get_def_rx_moderation(cq_period_mode);
4863         }
4864 }
4865
4866 void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
4867 {
4868         mlx5e_reset_tx_moderation(params, cq_period_mode);
4869         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_TX_CQE_BASED_MODER,
4870                         params->tx_cq_moderation.cq_period_mode ==
4871                                 MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
4872 }
4873
4874 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
4875 {
4876         mlx5e_reset_rx_moderation(params, cq_period_mode);
4877         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER,
4878                         params->rx_cq_moderation.cq_period_mode ==
4879                                 MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
4880 }
4881
4882 static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
4883 {
4884         int i;
4885
4886         /* The supported periods are organized in ascending order */
4887         for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++)
4888                 if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout)
4889                         break;
4890
4891         return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]);
4892 }
4893
4894 void mlx5e_build_rq_params(struct mlx5_core_dev *mdev,
4895                            struct mlx5e_params *params)
4896 {
4897         /* Prefer Striding RQ, unless any of the following holds:
4898          * - Striding RQ configuration is not possible/supported.
4899          * - Slow PCI heuristic.
4900          * - Legacy RQ would use linear SKB while Striding RQ would use non-linear.
4901          *
4902          * No XSK params: checking the availability of striding RQ in general.
4903          */
4904         if (!slow_pci_heuristic(mdev) &&
4905             mlx5e_striding_rq_possible(mdev, params) &&
4906             (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ||
4907              !mlx5e_rx_is_linear_skb(params, NULL)))
4908                 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true);
4909         mlx5e_set_rq_type(mdev, params);
4910         mlx5e_init_rq_type_params(mdev, params);
4911 }
4912
4913 void mlx5e_build_rss_params(struct mlx5e_rss_params *rss_params,
4914                             u16 num_channels)
4915 {
4916         enum mlx5e_traffic_types tt;
4917
4918         rss_params->hfunc = ETH_RSS_HASH_TOP;
4919         netdev_rss_key_fill(rss_params->toeplitz_hash_key,
4920                             sizeof(rss_params->toeplitz_hash_key));
4921         mlx5e_build_default_indir_rqt(rss_params->indirection_rqt,
4922                                       MLX5E_INDIR_RQT_SIZE, num_channels);
4923         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
4924                 rss_params->rx_hash_fields[tt] =
4925                         tirc_default_config[tt].rx_hash_fields;
4926 }
4927
4928 void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu)
4929 {
4930         struct mlx5e_rss_params *rss_params = &priv->rss_params;
4931         struct mlx5e_params *params = &priv->channels.params;
4932         struct mlx5_core_dev *mdev = priv->mdev;
4933         u8 rx_cq_period_mode;
4934
4935         priv->max_nch = mlx5e_calc_max_nch(priv, priv->profile);
4936
4937         params->sw_mtu = mtu;
4938         params->hard_mtu = MLX5E_ETH_HARD_MTU;
4939         params->num_channels = min_t(unsigned int, MLX5E_MAX_NUM_CHANNELS / 2,
4940                                      priv->max_nch);
4941         params->num_tc       = 1;
4942
4943         /* SQ */
4944         params->log_sq_size = is_kdump_kernel() ?
4945                 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE :
4946                 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
4947         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE,
4948                         MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe));
4949
4950         /* XDP SQ */
4951         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE,
4952                         MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe));
4953
4954         /* set CQE compression */
4955         params->rx_cqe_compress_def = false;
4956         if (MLX5_CAP_GEN(mdev, cqe_compression) &&
4957             MLX5_CAP_GEN(mdev, vport_group_manager))
4958                 params->rx_cqe_compress_def = slow_pci_heuristic(mdev);
4959
4960         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def);
4961         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false);
4962
4963         /* RQ */
4964         mlx5e_build_rq_params(mdev, params);
4965
4966         /* HW LRO */
4967         if (MLX5_CAP_ETH(mdev, lro_cap) &&
4968             params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
4969                 /* No XSK params: checking the availability of striding RQ in general. */
4970                 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL))
4971                         params->lro_en = !slow_pci_heuristic(mdev);
4972         }
4973         params->lro_timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
4974
4975         /* CQ moderation params */
4976         rx_cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
4977                         MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
4978                         MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
4979         params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
4980         params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
4981         mlx5e_set_rx_cq_mode_params(params, rx_cq_period_mode);
4982         mlx5e_set_tx_cq_mode_params(params, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
4983
4984         /* TX inline */
4985         mlx5_query_min_inline(mdev, &params->tx_min_inline_mode);
4986
4987         /* RSS */
4988         mlx5e_build_rss_params(rss_params, params->num_channels);
4989         params->tunneled_offload_en =
4990                 mlx5e_tunnel_inner_ft_supported(mdev);
4991
4992         /* AF_XDP */
4993         params->xsk = xsk;
4994
4995         /* Do not update netdev->features directly in here
4996          * on mlx5e_attach_netdev() we will call mlx5e_update_features()
4997          * To update netdev->features please modify mlx5e_fix_features()
4998          */
4999 }
5000
5001 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
5002 {
5003         struct mlx5e_priv *priv = netdev_priv(netdev);
5004
5005         mlx5_query_mac_address(priv->mdev, netdev->dev_addr);
5006         if (is_zero_ether_addr(netdev->dev_addr) &&
5007             !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
5008                 eth_hw_addr_random(netdev);
5009                 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
5010         }
5011 }
5012
5013 static int mlx5e_vxlan_set_port(struct net_device *netdev, unsigned int table,
5014                                 unsigned int entry, struct udp_tunnel_info *ti)
5015 {
5016         struct mlx5e_priv *priv = netdev_priv(netdev);
5017
5018         return mlx5_vxlan_add_port(priv->mdev->vxlan, ntohs(ti->port));
5019 }
5020
5021 static int mlx5e_vxlan_unset_port(struct net_device *netdev, unsigned int table,
5022                                   unsigned int entry, struct udp_tunnel_info *ti)
5023 {
5024         struct mlx5e_priv *priv = netdev_priv(netdev);
5025
5026         return mlx5_vxlan_del_port(priv->mdev->vxlan, ntohs(ti->port));
5027 }
5028
5029 void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv)
5030 {
5031         if (!mlx5_vxlan_allowed(priv->mdev->vxlan))
5032                 return;
5033
5034         priv->nic_info.set_port = mlx5e_vxlan_set_port;
5035         priv->nic_info.unset_port = mlx5e_vxlan_unset_port;
5036         priv->nic_info.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
5037                                 UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN;
5038         priv->nic_info.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN;
5039         /* Don't count the space hard-coded to the IANA port */
5040         priv->nic_info.tables[0].n_entries =
5041                 mlx5_vxlan_max_udp_ports(priv->mdev) - 1;
5042
5043         priv->netdev->udp_tunnel_nic_info = &priv->nic_info;
5044 }
5045
5046 static bool mlx5e_tunnel_any_tx_proto_supported(struct mlx5_core_dev *mdev)
5047 {
5048         int tt;
5049
5050         for (tt = 0; tt < MLX5E_NUM_TUNNEL_TT; tt++) {
5051                 if (mlx5e_tunnel_proto_supported_tx(mdev, mlx5e_get_proto_by_tunnel_type(tt)))
5052                         return true;
5053         }
5054         return (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev));
5055 }
5056
5057 static void mlx5e_build_nic_netdev(struct net_device *netdev)
5058 {
5059         struct mlx5e_priv *priv = netdev_priv(netdev);
5060         struct mlx5_core_dev *mdev = priv->mdev;
5061         bool fcs_supported;
5062         bool fcs_enabled;
5063
5064         SET_NETDEV_DEV(netdev, mdev->device);
5065
5066         netdev->netdev_ops = &mlx5e_netdev_ops;
5067
5068         mlx5e_dcbnl_build_netdev(netdev);
5069
5070         netdev->watchdog_timeo    = 15 * HZ;
5071
5072         netdev->ethtool_ops       = &mlx5e_ethtool_ops;
5073
5074         netdev->vlan_features    |= NETIF_F_SG;
5075         netdev->vlan_features    |= NETIF_F_HW_CSUM;
5076         netdev->vlan_features    |= NETIF_F_GRO;
5077         netdev->vlan_features    |= NETIF_F_TSO;
5078         netdev->vlan_features    |= NETIF_F_TSO6;
5079         netdev->vlan_features    |= NETIF_F_RXCSUM;
5080         netdev->vlan_features    |= NETIF_F_RXHASH;
5081
5082         netdev->mpls_features    |= NETIF_F_SG;
5083         netdev->mpls_features    |= NETIF_F_HW_CSUM;
5084         netdev->mpls_features    |= NETIF_F_TSO;
5085         netdev->mpls_features    |= NETIF_F_TSO6;
5086
5087         netdev->hw_enc_features  |= NETIF_F_HW_VLAN_CTAG_TX;
5088         netdev->hw_enc_features  |= NETIF_F_HW_VLAN_CTAG_RX;
5089
5090         if (!!MLX5_CAP_ETH(mdev, lro_cap) &&
5091             mlx5e_check_fragmented_striding_rq_cap(mdev))
5092                 netdev->vlan_features    |= NETIF_F_LRO;
5093
5094         netdev->hw_features       = netdev->vlan_features;
5095         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_TX;
5096         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_RX;
5097         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_FILTER;
5098         netdev->hw_features      |= NETIF_F_HW_VLAN_STAG_TX;
5099
5100         if (mlx5e_tunnel_any_tx_proto_supported(mdev)) {
5101                 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
5102                 netdev->hw_enc_features |= NETIF_F_TSO;
5103                 netdev->hw_enc_features |= NETIF_F_TSO6;
5104                 netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL;
5105         }
5106
5107         if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) {
5108                 netdev->hw_features     |= NETIF_F_GSO_UDP_TUNNEL |
5109                                            NETIF_F_GSO_UDP_TUNNEL_CSUM;
5110                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL |
5111                                            NETIF_F_GSO_UDP_TUNNEL_CSUM;
5112                 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
5113                 netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL |
5114                                          NETIF_F_GSO_UDP_TUNNEL_CSUM;
5115         }
5116
5117         if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_GRE)) {
5118                 netdev->hw_features     |= NETIF_F_GSO_GRE |
5119                                            NETIF_F_GSO_GRE_CSUM;
5120                 netdev->hw_enc_features |= NETIF_F_GSO_GRE |
5121                                            NETIF_F_GSO_GRE_CSUM;
5122                 netdev->gso_partial_features |= NETIF_F_GSO_GRE |
5123                                                 NETIF_F_GSO_GRE_CSUM;
5124         }
5125
5126         if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_IPIP)) {
5127                 netdev->hw_features |= NETIF_F_GSO_IPXIP4 |
5128                                        NETIF_F_GSO_IPXIP6;
5129                 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4 |
5130                                            NETIF_F_GSO_IPXIP6;
5131                 netdev->gso_partial_features |= NETIF_F_GSO_IPXIP4 |
5132                                                 NETIF_F_GSO_IPXIP6;
5133         }
5134
5135         netdev->hw_features                      |= NETIF_F_GSO_PARTIAL;
5136         netdev->gso_partial_features             |= NETIF_F_GSO_UDP_L4;
5137         netdev->hw_features                      |= NETIF_F_GSO_UDP_L4;
5138         netdev->features                         |= NETIF_F_GSO_UDP_L4;
5139
5140         mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
5141
5142         if (fcs_supported)
5143                 netdev->hw_features |= NETIF_F_RXALL;
5144
5145         if (MLX5_CAP_ETH(mdev, scatter_fcs))
5146                 netdev->hw_features |= NETIF_F_RXFCS;
5147
5148         netdev->features          = netdev->hw_features;
5149
5150         /* Defaults */
5151         if (fcs_enabled)
5152                 netdev->features  &= ~NETIF_F_RXALL;
5153         netdev->features  &= ~NETIF_F_LRO;
5154         netdev->features  &= ~NETIF_F_RXFCS;
5155
5156 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
5157         if (FT_CAP(flow_modify_en) &&
5158             FT_CAP(modify_root) &&
5159             FT_CAP(identified_miss_table_mode) &&
5160             FT_CAP(flow_table_modify)) {
5161 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
5162                 netdev->hw_features      |= NETIF_F_HW_TC;
5163 #endif
5164 #ifdef CONFIG_MLX5_EN_ARFS
5165                 netdev->hw_features      |= NETIF_F_NTUPLE;
5166 #endif
5167         }
5168         if (mlx5_qos_is_supported(mdev))
5169                 netdev->features |= NETIF_F_HW_TC;
5170
5171         netdev->features         |= NETIF_F_HIGHDMA;
5172         netdev->features         |= NETIF_F_HW_VLAN_STAG_FILTER;
5173
5174         netdev->priv_flags       |= IFF_UNICAST_FLT;
5175
5176         mlx5e_set_netdev_dev_addr(netdev);
5177         mlx5e_ipsec_build_netdev(priv);
5178         mlx5e_tls_build_netdev(priv);
5179 }
5180
5181 void mlx5e_create_q_counters(struct mlx5e_priv *priv)
5182 {
5183         u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
5184         u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
5185         struct mlx5_core_dev *mdev = priv->mdev;
5186         int err;
5187
5188         MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
5189         err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out);
5190         if (!err)
5191                 priv->q_counter =
5192                         MLX5_GET(alloc_q_counter_out, out, counter_set_id);
5193
5194         err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out);
5195         if (!err)
5196                 priv->drop_rq_q_counter =
5197                         MLX5_GET(alloc_q_counter_out, out, counter_set_id);
5198 }
5199
5200 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv)
5201 {
5202         u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
5203
5204         MLX5_SET(dealloc_q_counter_in, in, opcode,
5205                  MLX5_CMD_OP_DEALLOC_Q_COUNTER);
5206         if (priv->q_counter) {
5207                 MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
5208                          priv->q_counter);
5209                 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in);
5210         }
5211
5212         if (priv->drop_rq_q_counter) {
5213                 MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
5214                          priv->drop_rq_q_counter);
5215                 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in);
5216         }
5217 }
5218
5219 static int mlx5e_nic_init(struct mlx5_core_dev *mdev,
5220                           struct net_device *netdev)
5221 {
5222         struct mlx5e_priv *priv = netdev_priv(netdev);
5223         int err;
5224
5225         mlx5e_build_nic_params(priv, &priv->xsk, netdev->mtu);
5226         mlx5e_vxlan_set_netdev_info(priv);
5227
5228         mlx5e_timestamp_init(priv);
5229
5230         err = mlx5e_ipsec_init(priv);
5231         if (err)
5232                 mlx5_core_err(mdev, "IPSec initialization failed, %d\n", err);
5233
5234         err = mlx5e_tls_init(priv);
5235         if (err)
5236                 mlx5_core_err(mdev, "TLS initialization failed, %d\n", err);
5237
5238         err = mlx5e_devlink_port_register(priv);
5239         if (err)
5240                 mlx5_core_err(mdev, "mlx5e_devlink_port_register failed, %d\n", err);
5241
5242         mlx5e_health_create_reporters(priv);
5243
5244         return 0;
5245 }
5246
5247 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
5248 {
5249         mlx5e_health_destroy_reporters(priv);
5250         mlx5e_devlink_port_unregister(priv);
5251         mlx5e_tls_cleanup(priv);
5252         mlx5e_ipsec_cleanup(priv);
5253 }
5254
5255 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
5256 {
5257         struct mlx5_core_dev *mdev = priv->mdev;
5258         int err;
5259
5260         mlx5e_create_q_counters(priv);
5261
5262         err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
5263         if (err) {
5264                 mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
5265                 goto err_destroy_q_counters;
5266         }
5267
5268         err = mlx5e_create_indirect_rqt(priv);
5269         if (err)
5270                 goto err_close_drop_rq;
5271
5272         err = mlx5e_create_direct_rqts(priv, priv->direct_tir);
5273         if (err)
5274                 goto err_destroy_indirect_rqts;
5275
5276         err = mlx5e_create_indirect_tirs(priv, true);
5277         if (err)
5278                 goto err_destroy_direct_rqts;
5279
5280         err = mlx5e_create_direct_tirs(priv, priv->direct_tir);
5281         if (err)
5282                 goto err_destroy_indirect_tirs;
5283
5284         err = mlx5e_create_direct_rqts(priv, priv->xsk_tir);
5285         if (unlikely(err))
5286                 goto err_destroy_direct_tirs;
5287
5288         err = mlx5e_create_direct_tirs(priv, priv->xsk_tir);
5289         if (unlikely(err))
5290                 goto err_destroy_xsk_rqts;
5291
5292         err = mlx5e_create_flow_steering(priv);
5293         if (err) {
5294                 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
5295                 goto err_destroy_xsk_tirs;
5296         }
5297
5298         err = mlx5e_tc_nic_init(priv);
5299         if (err)
5300                 goto err_destroy_flow_steering;
5301
5302         err = mlx5e_accel_init_rx(priv);
5303         if (err)
5304                 goto err_tc_nic_cleanup;
5305
5306 #ifdef CONFIG_MLX5_EN_ARFS
5307         priv->netdev->rx_cpu_rmap =  mlx5_eq_table_get_rmap(priv->mdev);
5308 #endif
5309
5310         return 0;
5311
5312 err_tc_nic_cleanup:
5313         mlx5e_tc_nic_cleanup(priv);
5314 err_destroy_flow_steering:
5315         mlx5e_destroy_flow_steering(priv);
5316 err_destroy_xsk_tirs:
5317         mlx5e_destroy_direct_tirs(priv, priv->xsk_tir);
5318 err_destroy_xsk_rqts:
5319         mlx5e_destroy_direct_rqts(priv, priv->xsk_tir);
5320 err_destroy_direct_tirs:
5321         mlx5e_destroy_direct_tirs(priv, priv->direct_tir);
5322 err_destroy_indirect_tirs:
5323         mlx5e_destroy_indirect_tirs(priv);
5324 err_destroy_direct_rqts:
5325         mlx5e_destroy_direct_rqts(priv, priv->direct_tir);
5326 err_destroy_indirect_rqts:
5327         mlx5e_destroy_rqt(priv, &priv->indir_rqt);
5328 err_close_drop_rq:
5329         mlx5e_close_drop_rq(&priv->drop_rq);
5330 err_destroy_q_counters:
5331         mlx5e_destroy_q_counters(priv);
5332         return err;
5333 }
5334
5335 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
5336 {
5337         mlx5e_accel_cleanup_rx(priv);
5338         mlx5e_tc_nic_cleanup(priv);
5339         mlx5e_destroy_flow_steering(priv);
5340         mlx5e_destroy_direct_tirs(priv, priv->xsk_tir);
5341         mlx5e_destroy_direct_rqts(priv, priv->xsk_tir);
5342         mlx5e_destroy_direct_tirs(priv, priv->direct_tir);
5343         mlx5e_destroy_indirect_tirs(priv);
5344         mlx5e_destroy_direct_rqts(priv, priv->direct_tir);
5345         mlx5e_destroy_rqt(priv, &priv->indir_rqt);
5346         mlx5e_close_drop_rq(&priv->drop_rq);
5347         mlx5e_destroy_q_counters(priv);
5348 }
5349
5350 static int mlx5e_init_nic_tx(struct mlx5e_priv *priv)
5351 {
5352         int err;
5353
5354         err = mlx5e_create_tises(priv);
5355         if (err) {
5356                 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err);
5357                 return err;
5358         }
5359
5360         mlx5e_dcbnl_initialize(priv);
5361         return 0;
5362 }
5363
5364 static void mlx5e_nic_enable(struct mlx5e_priv *priv)
5365 {
5366         struct net_device *netdev = priv->netdev;
5367         struct mlx5_core_dev *mdev = priv->mdev;
5368
5369         mlx5e_init_l2_addr(priv);
5370
5371         /* Marking the link as currently not needed by the Driver */
5372         if (!netif_running(netdev))
5373                 mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN);
5374
5375         mlx5e_set_netdev_mtu_boundaries(priv);
5376         mlx5e_set_dev_port_mtu(priv);
5377
5378         mlx5_lag_add(mdev, netdev);
5379
5380         mlx5e_enable_async_events(priv);
5381         mlx5e_enable_blocking_events(priv);
5382         if (mlx5e_monitor_counter_supported(priv))
5383                 mlx5e_monitor_counter_init(priv);
5384
5385         mlx5e_hv_vhca_stats_create(priv);
5386         if (netdev->reg_state != NETREG_REGISTERED)
5387                 return;
5388         mlx5e_dcbnl_init_app(priv);
5389
5390         queue_work(priv->wq, &priv->set_rx_mode_work);
5391
5392         rtnl_lock();
5393         if (netif_running(netdev))
5394                 mlx5e_open(netdev);
5395         udp_tunnel_nic_reset_ntf(priv->netdev);
5396         netif_device_attach(netdev);
5397         rtnl_unlock();
5398 }
5399
5400 static void mlx5e_nic_disable(struct mlx5e_priv *priv)
5401 {
5402         struct mlx5_core_dev *mdev = priv->mdev;
5403
5404         if (priv->netdev->reg_state == NETREG_REGISTERED)
5405                 mlx5e_dcbnl_delete_app(priv);
5406
5407         rtnl_lock();
5408         if (netif_running(priv->netdev))
5409                 mlx5e_close(priv->netdev);
5410         netif_device_detach(priv->netdev);
5411         rtnl_unlock();
5412
5413         queue_work(priv->wq, &priv->set_rx_mode_work);
5414
5415         mlx5e_hv_vhca_stats_destroy(priv);
5416         if (mlx5e_monitor_counter_supported(priv))
5417                 mlx5e_monitor_counter_cleanup(priv);
5418
5419         mlx5e_disable_blocking_events(priv);
5420         if (priv->en_trap) {
5421                 mlx5e_deactivate_trap(priv);
5422                 mlx5e_close_trap(priv->en_trap);
5423                 priv->en_trap = NULL;
5424         }
5425         mlx5e_disable_async_events(priv);
5426         mlx5_lag_remove(mdev);
5427         mlx5_vxlan_reset_to_default(mdev->vxlan);
5428 }
5429
5430 int mlx5e_update_nic_rx(struct mlx5e_priv *priv)
5431 {
5432         return mlx5e_refresh_tirs(priv, false, false);
5433 }
5434
5435 static const struct mlx5e_profile mlx5e_nic_profile = {
5436         .init              = mlx5e_nic_init,
5437         .cleanup           = mlx5e_nic_cleanup,
5438         .init_rx           = mlx5e_init_nic_rx,
5439         .cleanup_rx        = mlx5e_cleanup_nic_rx,
5440         .init_tx           = mlx5e_init_nic_tx,
5441         .cleanup_tx        = mlx5e_cleanup_nic_tx,
5442         .enable            = mlx5e_nic_enable,
5443         .disable           = mlx5e_nic_disable,
5444         .update_rx         = mlx5e_update_nic_rx,
5445         .update_stats      = mlx5e_stats_update_ndo_stats,
5446         .update_carrier    = mlx5e_update_carrier,
5447         .rx_handlers       = &mlx5e_rx_handlers_nic,
5448         .max_tc            = MLX5E_MAX_NUM_TC,
5449         .rq_groups         = MLX5E_NUM_RQ_GROUPS(XSK),
5450         .stats_grps        = mlx5e_nic_stats_grps,
5451         .stats_grps_num    = mlx5e_nic_stats_grps_num,
5452 };
5453
5454 /* mlx5e generic netdev management API (move to en_common.c) */
5455 int mlx5e_priv_init(struct mlx5e_priv *priv,
5456                     struct net_device *netdev,
5457                     struct mlx5_core_dev *mdev)
5458 {
5459         memset(priv, 0, sizeof(*priv));
5460
5461         /* priv init */
5462         priv->mdev        = mdev;
5463         priv->netdev      = netdev;
5464         priv->msglevel    = MLX5E_MSG_LEVEL;
5465         priv->max_opened_tc = 1;
5466
5467         if (!alloc_cpumask_var(&priv->scratchpad.cpumask, GFP_KERNEL))
5468                 return -ENOMEM;
5469
5470         mutex_init(&priv->state_lock);
5471         hash_init(priv->htb.qos_tc2node);
5472         INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
5473         INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
5474         INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work);
5475         INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
5476
5477         priv->wq = create_singlethread_workqueue("mlx5e");
5478         if (!priv->wq)
5479                 goto err_free_cpumask;
5480
5481         return 0;
5482
5483 err_free_cpumask:
5484         free_cpumask_var(priv->scratchpad.cpumask);
5485
5486         return -ENOMEM;
5487 }
5488
5489 void mlx5e_priv_cleanup(struct mlx5e_priv *priv)
5490 {
5491         int i;
5492
5493         destroy_workqueue(priv->wq);
5494         free_cpumask_var(priv->scratchpad.cpumask);
5495
5496         for (i = 0; i < priv->htb.max_qos_sqs; i++)
5497                 kfree(priv->htb.qos_sq_stats[i]);
5498         kvfree(priv->htb.qos_sq_stats);
5499 }
5500
5501 struct net_device *
5502 mlx5e_create_netdev(struct mlx5_core_dev *mdev, unsigned int txqs, unsigned int rxqs)
5503 {
5504         struct net_device *netdev;
5505         int err;
5506
5507         netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), txqs, rxqs);
5508         if (!netdev) {
5509                 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
5510                 return NULL;
5511         }
5512
5513         err = mlx5e_priv_init(netdev_priv(netdev), netdev, mdev);
5514         if (err) {
5515                 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err);
5516                 goto err_free_netdev;
5517         }
5518
5519         netif_carrier_off(netdev);
5520         dev_net_set(netdev, mlx5_core_net(mdev));
5521
5522         return netdev;
5523
5524 err_free_netdev:
5525         free_netdev(netdev);
5526
5527         return NULL;
5528 }
5529
5530 static void mlx5e_update_features(struct net_device *netdev)
5531 {
5532         if (netdev->reg_state != NETREG_REGISTERED)
5533                 return; /* features will be updated on netdev registration */
5534
5535         rtnl_lock();
5536         netdev_update_features(netdev);
5537         rtnl_unlock();
5538 }
5539
5540 int mlx5e_attach_netdev(struct mlx5e_priv *priv)
5541 {
5542         const bool take_rtnl = priv->netdev->reg_state == NETREG_REGISTERED;
5543         const struct mlx5e_profile *profile = priv->profile;
5544         int max_nch;
5545         int err;
5546
5547         clear_bit(MLX5E_STATE_DESTROYING, &priv->state);
5548
5549         /* max number of channels may have changed */
5550         max_nch = mlx5e_get_max_num_channels(priv->mdev);
5551         if (priv->channels.params.num_channels > max_nch) {
5552                 mlx5_core_warn(priv->mdev, "MLX5E: Reducing number of channels to %d\n", max_nch);
5553                 /* Reducing the number of channels - RXFH has to be reset, and
5554                  * mlx5e_num_channels_changed below will build the RQT.
5555                  */
5556                 priv->netdev->priv_flags &= ~IFF_RXFH_CONFIGURED;
5557                 priv->channels.params.num_channels = max_nch;
5558         }
5559         /* 1. Set the real number of queues in the kernel the first time.
5560          * 2. Set our default XPS cpumask.
5561          * 3. Build the RQT.
5562          *
5563          * rtnl_lock is required by netif_set_real_num_*_queues in case the
5564          * netdev has been registered by this point (if this function was called
5565          * in the reload or resume flow).
5566          */
5567         if (take_rtnl)
5568                 rtnl_lock();
5569         err = mlx5e_num_channels_changed(priv);
5570         if (take_rtnl)
5571                 rtnl_unlock();
5572         if (err)
5573                 goto out;
5574
5575         err = profile->init_tx(priv);
5576         if (err)
5577                 goto out;
5578
5579         err = profile->init_rx(priv);
5580         if (err)
5581                 goto err_cleanup_tx;
5582
5583         if (profile->enable)
5584                 profile->enable(priv);
5585
5586         mlx5e_update_features(priv->netdev);
5587
5588         return 0;
5589
5590 err_cleanup_tx:
5591         profile->cleanup_tx(priv);
5592
5593 out:
5594         set_bit(MLX5E_STATE_DESTROYING, &priv->state);
5595         cancel_work_sync(&priv->update_stats_work);
5596         return err;
5597 }
5598
5599 void mlx5e_detach_netdev(struct mlx5e_priv *priv)
5600 {
5601         const struct mlx5e_profile *profile = priv->profile;
5602
5603         set_bit(MLX5E_STATE_DESTROYING, &priv->state);
5604
5605         if (profile->disable)
5606                 profile->disable(priv);
5607         flush_workqueue(priv->wq);
5608
5609         profile->cleanup_rx(priv);
5610         profile->cleanup_tx(priv);
5611         cancel_work_sync(&priv->update_stats_work);
5612 }
5613
5614 static int
5615 mlx5e_netdev_attach_profile(struct mlx5e_priv *priv,
5616                             const struct mlx5e_profile *new_profile, void *new_ppriv)
5617 {
5618         struct net_device *netdev = priv->netdev;
5619         struct mlx5_core_dev *mdev = priv->mdev;
5620         int err;
5621
5622         err = mlx5e_priv_init(priv, netdev, mdev);
5623         if (err) {
5624                 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err);
5625                 return err;
5626         }
5627         netif_carrier_off(netdev);
5628         priv->profile = new_profile;
5629         priv->ppriv = new_ppriv;
5630         err = new_profile->init(priv->mdev, priv->netdev);
5631         if (err)
5632                 return err;
5633         err = mlx5e_attach_netdev(priv);
5634         if (err)
5635                 new_profile->cleanup(priv);
5636         return err;
5637 }
5638
5639 int mlx5e_netdev_change_profile(struct mlx5e_priv *priv,
5640                                 const struct mlx5e_profile *new_profile, void *new_ppriv)
5641 {
5642         unsigned int new_max_nch = mlx5e_calc_max_nch(priv, new_profile);
5643         const struct mlx5e_profile *orig_profile = priv->profile;
5644         void *orig_ppriv = priv->ppriv;
5645         int err, rollback_err;
5646
5647         /* sanity */
5648         if (new_max_nch != priv->max_nch) {
5649                 netdev_warn(priv->netdev,
5650                             "%s: Replacing profile with different max channels\n",
5651                             __func__);
5652                 return -EINVAL;
5653         }
5654
5655         /* cleanup old profile */
5656         mlx5e_detach_netdev(priv);
5657         priv->profile->cleanup(priv);
5658         mlx5e_priv_cleanup(priv);
5659
5660         err = mlx5e_netdev_attach_profile(priv, new_profile, new_ppriv);
5661         if (err) { /* roll back to original profile */
5662                 netdev_warn(priv->netdev, "%s: new profile init failed, %d\n",
5663                             __func__, err);
5664                 goto rollback;
5665         }
5666
5667         return 0;
5668
5669 rollback:
5670         rollback_err = mlx5e_netdev_attach_profile(priv, orig_profile, orig_ppriv);
5671         if (rollback_err) {
5672                 netdev_err(priv->netdev,
5673                            "%s: failed to rollback to orig profile, %d\n",
5674                            __func__, rollback_err);
5675         }
5676         return err;
5677 }
5678
5679 void mlx5e_destroy_netdev(struct mlx5e_priv *priv)
5680 {
5681         struct net_device *netdev = priv->netdev;
5682
5683         mlx5e_priv_cleanup(priv);
5684         free_netdev(netdev);
5685 }
5686
5687 static int mlx5e_resume(struct auxiliary_device *adev)
5688 {
5689         struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
5690         struct mlx5e_priv *priv = dev_get_drvdata(&adev->dev);
5691         struct net_device *netdev = priv->netdev;
5692         struct mlx5_core_dev *mdev = edev->mdev;
5693         int err;
5694
5695         if (netif_device_present(netdev))
5696                 return 0;
5697
5698         err = mlx5e_create_mdev_resources(mdev);
5699         if (err)
5700                 return err;
5701
5702         err = mlx5e_attach_netdev(priv);
5703         if (err) {
5704                 mlx5e_destroy_mdev_resources(mdev);
5705                 return err;
5706         }
5707
5708         return 0;
5709 }
5710
5711 static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state)
5712 {
5713         struct mlx5e_priv *priv = dev_get_drvdata(&adev->dev);
5714         struct net_device *netdev = priv->netdev;
5715         struct mlx5_core_dev *mdev = priv->mdev;
5716
5717         if (!netif_device_present(netdev))
5718                 return -ENODEV;
5719
5720         mlx5e_detach_netdev(priv);
5721         mlx5e_destroy_mdev_resources(mdev);
5722         return 0;
5723 }
5724
5725 static int mlx5e_probe(struct auxiliary_device *adev,
5726                        const struct auxiliary_device_id *id)
5727 {
5728         struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
5729         const struct mlx5e_profile *profile = &mlx5e_nic_profile;
5730         struct mlx5_core_dev *mdev = edev->mdev;
5731         struct net_device *netdev;
5732         pm_message_t state = {};
5733         unsigned int txqs, rxqs, ptp_txqs = 0;
5734         struct mlx5e_priv *priv;
5735         int qos_sqs = 0;
5736         int err;
5737         int nch;
5738
5739         if (MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn))
5740                 ptp_txqs = profile->max_tc;
5741
5742         if (mlx5_qos_is_supported(mdev))
5743                 qos_sqs = mlx5e_qos_max_leaf_nodes(mdev);
5744
5745         nch = mlx5e_get_max_num_channels(mdev);
5746         txqs = nch * profile->max_tc + ptp_txqs + qos_sqs;
5747         rxqs = nch * profile->rq_groups;
5748         netdev = mlx5e_create_netdev(mdev, txqs, rxqs);
5749         if (!netdev) {
5750                 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n");
5751                 return -ENOMEM;
5752         }
5753
5754         mlx5e_build_nic_netdev(netdev);
5755
5756         priv = netdev_priv(netdev);
5757         dev_set_drvdata(&adev->dev, priv);
5758
5759         priv->profile = profile;
5760         priv->ppriv = NULL;
5761         err = profile->init(mdev, netdev);
5762         if (err) {
5763                 mlx5_core_err(mdev, "mlx5e_nic_profile init failed, %d\n", err);
5764                 goto err_destroy_netdev;
5765         }
5766
5767         err = mlx5e_resume(adev);
5768         if (err) {
5769                 mlx5_core_err(mdev, "mlx5e_resume failed, %d\n", err);
5770                 goto err_profile_cleanup;
5771         }
5772
5773         err = register_netdev(netdev);
5774         if (err) {
5775                 mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
5776                 goto err_resume;
5777         }
5778
5779         mlx5e_devlink_port_type_eth_set(priv);
5780
5781         mlx5e_dcbnl_init_app(priv);
5782         return 0;
5783
5784 err_resume:
5785         mlx5e_suspend(adev, state);
5786 err_profile_cleanup:
5787         profile->cleanup(priv);
5788 err_destroy_netdev:
5789         mlx5e_destroy_netdev(priv);
5790         return err;
5791 }
5792
5793 static void mlx5e_remove(struct auxiliary_device *adev)
5794 {
5795         struct mlx5e_priv *priv = dev_get_drvdata(&adev->dev);
5796         pm_message_t state = {};
5797
5798         mlx5e_dcbnl_delete_app(priv);
5799         unregister_netdev(priv->netdev);
5800         mlx5e_suspend(adev, state);
5801         priv->profile->cleanup(priv);
5802         mlx5e_destroy_netdev(priv);
5803 }
5804
5805 static const struct auxiliary_device_id mlx5e_id_table[] = {
5806         { .name = MLX5_ADEV_NAME ".eth", },
5807         {},
5808 };
5809
5810 MODULE_DEVICE_TABLE(auxiliary, mlx5e_id_table);
5811
5812 static struct auxiliary_driver mlx5e_driver = {
5813         .name = "eth",
5814         .probe = mlx5e_probe,
5815         .remove = mlx5e_remove,
5816         .suspend = mlx5e_suspend,
5817         .resume = mlx5e_resume,
5818         .id_table = mlx5e_id_table,
5819 };
5820
5821 int mlx5e_init(void)
5822 {
5823         int ret;
5824
5825         mlx5e_ipsec_build_inverse_table();
5826         mlx5e_build_ptys2ethtool_map();
5827         ret = mlx5e_rep_init();
5828         if (ret)
5829                 return ret;
5830
5831         ret = auxiliary_driver_register(&mlx5e_driver);
5832         if (ret)
5833                 mlx5e_rep_cleanup();
5834         return ret;
5835 }
5836
5837 void mlx5e_cleanup(void)
5838 {
5839         auxiliary_driver_unregister(&mlx5e_driver);
5840         mlx5e_rep_cleanup();
5841 }