drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

   1 /*
   2  * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/ip.h>
  34 #include <linux/ipv6.h>
  35 #include <linux/tcp.h>
  36 #include <linux/bitmap.h>
  37 #include <linux/filter.h>
  38 #include <net/ip6_checksum.h>
  39 #include <net/page_pool.h>
  40 #include <net/inet_ecn.h>
  41 #include <net/gro.h>
  42 #include <net/udp.h>
  43 #include <net/tcp.h>
  44 #include "en.h"
  45 #include "en/txrx.h"
  46 #include "en_tc.h"
  47 #include "eswitch.h"
  48 #include "en_rep.h"
  49 #include "en/rep/tc.h"
  50 #include "ipoib/ipoib.h"
  51 #include "accel/ipsec.h"
  52 #include "fpga/ipsec.h"
  53 #include "en_accel/ipsec_rxtx.h"
  54 #include "en_accel/tls_rxtx.h"
  55 #include "en/xdp.h"
  56 #include "en/xsk/rx.h"
  57 #include "en/health.h"
  58 #include "en/params.h"
  59 #include "devlink.h"
  60 #include "en/devlink.h"
  61
  62 static struct sk_buff *
  63 mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
  64                                 u16 cqe_bcnt, u32 head_offset, u32 page_idx);
  65 static struct sk_buff *
  66 mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
  67                                    u16 cqe_bcnt, u32 head_offset, u32 page_idx);
  68 static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
  69 static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
  70 static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
  71
  72 const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic = {
  73         .handle_rx_cqe       = mlx5e_handle_rx_cqe,
  74         .handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq,
  75         .handle_rx_cqe_mpwqe_shampo = mlx5e_handle_rx_cqe_mpwrq_shampo,
  76 };
  77
  78 static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
  79 {
  80         return config->rx_filter == HWTSTAMP_FILTER_ALL;
  81 }
  82
  83 static inline void mlx5e_read_cqe_slot(struct mlx5_cqwq *wq,
  84                                        u32 cqcc, void *data)
  85 {
  86         u32 ci = mlx5_cqwq_ctr2ix(wq, cqcc);
  87
  88         memcpy(data, mlx5_cqwq_get_wqe(wq, ci), sizeof(struct mlx5_cqe64));
  89 }
  90
  91 static inline void mlx5e_read_title_slot(struct mlx5e_rq *rq,
  92                                          struct mlx5_cqwq *wq,
  93                                          u32 cqcc)
  94 {
  95         struct mlx5e_cq_decomp *cqd = &rq->cqd;
  96         struct mlx5_cqe64 *title = &cqd->title;
  97
  98         mlx5e_read_cqe_slot(wq, cqcc, title);
  99         cqd->left        = be32_to_cpu(title->byte_cnt);
 100         cqd->wqe_counter = be16_to_cpu(title->wqe_counter);
 101         rq->stats->cqe_compress_blks++;
 102 }
 103
 104 static inline void mlx5e_read_mini_arr_slot(struct mlx5_cqwq *wq,
 105                                             struct mlx5e_cq_decomp *cqd,
 106                                             u32 cqcc)
 107 {
 108         mlx5e_read_cqe_slot(wq, cqcc, cqd->mini_arr);
 109         cqd->mini_arr_idx = 0;
 110 }
 111
 112 static inline void mlx5e_cqes_update_owner(struct mlx5_cqwq *wq, int n)
 113 {
 114         u32 cqcc   = wq->cc;
 115         u8  op_own = mlx5_cqwq_get_ctr_wrap_cnt(wq, cqcc) & 1;
 116         u32 ci     = mlx5_cqwq_ctr2ix(wq, cqcc);
 117         u32 wq_sz  = mlx5_cqwq_get_size(wq);
 118         u32 ci_top = min_t(u32, wq_sz, ci + n);
 119
 120         for (; ci < ci_top; ci++, n--) {
 121                 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci);
 122
 123                 cqe->op_own = op_own;
 124         }
 125
 126         if (unlikely(ci == wq_sz)) {
 127                 op_own = !op_own;
 128                 for (ci = 0; ci < n; ci++) {
 129                         struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci);
 130
 131                         cqe->op_own = op_own;
 132                 }
 133         }
 134 }
 135
 136 static inline void mlx5e_decompress_cqe(struct mlx5e_rq *rq,
 137                                         struct mlx5_cqwq *wq,
 138                                         u32 cqcc)
 139 {
 140         struct mlx5e_cq_decomp *cqd = &rq->cqd;
 141         struct mlx5_mini_cqe8 *mini_cqe = &cqd->mini_arr[cqd->mini_arr_idx];
 142         struct mlx5_cqe64 *title = &cqd->title;
 143
 144         title->byte_cnt     = mini_cqe->byte_cnt;
 145         title->check_sum    = mini_cqe->checksum;
 146         title->op_own      &= 0xf0;
 147         title->op_own      |= 0x01 & (cqcc >> wq->fbc.log_sz);
 148
 149         /* state bit set implies linked-list striding RQ wq type and
 150          * HW stride index capability supported
 151          */
 152         if (test_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &rq->state)) {
 153                 title->wqe_counter = mini_cqe->stridx;
 154                 return;
 155         }
 156
 157         /* HW stride index capability not supported */
 158         title->wqe_counter = cpu_to_be16(cqd->wqe_counter);
 159         if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
 160                 cqd->wqe_counter += mpwrq_get_cqe_consumed_strides(title);
 161         else
 162                 cqd->wqe_counter =
 163                         mlx5_wq_cyc_ctr2ix(&rq->wqe.wq, cqd->wqe_counter + 1);
 164 }
 165
 166 static inline void mlx5e_decompress_cqe_no_hash(struct mlx5e_rq *rq,
 167                                                 struct mlx5_cqwq *wq,
 168                                                 u32 cqcc)
 169 {
 170         struct mlx5e_cq_decomp *cqd = &rq->cqd;
 171
 172         mlx5e_decompress_cqe(rq, wq, cqcc);
 173         cqd->title.rss_hash_type   = 0;
 174         cqd->title.rss_hash_result = 0;
 175 }
 176
 177 static inline u32 mlx5e_decompress_cqes_cont(struct mlx5e_rq *rq,
 178                                              struct mlx5_cqwq *wq,
 179                                              int update_owner_only,
 180                                              int budget_rem)
 181 {
 182         struct mlx5e_cq_decomp *cqd = &rq->cqd;
 183         u32 cqcc = wq->cc + update_owner_only;
 184         u32 cqe_count;
 185         u32 i;
 186
 187         cqe_count = min_t(u32, cqd->left, budget_rem);
 188
 189         for (i = update_owner_only; i < cqe_count;
 190              i++, cqd->mini_arr_idx++, cqcc++) {
 191                 if (cqd->mini_arr_idx == MLX5_MINI_CQE_ARRAY_SIZE)
 192                         mlx5e_read_mini_arr_slot(wq, cqd, cqcc);
 193
 194                 mlx5e_decompress_cqe_no_hash(rq, wq, cqcc);
 195                 INDIRECT_CALL_3(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
 196                                 mlx5e_handle_rx_cqe_mpwrq_shampo, mlx5e_handle_rx_cqe,
 197                                 rq, &cqd->title);
 198         }
 199         mlx5e_cqes_update_owner(wq, cqcc - wq->cc);
 200         wq->cc = cqcc;
 201         cqd->left -= cqe_count;
 202         rq->stats->cqe_compress_pkts += cqe_count;
 203
 204         return cqe_count;
 205 }
 206
 207 static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
 208                                               struct mlx5_cqwq *wq,
 209                                               int budget_rem)
 210 {
 211         struct mlx5e_cq_decomp *cqd = &rq->cqd;
 212         u32 cc = wq->cc;
 213
 214         mlx5e_read_title_slot(rq, wq, cc);
 215         mlx5e_read_mini_arr_slot(wq, cqd, cc + 1);
 216         mlx5e_decompress_cqe(rq, wq, cc);
 217         INDIRECT_CALL_3(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
 218                         mlx5e_handle_rx_cqe_mpwrq_shampo, mlx5e_handle_rx_cqe,
 219                         rq, &cqd->title);
 220         cqd->mini_arr_idx++;
 221
 222         return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem) - 1;
 223 }
 224
 225 static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
 226                                       struct mlx5e_dma_info *dma_info)
 227 {
 228         struct mlx5e_page_cache *cache = &rq->page_cache;
 229         u32 tail_next = (cache->tail + 1) & (MLX5E_CACHE_SIZE - 1);
 230         struct mlx5e_rq_stats *stats = rq->stats;
 231
 232         if (tail_next == cache->head) {
 233                 stats->cache_full++;
 234                 return false;
 235         }
 236
 237         if (!dev_page_is_reusable(dma_info->page)) {
 238                 stats->cache_waive++;
 239                 return false;
 240         }
 241
 242         cache->page_cache[cache->tail] = *dma_info;
 243         cache->tail = tail_next;
 244         return true;
 245 }
 246
 247 static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq,
 248                                       struct mlx5e_dma_info *dma_info)
 249 {
 250         struct mlx5e_page_cache *cache = &rq->page_cache;
 251         struct mlx5e_rq_stats *stats = rq->stats;
 252
 253         if (unlikely(cache->head == cache->tail)) {
 254                 stats->cache_empty++;
 255                 return false;
 256         }
 257
 258         if (page_ref_count(cache->page_cache[cache->head].page) != 1) {
 259                 stats->cache_busy++;
 260                 return false;
 261         }
 262
 263         *dma_info = cache->page_cache[cache->head];
 264         cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1);
 265         stats->cache_reuse++;
 266
 267         dma_sync_single_for_device(rq->pdev, dma_info->addr,
 268                                    PAGE_SIZE,
 269                                    DMA_FROM_DEVICE);
 270         return true;
 271 }
 272
 273 static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq,
 274                                         struct mlx5e_dma_info *dma_info)
 275 {
 276         if (mlx5e_rx_cache_get(rq, dma_info))
 277                 return 0;
 278
 279         dma_info->page = page_pool_dev_alloc_pages(rq->page_pool);
 280         if (unlikely(!dma_info->page))
 281                 return -ENOMEM;
 282
 283         dma_info->addr = dma_map_page_attrs(rq->pdev, dma_info->page, 0, PAGE_SIZE,
 284                                             rq->buff.map_dir, DMA_ATTR_SKIP_CPU_SYNC);
 285         if (unlikely(dma_mapping_error(rq->pdev, dma_info->addr))) {
 286                 page_pool_recycle_direct(rq->page_pool, dma_info->page);
 287                 dma_info->page = NULL;
 288                 return -ENOMEM;
 289         }
 290
 291         return 0;
 292 }
 293
 294 static inline int mlx5e_page_alloc(struct mlx5e_rq *rq,
 295                                    struct mlx5e_dma_info *dma_info)
 296 {
 297         if (rq->xsk_pool)
 298                 return mlx5e_xsk_page_alloc_pool(rq, dma_info);
 299         else
 300                 return mlx5e_page_alloc_pool(rq, dma_info);
 301 }
 302
 303 void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info)
 304 {
 305         dma_unmap_page_attrs(rq->pdev, dma_info->addr, PAGE_SIZE, rq->buff.map_dir,
 306                              DMA_ATTR_SKIP_CPU_SYNC);
 307 }
 308
 309 void mlx5e_page_release_dynamic(struct mlx5e_rq *rq,
 310                                 struct mlx5e_dma_info *dma_info,
 311                                 bool recycle)
 312 {
 313         if (likely(recycle)) {
 314                 if (mlx5e_rx_cache_put(rq, dma_info))
 315                         return;
 316
 317                 mlx5e_page_dma_unmap(rq, dma_info);
 318                 page_pool_recycle_direct(rq->page_pool, dma_info->page);
 319         } else {
 320                 mlx5e_page_dma_unmap(rq, dma_info);
 321                 page_pool_release_page(rq->page_pool, dma_info->page);
 322                 put_page(dma_info->page);
 323         }
 324 }
 325
 326 static inline void mlx5e_page_release(struct mlx5e_rq *rq,
 327                                       struct mlx5e_dma_info *dma_info,
 328                                       bool recycle)
 329 {
 330         if (rq->xsk_pool)
 331                 /* The `recycle` parameter is ignored, and the page is always
 332                  * put into the Reuse Ring, because there is no way to return
 333                  * the page to the userspace when the interface goes down.
 334                  */
 335                 xsk_buff_free(dma_info->xsk);
 336         else
 337                 mlx5e_page_release_dynamic(rq, dma_info, recycle);
 338 }
 339
 340 static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq,
 341                                     struct mlx5e_wqe_frag_info *frag)
 342 {
 343         int err = 0;
 344
 345         if (!frag->offset)
 346                 /* On first frag (offset == 0), replenish page (dma_info actually).
 347                  * Other frags that point to the same dma_info (with a different
 348                  * offset) should just use the new one without replenishing again
 349                  * by themselves.
 350                  */
 351                 err = mlx5e_page_alloc(rq, frag->di);
 352
 353         return err;
 354 }
 355
 356 static inline void mlx5e_put_rx_frag(struct mlx5e_rq *rq,
 357                                      struct mlx5e_wqe_frag_info *frag,
 358                                      bool recycle)
 359 {
 360         if (frag->last_in_page)
 361                 mlx5e_page_release(rq, frag->di, recycle);
 362 }
 363
 364 static inline struct mlx5e_wqe_frag_info *get_frag(struct mlx5e_rq *rq, u16 ix)
 365 {
 366         return &rq->wqe.frags[ix << rq->wqe.info.log_num_frags];
 367 }
 368
 369 static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe,
 370                               u16 ix)
 371 {
 372         struct mlx5e_wqe_frag_info *frag = get_frag(rq, ix);
 373         int err;
 374         int i;
 375
 376         for (i = 0; i < rq->wqe.info.num_frags; i++, frag++) {
 377                 u16 headroom;
 378
 379                 err = mlx5e_get_rx_frag(rq, frag);
 380                 if (unlikely(err))
 381                         goto free_frags;
 382
 383                 headroom = i == 0 ? rq->buff.headroom : 0;
 384                 wqe->data[i].addr = cpu_to_be64(frag->di->addr +
 385                                                 frag->offset + headroom);
 386         }
 387
 388         return 0;
 389
 390 free_frags:
 391         while (--i >= 0)
 392                 mlx5e_put_rx_frag(rq, --frag, true);
 393
 394         return err;
 395 }
 396
 397 static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq,
 398                                      struct mlx5e_wqe_frag_info *wi,
 399                                      bool recycle)
 400 {
 401         int i;
 402
 403         for (i = 0; i < rq->wqe.info.num_frags; i++, wi++)
 404                 mlx5e_put_rx_frag(rq, wi, recycle);
 405 }
 406
 407 static void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
 408 {
 409         struct mlx5e_wqe_frag_info *wi = get_frag(rq, ix);
 410
 411         mlx5e_free_rx_wqe(rq, wi, false);
 412 }
 413
 414 static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, u8 wqe_bulk)
 415 {
 416         struct mlx5_wq_cyc *wq = &rq->wqe.wq;
 417         int err;
 418         int i;
 419
 420         if (rq->xsk_pool) {
 421                 int pages_desired = wqe_bulk << rq->wqe.info.log_num_frags;
 422
 423                 /* Check in advance that we have enough frames, instead of
 424                  * allocating one-by-one, failing and moving frames to the
 425                  * Reuse Ring.
 426                  */
 427                 if (unlikely(!xsk_buff_can_alloc(rq->xsk_pool, pages_desired)))
 428                         return -ENOMEM;
 429         }
 430
 431         for (i = 0; i < wqe_bulk; i++) {
 432                 struct mlx5e_rx_wqe_cyc *wqe = mlx5_wq_cyc_get_wqe(wq, ix + i);
 433
 434                 err = mlx5e_alloc_rx_wqe(rq, wqe, ix + i);
 435                 if (unlikely(err))
 436                         goto free_wqes;
 437         }
 438
 439         return 0;
 440
 441 free_wqes:
 442         while (--i >= 0)
 443                 mlx5e_dealloc_rx_wqe(rq, ix + i);
 444
 445         return err;
 446 }
 447
 448 static inline void
 449 mlx5e_add_skb_frag(struct mlx5e_rq *rq, struct sk_buff *skb,
 450                    struct mlx5e_dma_info *di, u32 frag_offset, u32 len,
 451                    unsigned int truesize)
 452 {
 453         dma_sync_single_for_cpu(rq->pdev,
 454                                 di->addr + frag_offset,
 455                                 len, DMA_FROM_DEVICE);
 456         page_ref_inc(di->page);
 457         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
 458                         di->page, frag_offset, len, truesize);
 459 }
 460
 461 static inline void
 462 mlx5e_copy_skb_header(struct device *pdev, struct sk_buff *skb,
 463                       struct mlx5e_dma_info *dma_info,
 464                       int offset_from, int dma_offset, u32 headlen)
 465 {
 466         const void *from = page_address(dma_info->page) + offset_from;
 467         /* Aligning len to sizeof(long) optimizes memcpy performance */
 468         unsigned int len = ALIGN(headlen, sizeof(long));
 469
 470         dma_sync_single_for_cpu(pdev, dma_info->addr + dma_offset, len,
 471                                 DMA_FROM_DEVICE);
 472         skb_copy_to_linear_data(skb, from, len);
 473 }
 474
 475 static void
 476 mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle)
 477 {
 478         bool no_xdp_xmit;
 479         struct mlx5e_dma_info *dma_info = wi->umr.dma_info;
 480         int i;
 481
 482         /* A common case for AF_XDP. */
 483         if (bitmap_full(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE))
 484                 return;
 485
 486         no_xdp_xmit = bitmap_empty(wi->xdp_xmit_bitmap,
 487                                    MLX5_MPWRQ_PAGES_PER_WQE);
 488
 489         for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++)
 490                 if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))
 491                         mlx5e_page_release(rq, &dma_info[i], recycle);
 492 }
 493
 494 static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq, u8 n)
 495 {
 496         struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
 497
 498         do {
 499                 u16 next_wqe_index = mlx5_wq_ll_get_wqe_next_ix(wq, wq->head);
 500
 501                 mlx5_wq_ll_push(wq, next_wqe_index);
 502         } while (--n);
 503
 504         /* ensure wqes are visible to device before updating doorbell record */
 505         dma_wmb();
 506
 507         mlx5_wq_ll_update_db_record(wq);
 508 }
 509
 510 /* This function returns the size of the continuous free space inside a bitmap
 511  * that starts from first and no longer than len including circular ones.
 512  */
 513 static int bitmap_find_window(unsigned long *bitmap, int len,
 514                               int bitmap_size, int first)
 515 {
 516         int next_one, count;
 517
 518         next_one = find_next_bit(bitmap, bitmap_size, first);
 519         if (next_one == bitmap_size) {
 520                 if (bitmap_size - first >= len)
 521                         return len;
 522                 next_one = find_next_bit(bitmap, bitmap_size, 0);
 523                 count = next_one + bitmap_size - first;
 524         } else {
 525                 count = next_one - first;
 526         }
 527
 528         return min(len, count);
 529 }
 530
 531 static void build_klm_umr(struct mlx5e_icosq *sq, struct mlx5e_umr_wqe *umr_wqe,
 532                           __be32 key, u16 offset, u16 klm_len, u16 wqe_bbs)
 533 {
 534         memset(umr_wqe, 0, offsetof(struct mlx5e_umr_wqe, inline_klms));
 535         umr_wqe->ctrl.opmod_idx_opcode =
 536                 cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
 537                              MLX5_OPCODE_UMR);
 538         umr_wqe->ctrl.umr_mkey = key;
 539         umr_wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT)
 540                                             | MLX5E_KLM_UMR_DS_CNT(klm_len));
 541         umr_wqe->uctrl.flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE;
 542         umr_wqe->uctrl.xlt_offset = cpu_to_be16(offset);
 543         umr_wqe->uctrl.xlt_octowords = cpu_to_be16(klm_len);
 544         umr_wqe->uctrl.mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);
 545 }
 546
 547 static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq,
 548                                      struct mlx5e_icosq *sq,
 549                                      u16 klm_entries, u16 index)
 550 {
 551         struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
 552         u16 entries, pi, header_offset, err, wqe_bbs, new_entries;
 553         u32 lkey = rq->mdev->mlx5e_res.hw_objs.mkey;
 554         struct page *page = shampo->last_page;
 555         u64 addr = shampo->last_addr;
 556         struct mlx5e_dma_info *dma_info;
 557         struct mlx5e_umr_wqe *umr_wqe;
 558         int headroom, i;
 559
 560         headroom = rq->buff.headroom;
 561         new_entries = klm_entries - (shampo->pi & (MLX5_UMR_KLM_ALIGNMENT - 1));
 562         entries = ALIGN(klm_entries, MLX5_UMR_KLM_ALIGNMENT);
 563         wqe_bbs = MLX5E_KLM_UMR_WQEBBS(entries);
 564         pi = mlx5e_icosq_get_next_pi(sq, wqe_bbs);
 565         umr_wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
 566         build_klm_umr(sq, umr_wqe, shampo->key, index, entries, wqe_bbs);
 567
 568         for (i = 0; i < entries; i++, index++) {
 569                 dma_info = &shampo->info[index];
 570                 if (i >= klm_entries || (index < shampo->pi && shampo->pi - index <
 571                                          MLX5_UMR_KLM_ALIGNMENT))
 572                         goto update_klm;
 573                 header_offset = (index & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) <<
 574                         MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE;
 575                 if (!(header_offset & (PAGE_SIZE - 1))) {
 576                         err = mlx5e_page_alloc(rq, dma_info);
 577                         if (unlikely(err))
 578                                 goto err_unmap;
 579                         addr = dma_info->addr;
 580                         page = dma_info->page;
 581                 } else {
 582                         dma_info->addr = addr + header_offset;
 583                         dma_info->page = page;
 584                 }
 585
 586 update_klm:
 587                 umr_wqe->inline_klms[i].bcount =
 588                         cpu_to_be32(MLX5E_RX_MAX_HEAD);
 589                 umr_wqe->inline_klms[i].key    = cpu_to_be32(lkey);
 590                 umr_wqe->inline_klms[i].va     =
 591                         cpu_to_be64(dma_info->addr + headroom);
 592         }
 593
 594         sq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) {
 595                 .wqe_type       = MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR,
 596                 .num_wqebbs     = wqe_bbs,
 597                 .shampo.len     = new_entries,
 598         };
 599
 600         shampo->pi = (shampo->pi + new_entries) & (shampo->hd_per_wq - 1);
 601         shampo->last_page = page;
 602         shampo->last_addr = addr;
 603         sq->pc += wqe_bbs;
 604         sq->doorbell_cseg = &umr_wqe->ctrl;
 605
 606         return 0;
 607
 608 err_unmap:
 609         while (--i >= 0) {
 610                 dma_info = &shampo->info[--index];
 611                 if (!(i & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1))) {
 612                         dma_info->addr = ALIGN_DOWN(dma_info->addr, PAGE_SIZE);
 613                         mlx5e_page_release(rq, dma_info, true);
 614                 }
 615         }
 616         rq->stats->buff_alloc_err++;
 617         return err;
 618 }
 619
 620 static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq)
 621 {
 622         struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
 623         u16 klm_entries, num_wqe, index, entries_before;
 624         struct mlx5e_icosq *sq = rq->icosq;
 625         int i, err, max_klm_entries, len;
 626
 627         max_klm_entries = MLX5E_MAX_KLM_PER_WQE(rq->mdev);
 628         klm_entries = bitmap_find_window(shampo->bitmap,
 629                                          shampo->hd_per_wqe,
 630                                          shampo->hd_per_wq, shampo->pi);
 631         if (!klm_entries)
 632                 return 0;
 633
 634         klm_entries += (shampo->pi & (MLX5_UMR_KLM_ALIGNMENT - 1));
 635         index = ALIGN_DOWN(shampo->pi, MLX5_UMR_KLM_ALIGNMENT);
 636         entries_before = shampo->hd_per_wq - index;
 637
 638         if (unlikely(entries_before < klm_entries))
 639                 num_wqe = DIV_ROUND_UP(entries_before, max_klm_entries) +
 640                           DIV_ROUND_UP(klm_entries - entries_before, max_klm_entries);
 641         else
 642                 num_wqe = DIV_ROUND_UP(klm_entries, max_klm_entries);
 643
 644         for (i = 0; i < num_wqe; i++) {
 645                 len = (klm_entries > max_klm_entries) ? max_klm_entries :
 646                                                         klm_entries;
 647                 if (unlikely(index + len > shampo->hd_per_wq))
 648                         len = shampo->hd_per_wq - index;
 649                 err = mlx5e_build_shampo_hd_umr(rq, sq, len, index);
 650                 if (unlikely(err))
 651                         return err;
 652                 index = (index + len) & (rq->mpwqe.shampo->hd_per_wq - 1);
 653                 klm_entries -= len;
 654         }
 655
 656         return 0;
 657 }
 658
 659 static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 660 {
 661         struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
 662         struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0];
 663         struct mlx5e_icosq *sq = rq->icosq;
 664         struct mlx5_wq_cyc *wq = &sq->wq;
 665         struct mlx5e_umr_wqe *umr_wqe;
 666         u16 pi;
 667         int err;
 668         int i;
 669
 670         /* Check in advance that we have enough frames, instead of allocating
 671          * one-by-one, failing and moving frames to the Reuse Ring.
 672          */
 673         if (rq->xsk_pool &&
 674             unlikely(!xsk_buff_can_alloc(rq->xsk_pool, MLX5_MPWRQ_PAGES_PER_WQE))) {
 675                 err = -ENOMEM;
 676                 goto err;
 677         }
 678
 679         if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) {
 680                 err = mlx5e_alloc_rx_hd_mpwqe(rq);
 681                 if (unlikely(err))
 682                         goto err;
 683         }
 684
 685         pi = mlx5e_icosq_get_next_pi(sq, MLX5E_UMR_WQEBBS);
 686         umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi);
 687         memcpy(umr_wqe, &rq->mpwqe.umr_wqe, offsetof(struct mlx5e_umr_wqe, inline_mtts));
 688
 689         for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) {
 690                 err = mlx5e_page_alloc(rq, dma_info);
 691                 if (unlikely(err))
 692                         goto err_unmap;
 693                 umr_wqe->inline_mtts[i].ptag = cpu_to_be64(dma_info->addr | MLX5_EN_WR);
 694         }
 695
 696         bitmap_zero(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
 697         wi->consumed_strides = 0;
 698
 699         umr_wqe->ctrl.opmod_idx_opcode =
 700                 cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
 701                             MLX5_OPCODE_UMR);
 702         umr_wqe->uctrl.xlt_offset =
 703                 cpu_to_be16(MLX5_ALIGNED_MTTS_OCTW(MLX5E_REQUIRED_MTTS(ix)));
 704
 705         sq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) {
 706                 .wqe_type   = MLX5E_ICOSQ_WQE_UMR_RX,
 707                 .num_wqebbs = MLX5E_UMR_WQEBBS,
 708                 .umr.rq     = rq,
 709         };
 710
 711         sq->pc += MLX5E_UMR_WQEBBS;
 712
 713         sq->doorbell_cseg = &umr_wqe->ctrl;
 714
 715         return 0;
 716
 717 err_unmap:
 718         while (--i >= 0) {
 719                 dma_info--;
 720                 mlx5e_page_release(rq, dma_info, true);
 721         }
 722
 723 err:
 724         rq->stats->buff_alloc_err++;
 725
 726         return err;
 727 }
 728
 729 /* This function is responsible to dealloc SHAMPO header buffer.
 730  * close == true specifies that we are in the middle of closing RQ operation so
 731  * we go over all the entries and if they are not in use we free them,
 732  * otherwise we only go over a specific range inside the header buffer that are
 733  * not in use.
 734  */
 735 void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close)
 736 {
 737         struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
 738         int hd_per_wq = shampo->hd_per_wq;
 739         struct page *deleted_page = NULL;
 740         struct mlx5e_dma_info *hd_info;
 741         int i, index = start;
 742
 743         for (i = 0; i < len; i++, index++) {
 744                 if (index == hd_per_wq)
 745                         index = 0;
 746
 747                 if (close && !test_bit(index, shampo->bitmap))
 748                         continue;
 749
 750                 hd_info = &shampo->info[index];
 751                 hd_info->addr = ALIGN_DOWN(hd_info->addr, PAGE_SIZE);
 752                 if (hd_info->page != deleted_page) {
 753                         deleted_page = hd_info->page;
 754                         mlx5e_page_release(rq, hd_info, false);
 755                 }
 756         }
 757
 758         if (start + len > hd_per_wq) {
 759                 len -= hd_per_wq - start;
 760                 bitmap_clear(shampo->bitmap, start, hd_per_wq - start);
 761                 start = 0;
 762         }
 763
 764         bitmap_clear(shampo->bitmap, start, len);
 765 }
 766
 767 static void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 768 {
 769         struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
 770         /* Don't recycle, this function is called on rq/netdev close */
 771         mlx5e_free_rx_mpwqe(rq, wi, false);
 772 }
 773
 774 INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
 775 {
 776         struct mlx5_wq_cyc *wq = &rq->wqe.wq;
 777         u8 wqe_bulk;
 778         int err;
 779
 780         if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
 781                 return false;
 782
 783         wqe_bulk = rq->wqe.info.wqe_bulk;
 784
 785         if (mlx5_wq_cyc_missing(wq) < wqe_bulk)
 786                 return false;
 787
 788         if (rq->page_pool)
 789                 page_pool_nid_changed(rq->page_pool, numa_mem_id());
 790
 791         do {
 792                 u16 head = mlx5_wq_cyc_get_head(wq);
 793
 794                 err = mlx5e_alloc_rx_wqes(rq, head, wqe_bulk);
 795                 if (unlikely(err)) {
 796                         rq->stats->buff_alloc_err++;
 797                         break;
 798                 }
 799
 800                 mlx5_wq_cyc_push_n(wq, wqe_bulk);
 801         } while (mlx5_wq_cyc_missing(wq) >= wqe_bulk);
 802
 803         /* ensure wqes are visible to device before updating doorbell record */
 804         dma_wmb();
 805
 806         mlx5_wq_cyc_update_db_record(wq);
 807
 808         return !!err;
 809 }
 810
 811 void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq)
 812 {
 813         u16 sqcc;
 814
 815         sqcc = sq->cc;
 816
 817         while (sqcc != sq->pc) {
 818                 struct mlx5e_icosq_wqe_info *wi;
 819                 u16 ci;
 820
 821                 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
 822                 wi = &sq->db.wqe_info[ci];
 823                 sqcc += wi->num_wqebbs;
 824 #ifdef CONFIG_MLX5_EN_TLS
 825                 switch (wi->wqe_type) {
 826                 case MLX5E_ICOSQ_WQE_SET_PSV_TLS:
 827                         mlx5e_ktls_handle_ctx_completion(wi);
 828                         break;
 829                 case MLX5E_ICOSQ_WQE_GET_PSV_TLS:
 830                         mlx5e_ktls_handle_get_psv_completion(wi, sq);
 831                         break;
 832                 }
 833 #endif
 834         }
 835         sq->cc = sqcc;
 836 }
 837
 838 static void mlx5e_handle_shampo_hd_umr(struct mlx5e_shampo_umr umr,
 839                                        struct mlx5e_icosq *sq)
 840 {
 841         struct mlx5e_channel *c = container_of(sq, struct mlx5e_channel, icosq);
 842         struct mlx5e_shampo_hd *shampo;
 843         /* assume 1:1 relationship between RQ and icosq */
 844         struct mlx5e_rq *rq = &c->rq;
 845         int end, from, len = umr.len;
 846
 847         shampo = rq->mpwqe.shampo;
 848         end = shampo->hd_per_wq;
 849         from = shampo->ci;
 850         if (from + len > shampo->hd_per_wq) {
 851                 len -= end - from;
 852                 bitmap_set(shampo->bitmap, from, end - from);
 853                 from = 0;
 854         }
 855
 856         bitmap_set(shampo->bitmap, from, len);
 857         shampo->ci = (shampo->ci + umr.len) & (shampo->hd_per_wq - 1);
 858 }
 859
 860 int mlx5e_poll_ico_cq(struct mlx5e_cq *cq)
 861 {
 862         struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq);
 863         struct mlx5_cqe64 *cqe;
 864         u16 sqcc;
 865         int i;
 866
 867         if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
 868                 return 0;
 869
 870         cqe = mlx5_cqwq_get_cqe(&cq->wq);
 871         if (likely(!cqe))
 872                 return 0;
 873
 874         /* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
 875          * otherwise a cq overrun may occur
 876          */
 877         sqcc = sq->cc;
 878
 879         i = 0;
 880         do {
 881                 u16 wqe_counter;
 882                 bool last_wqe;
 883
 884                 mlx5_cqwq_pop(&cq->wq);
 885
 886                 wqe_counter = be16_to_cpu(cqe->wqe_counter);
 887
 888                 do {
 889                         struct mlx5e_icosq_wqe_info *wi;
 890                         u16 ci;
 891
 892                         last_wqe = (sqcc == wqe_counter);
 893
 894                         ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
 895                         wi = &sq->db.wqe_info[ci];
 896                         sqcc += wi->num_wqebbs;
 897
 898                         if (last_wqe && unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
 899                                 netdev_WARN_ONCE(cq->netdev,
 900                                                  "Bad OP in ICOSQ CQE: 0x%x\n",
 901                                                  get_cqe_opcode(cqe));
 902                                 mlx5e_dump_error_cqe(&sq->cq, sq->sqn,
 903                                                      (struct mlx5_err_cqe *)cqe);
 904                                 mlx5_wq_cyc_wqe_dump(&sq->wq, ci, wi->num_wqebbs);
 905                                 if (!test_and_set_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state))
 906                                         queue_work(cq->priv->wq, &sq->recover_work);
 907                                 break;
 908                         }
 909
 910                         switch (wi->wqe_type) {
 911                         case MLX5E_ICOSQ_WQE_UMR_RX:
 912                                 wi->umr.rq->mpwqe.umr_completed++;
 913                                 break;
 914                         case MLX5E_ICOSQ_WQE_NOP:
 915                                 break;
 916                         case MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR:
 917                                 mlx5e_handle_shampo_hd_umr(wi->shampo, sq);
 918                                 break;
 919 #ifdef CONFIG_MLX5_EN_TLS
 920                         case MLX5E_ICOSQ_WQE_UMR_TLS:
 921                                 break;
 922                         case MLX5E_ICOSQ_WQE_SET_PSV_TLS:
 923                                 mlx5e_ktls_handle_ctx_completion(wi);
 924                                 break;
 925                         case MLX5E_ICOSQ_WQE_GET_PSV_TLS:
 926                                 mlx5e_ktls_handle_get_psv_completion(wi, sq);
 927                                 break;
 928 #endif
 929                         default:
 930                                 netdev_WARN_ONCE(cq->netdev,
 931                                                  "Bad WQE type in ICOSQ WQE info: 0x%x\n",
 932                                                  wi->wqe_type);
 933                         }
 934                 } while (!last_wqe);
 935         } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
 936
 937         sq->cc = sqcc;
 938
 939         mlx5_cqwq_update_db_record(&cq->wq);
 940
 941         return i;
 942 }
 943
 944 INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
 945 {
 946         struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
 947         u8  umr_completed = rq->mpwqe.umr_completed;
 948         struct mlx5e_icosq *sq = rq->icosq;
 949         int alloc_err = 0;
 950         u8  missing, i;
 951         u16 head;
 952
 953         if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
 954                 return false;
 955
 956         if (umr_completed) {
 957                 mlx5e_post_rx_mpwqe(rq, umr_completed);
 958                 rq->mpwqe.umr_in_progress -= umr_completed;
 959                 rq->mpwqe.umr_completed = 0;
 960         }
 961
 962         missing = mlx5_wq_ll_missing(wq) - rq->mpwqe.umr_in_progress;
 963
 964         if (unlikely(rq->mpwqe.umr_in_progress > rq->mpwqe.umr_last_bulk))
 965                 rq->stats->congst_umr++;
 966
 967         if (likely(missing < rq->mpwqe.min_wqe_bulk))
 968                 return false;
 969
 970         if (rq->page_pool)
 971                 page_pool_nid_changed(rq->page_pool, numa_mem_id());
 972
 973         head = rq->mpwqe.actual_wq_head;
 974         i = missing;
 975         do {
 976                 alloc_err = mlx5e_alloc_rx_mpwqe(rq, head);
 977
 978                 if (unlikely(alloc_err))
 979                         break;
 980                 head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
 981         } while (--i);
 982
 983         rq->mpwqe.umr_last_bulk    = missing - i;
 984         if (sq->doorbell_cseg) {
 985                 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, sq->doorbell_cseg);
 986                 sq->doorbell_cseg = NULL;
 987         }
 988
 989         rq->mpwqe.umr_in_progress += rq->mpwqe.umr_last_bulk;
 990         rq->mpwqe.actual_wq_head   = head;
 991
 992         /* If XSK Fill Ring doesn't have enough frames, report the error, so
 993          * that one of the actions can be performed:
 994          * 1. If need_wakeup is used, signal that the application has to kick
 995          * the driver when it refills the Fill Ring.
 996          * 2. Otherwise, busy poll by rescheduling the NAPI poll.
 997          */
 998         if (unlikely(alloc_err == -ENOMEM && rq->xsk_pool))
 999                 return true;
1000
1001         return false;
1002 }
1003
1004 static void mlx5e_lro_update_tcp_hdr(struct mlx5_cqe64 *cqe, struct tcphdr *tcp)
1005 {
1006         u8 l4_hdr_type = get_cqe_l4_hdr_type(cqe);
1007         u8 tcp_ack     = (l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA) ||
1008                          (l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA);
1009
1010         tcp->check                      = 0;
1011         tcp->psh                        = get_cqe_lro_tcppsh(cqe);
1012
1013         if (tcp_ack) {
1014                 tcp->ack                = 1;
1015                 tcp->ack_seq            = cqe->lro.ack_seq_num;
1016                 tcp->window             = cqe->lro.tcp_win;
1017         }
1018 }
1019
1020 static void mlx5e_lro_update_hdr(struct sk_buff *skb, struct mlx5_cqe64 *cqe,
1021                                  u32 cqe_bcnt)
1022 {
1023         struct ethhdr   *eth = (struct ethhdr *)(skb->data);
1024         struct tcphdr   *tcp;
1025         int network_depth = 0;
1026         __wsum check;
1027         __be16 proto;
1028         u16 tot_len;
1029         void *ip_p;
1030
1031         proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth);
1032
1033         tot_len = cqe_bcnt - network_depth;
1034         ip_p = skb->data + network_depth;
1035
1036         if (proto == htons(ETH_P_IP)) {
1037                 struct iphdr *ipv4 = ip_p;
1038
1039                 tcp = ip_p + sizeof(struct iphdr);
1040                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1041
1042                 ipv4->ttl               = cqe->lro.min_ttl;
1043                 ipv4->tot_len           = cpu_to_be16(tot_len);
1044                 ipv4->check             = 0;
1045                 ipv4->check             = ip_fast_csum((unsigned char *)ipv4,
1046                                                        ipv4->ihl);
1047
1048                 mlx5e_lro_update_tcp_hdr(cqe, tcp);
1049                 check = csum_partial(tcp, tcp->doff * 4,
1050                                      csum_unfold((__force __sum16)cqe->check_sum));
1051                 /* Almost done, don't forget the pseudo header */
1052                 tcp->check = csum_tcpudp_magic(ipv4->saddr, ipv4->daddr,
1053                                                tot_len - sizeof(struct iphdr),
1054                                                IPPROTO_TCP, check);
1055         } else {
1056                 u16 payload_len = tot_len - sizeof(struct ipv6hdr);
1057                 struct ipv6hdr *ipv6 = ip_p;
1058
1059                 tcp = ip_p + sizeof(struct ipv6hdr);
1060                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
1061
1062                 ipv6->hop_limit         = cqe->lro.min_ttl;
1063                 ipv6->payload_len       = cpu_to_be16(payload_len);
1064
1065                 mlx5e_lro_update_tcp_hdr(cqe, tcp);
1066                 check = csum_partial(tcp, tcp->doff * 4,
1067                                      csum_unfold((__force __sum16)cqe->check_sum));
1068                 /* Almost done, don't forget the pseudo header */
1069                 tcp->check = csum_ipv6_magic(&ipv6->saddr, &ipv6->daddr, payload_len,
1070                                              IPPROTO_TCP, check);
1071         }
1072 }
1073
1074 static void *mlx5e_shampo_get_packet_hd(struct mlx5e_rq *rq, u16 header_index)
1075 {
1076         struct mlx5e_dma_info *last_head = &rq->mpwqe.shampo->info[header_index];
1077         u16 head_offset = (last_head->addr & (PAGE_SIZE - 1)) + rq->buff.headroom;
1078
1079         return page_address(last_head->page) + head_offset;
1080 }
1081
1082 static void mlx5e_shampo_update_ipv4_udp_hdr(struct mlx5e_rq *rq, struct iphdr *ipv4)
1083 {
1084         int udp_off = rq->hw_gro_data->fk.control.thoff;
1085         struct sk_buff *skb = rq->hw_gro_data->skb;
1086         struct udphdr *uh;
1087
1088         uh = (struct udphdr *)(skb->data + udp_off);
1089         uh->len = htons(skb->len - udp_off);
1090
1091         if (uh->check)
1092                 uh->check = ~udp_v4_check(skb->len - udp_off, ipv4->saddr,
1093                                           ipv4->daddr, 0);
1094
1095         skb->csum_start = (unsigned char *)uh - skb->head;
1096         skb->csum_offset = offsetof(struct udphdr, check);
1097
1098         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_L4;
1099 }
1100
1101 static void mlx5e_shampo_update_ipv6_udp_hdr(struct mlx5e_rq *rq, struct ipv6hdr *ipv6)
1102 {
1103         int udp_off = rq->hw_gro_data->fk.control.thoff;
1104         struct sk_buff *skb = rq->hw_gro_data->skb;
1105         struct udphdr *uh;
1106
1107         uh = (struct udphdr *)(skb->data + udp_off);
1108         uh->len = htons(skb->len - udp_off);
1109
1110         if (uh->check)
1111                 uh->check = ~udp_v6_check(skb->len - udp_off, &ipv6->saddr,
1112                                           &ipv6->daddr, 0);
1113
1114         skb->csum_start = (unsigned char *)uh - skb->head;
1115         skb->csum_offset = offsetof(struct udphdr, check);
1116
1117         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_L4;
1118 }
1119
1120 static void mlx5e_shampo_update_fin_psh_flags(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
1121                                               struct tcphdr *skb_tcp_hd)
1122 {
1123         u16 header_index = mlx5e_shampo_get_cqe_header_index(rq, cqe);
1124         struct tcphdr *last_tcp_hd;
1125         void *last_hd_addr;
1126
1127         last_hd_addr = mlx5e_shampo_get_packet_hd(rq, header_index);
1128         last_tcp_hd =  last_hd_addr + ETH_HLEN + rq->hw_gro_data->fk.control.thoff;
1129         tcp_flag_word(skb_tcp_hd) |= tcp_flag_word(last_tcp_hd) & (TCP_FLAG_FIN | TCP_FLAG_PSH);
1130 }
1131
1132 static void mlx5e_shampo_update_ipv4_tcp_hdr(struct mlx5e_rq *rq, struct iphdr *ipv4,
1133                                              struct mlx5_cqe64 *cqe, bool match)
1134 {
1135         int tcp_off = rq->hw_gro_data->fk.control.thoff;
1136         struct sk_buff *skb = rq->hw_gro_data->skb;
1137         struct tcphdr *tcp;
1138
1139         tcp = (struct tcphdr *)(skb->data + tcp_off);
1140         if (match)
1141                 mlx5e_shampo_update_fin_psh_flags(rq, cqe, tcp);
1142
1143         tcp->check = ~tcp_v4_check(skb->len - tcp_off, ipv4->saddr,
1144                                    ipv4->daddr, 0);
1145         skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV4;
1146         if (ntohs(ipv4->id) == rq->hw_gro_data->second_ip_id)
1147                 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_FIXEDID;
1148
1149         skb->csum_start = (unsigned char *)tcp - skb->head;
1150         skb->csum_offset = offsetof(struct tcphdr, check);
1151
1152         if (tcp->cwr)
1153                 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
1154 }
1155
1156 static void mlx5e_shampo_update_ipv6_tcp_hdr(struct mlx5e_rq *rq, struct ipv6hdr *ipv6,
1157                                              struct mlx5_cqe64 *cqe, bool match)
1158 {
1159         int tcp_off = rq->hw_gro_data->fk.control.thoff;
1160         struct sk_buff *skb = rq->hw_gro_data->skb;
1161         struct tcphdr *tcp;
1162
1163         tcp = (struct tcphdr *)(skb->data + tcp_off);
1164         if (match)
1165                 mlx5e_shampo_update_fin_psh_flags(rq, cqe, tcp);
1166
1167         tcp->check = ~tcp_v6_check(skb->len - tcp_off, &ipv6->saddr,
1168                                    &ipv6->daddr, 0);
1169         skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6;
1170         skb->csum_start = (unsigned char *)tcp - skb->head;
1171         skb->csum_offset = offsetof(struct tcphdr, check);
1172
1173         if (tcp->cwr)
1174                 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
1175 }
1176
1177 static void mlx5e_shampo_update_hdr(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, bool match)
1178 {
1179         bool is_ipv4 = (rq->hw_gro_data->fk.basic.n_proto == htons(ETH_P_IP));
1180         struct sk_buff *skb = rq->hw_gro_data->skb;
1181
1182         skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
1183         skb->ip_summed = CHECKSUM_PARTIAL;
1184
1185         if (is_ipv4) {
1186                 int nhoff = rq->hw_gro_data->fk.control.thoff - sizeof(struct iphdr);
1187                 struct iphdr *ipv4 = (struct iphdr *)(skb->data + nhoff);
1188                 __be16 newlen = htons(skb->len - nhoff);
1189
1190                 csum_replace2(&ipv4->check, ipv4->tot_len, newlen);
1191                 ipv4->tot_len = newlen;
1192
1193                 if (ipv4->protocol == IPPROTO_TCP)
1194                         mlx5e_shampo_update_ipv4_tcp_hdr(rq, ipv4, cqe, match);
1195                 else
1196                         mlx5e_shampo_update_ipv4_udp_hdr(rq, ipv4);
1197         } else {
1198                 int nhoff = rq->hw_gro_data->fk.control.thoff - sizeof(struct ipv6hdr);
1199                 struct ipv6hdr *ipv6 = (struct ipv6hdr *)(skb->data + nhoff);
1200
1201                 ipv6->payload_len = htons(skb->len - nhoff - sizeof(*ipv6));
1202
1203                 if (ipv6->nexthdr == IPPROTO_TCP)
1204                         mlx5e_shampo_update_ipv6_tcp_hdr(rq, ipv6, cqe, match);
1205                 else
1206                         mlx5e_shampo_update_ipv6_udp_hdr(rq, ipv6);
1207         }
1208 }
1209
1210 static inline void mlx5e_skb_set_hash(struct mlx5_cqe64 *cqe,
1211                                       struct sk_buff *skb)
1212 {
1213         u8 cht = cqe->rss_hash_type;
1214         int ht = (cht & CQE_RSS_HTYPE_L4) ? PKT_HASH_TYPE_L4 :
1215                  (cht & CQE_RSS_HTYPE_IP) ? PKT_HASH_TYPE_L3 :
1216                                             PKT_HASH_TYPE_NONE;
1217         skb_set_hash(skb, be32_to_cpu(cqe->rss_hash_result), ht);
1218 }
1219
1220 static inline bool is_last_ethertype_ip(struct sk_buff *skb, int *network_depth,
1221                                         __be16 *proto)
1222 {
1223         *proto = ((struct ethhdr *)skb->data)->h_proto;
1224         *proto = __vlan_get_protocol(skb, *proto, network_depth);
1225
1226         if (*proto == htons(ETH_P_IP))
1227                 return pskb_may_pull(skb, *network_depth + sizeof(struct iphdr));
1228
1229         if (*proto == htons(ETH_P_IPV6))
1230                 return pskb_may_pull(skb, *network_depth + sizeof(struct ipv6hdr));
1231
1232         return false;
1233 }
1234
1235 static inline void mlx5e_enable_ecn(struct mlx5e_rq *rq, struct sk_buff *skb)
1236 {
1237         int network_depth = 0;
1238         __be16 proto;
1239         void *ip;
1240         int rc;
1241
1242         if (unlikely(!is_last_ethertype_ip(skb, &network_depth, &proto)))
1243                 return;
1244
1245         ip = skb->data + network_depth;
1246         rc = ((proto == htons(ETH_P_IP)) ? IP_ECN_set_ce((struct iphdr *)ip) :
1247                                          IP6_ECN_set_ce(skb, (struct ipv6hdr *)ip));
1248
1249         rq->stats->ecn_mark += !!rc;
1250 }
1251
1252 static u8 get_ip_proto(struct sk_buff *skb, int network_depth, __be16 proto)
1253 {
1254         void *ip_p = skb->data + network_depth;
1255
1256         return (proto == htons(ETH_P_IP)) ? ((struct iphdr *)ip_p)->protocol :
1257                                             ((struct ipv6hdr *)ip_p)->nexthdr;
1258 }
1259
1260 #define short_frame(size) ((size) <= ETH_ZLEN + ETH_FCS_LEN)
1261
1262 #define MAX_PADDING 8
1263
1264 static void
1265 tail_padding_csum_slow(struct sk_buff *skb, int offset, int len,
1266                        struct mlx5e_rq_stats *stats)
1267 {
1268         stats->csum_complete_tail_slow++;
1269         skb->csum = csum_block_add(skb->csum,
1270                                    skb_checksum(skb, offset, len, 0),
1271                                    offset);
1272 }
1273
1274 static void
1275 tail_padding_csum(struct sk_buff *skb, int offset,
1276                   struct mlx5e_rq_stats *stats)
1277 {
1278         u8 tail_padding[MAX_PADDING];
1279         int len = skb->len - offset;
1280         void *tail;
1281
1282         if (unlikely(len > MAX_PADDING)) {
1283                 tail_padding_csum_slow(skb, offset, len, stats);
1284                 return;
1285         }
1286
1287         tail = skb_header_pointer(skb, offset, len, tail_padding);
1288         if (unlikely(!tail)) {
1289                 tail_padding_csum_slow(skb, offset, len, stats);
1290                 return;
1291         }
1292
1293         stats->csum_complete_tail++;
1294         skb->csum = csum_block_add(skb->csum, csum_partial(tail, len, 0), offset);
1295 }
1296
1297 static void
1298 mlx5e_skb_csum_fixup(struct sk_buff *skb, int network_depth, __be16 proto,
1299                      struct mlx5e_rq_stats *stats)
1300 {
1301         struct ipv6hdr *ip6;
1302         struct iphdr   *ip4;
1303         int pkt_len;
1304
1305         /* Fixup vlan headers, if any */
1306         if (network_depth > ETH_HLEN)
1307                 /* CQE csum is calculated from the IP header and does
1308                  * not cover VLAN headers (if present). This will add
1309                  * the checksum manually.
1310                  */
1311                 skb->csum = csum_partial(skb->data + ETH_HLEN,
1312                                          network_depth - ETH_HLEN,
1313                                          skb->csum);
1314
1315         /* Fixup tail padding, if any */
1316         switch (proto) {
1317         case htons(ETH_P_IP):
1318                 ip4 = (struct iphdr *)(skb->data + network_depth);
1319                 pkt_len = network_depth + ntohs(ip4->tot_len);
1320                 break;
1321         case htons(ETH_P_IPV6):
1322                 ip6 = (struct ipv6hdr *)(skb->data + network_depth);
1323                 pkt_len = network_depth + sizeof(*ip6) + ntohs(ip6->payload_len);
1324                 break;
1325         default:
1326                 return;
1327         }
1328
1329         if (likely(pkt_len >= skb->len))
1330                 return;
1331
1332         tail_padding_csum(skb, pkt_len, stats);
1333 }
1334
1335 static inline void mlx5e_handle_csum(struct net_device *netdev,
1336                                      struct mlx5_cqe64 *cqe,
1337                                      struct mlx5e_rq *rq,
1338                                      struct sk_buff *skb,
1339                                      bool   lro)
1340 {
1341         struct mlx5e_rq_stats *stats = rq->stats;
1342         int network_depth = 0;
1343         __be16 proto;
1344
1345         if (unlikely(!(netdev->features & NETIF_F_RXCSUM)))
1346                 goto csum_none;
1347
1348         if (lro) {
1349                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1350                 stats->csum_unnecessary++;
1351                 return;
1352         }
1353
1354         /* True when explicitly set via priv flag, or XDP prog is loaded */
1355         if (test_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state) ||
1356             get_cqe_tls_offload(cqe))
1357                 goto csum_unnecessary;
1358
1359         /* CQE csum doesn't cover padding octets in short ethernet
1360          * frames. And the pad field is appended prior to calculating
1361          * and appending the FCS field.
1362          *
1363          * Detecting these padded frames requires to verify and parse
1364          * IP headers, so we simply force all those small frames to be
1365          * CHECKSUM_UNNECESSARY even if they are not padded.
1366          */
1367         if (short_frame(skb->len))
1368                 goto csum_unnecessary;
1369
1370         if (likely(is_last_ethertype_ip(skb, &network_depth, &proto))) {
1371                 if (unlikely(get_ip_proto(skb, network_depth, proto) == IPPROTO_SCTP))
1372                         goto csum_unnecessary;
1373
1374                 stats->csum_complete++;
1375                 skb->ip_summed = CHECKSUM_COMPLETE;
1376                 skb->csum = csum_unfold((__force __sum16)cqe->check_sum);
1377
1378                 if (test_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state))
1379                         return; /* CQE csum covers all received bytes */
1380
1381                 /* csum might need some fixups ...*/
1382                 mlx5e_skb_csum_fixup(skb, network_depth, proto, stats);
1383                 return;
1384         }
1385
1386 csum_unnecessary:
1387         if (likely((cqe->hds_ip_ext & CQE_L3_OK) &&
1388                    (cqe->hds_ip_ext & CQE_L4_OK))) {
1389                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1390                 if (cqe_is_tunneled(cqe)) {
1391                         skb->csum_level = 1;
1392                         skb->encapsulation = 1;
1393                         stats->csum_unnecessary_inner++;
1394                         return;
1395                 }
1396                 stats->csum_unnecessary++;
1397                 return;
1398         }
1399 csum_none:
1400         skb->ip_summed = CHECKSUM_NONE;
1401         stats->csum_none++;
1402 }
1403
1404 #define MLX5E_CE_BIT_MASK 0x80
1405
1406 static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe,
1407                                       u32 cqe_bcnt,
1408                                       struct mlx5e_rq *rq,
1409                                       struct sk_buff *skb)
1410 {
1411         u8 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24;
1412         struct mlx5e_rq_stats *stats = rq->stats;
1413         struct net_device *netdev = rq->netdev;
1414
1415         skb->mac_len = ETH_HLEN;
1416
1417         mlx5e_tls_handle_rx_skb(rq, skb, cqe, &cqe_bcnt);
1418
1419         if (unlikely(mlx5_ipsec_is_rx_flow(cqe)))
1420                 mlx5e_ipsec_offload_handle_rx_skb(netdev, skb, cqe);
1421
1422         if (lro_num_seg > 1) {
1423                 mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt);
1424                 skb_shinfo(skb)->gso_size = DIV_ROUND_UP(cqe_bcnt, lro_num_seg);
1425                 /* Subtract one since we already counted this as one
1426                  * "regular" packet in mlx5e_complete_rx_cqe()
1427                  */
1428                 stats->packets += lro_num_seg - 1;
1429                 stats->lro_packets++;
1430                 stats->lro_bytes += cqe_bcnt;
1431         }
1432
1433         if (unlikely(mlx5e_rx_hw_stamp(rq->tstamp)))
1434                 skb_hwtstamps(skb)->hwtstamp = mlx5e_cqe_ts_to_ns(rq->ptp_cyc2time,
1435                                                                   rq->clock, get_cqe_ts(cqe));
1436         skb_record_rx_queue(skb, rq->ix);
1437
1438         if (likely(netdev->features & NETIF_F_RXHASH))
1439                 mlx5e_skb_set_hash(cqe, skb);
1440
1441         if (cqe_has_vlan(cqe)) {
1442                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1443                                        be16_to_cpu(cqe->vlan_info));
1444                 stats->removed_vlan_packets++;
1445         }
1446
1447         skb->mark = be32_to_cpu(cqe->sop_drop_qpn) & MLX5E_TC_FLOW_ID_MASK;
1448
1449         mlx5e_handle_csum(netdev, cqe, rq, skb, !!lro_num_seg);
1450         /* checking CE bit in cqe - MSB in ml_path field */
1451         if (unlikely(cqe->ml_path & MLX5E_CE_BIT_MASK))
1452                 mlx5e_enable_ecn(rq, skb);
1453
1454         skb->protocol = eth_type_trans(skb, netdev);
1455
1456         if (unlikely(mlx5e_skb_is_multicast(skb)))
1457                 stats->mcast_packets++;
1458 }
1459
1460 static void mlx5e_shampo_complete_rx_cqe(struct mlx5e_rq *rq,
1461                                          struct mlx5_cqe64 *cqe,
1462                                          u32 cqe_bcnt,
1463                                          struct sk_buff *skb)
1464 {
1465         struct mlx5e_rq_stats *stats = rq->stats;
1466
1467         stats->packets++;
1468         stats->gro_packets++;
1469         stats->bytes += cqe_bcnt;
1470         stats->gro_bytes += cqe_bcnt;
1471         if (NAPI_GRO_CB(skb)->count != 1)
1472                 return;
1473         mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb);
1474         skb_reset_network_header(skb);
1475         if (!skb_flow_dissect_flow_keys(skb, &rq->hw_gro_data->fk, 0)) {
1476                 napi_gro_receive(rq->cq.napi, skb);
1477                 rq->hw_gro_data->skb = NULL;
1478         }
1479 }
1480
1481 static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq *rq,
1482                                          struct mlx5_cqe64 *cqe,
1483                                          u32 cqe_bcnt,
1484                                          struct sk_buff *skb)
1485 {
1486         struct mlx5e_rq_stats *stats = rq->stats;
1487
1488         stats->packets++;
1489         stats->bytes += cqe_bcnt;
1490         mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb);
1491 }
1492
1493 static inline
1494 struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va,
1495                                        u32 frag_size, u16 headroom,
1496                                        u32 cqe_bcnt, u32 metasize)
1497 {
1498         struct sk_buff *skb = build_skb(va, frag_size);
1499
1500         if (unlikely(!skb)) {
1501                 rq->stats->buff_alloc_err++;
1502                 return NULL;
1503         }
1504
1505         skb_reserve(skb, headroom);
1506         skb_put(skb, cqe_bcnt);
1507
1508         if (metasize)
1509                 skb_metadata_set(skb, metasize);
1510
1511         return skb;
1512 }
1513
1514 static void mlx5e_fill_xdp_buff(struct mlx5e_rq *rq, void *va, u16 headroom,
1515                                 u32 len, struct xdp_buff *xdp)
1516 {
1517         xdp_init_buff(xdp, rq->buff.frame0_sz, &rq->xdp_rxq);
1518         xdp_prepare_buff(xdp, va, headroom, len, true);
1519 }
1520
1521 static struct sk_buff *
1522 mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
1523                           struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
1524 {
1525         struct mlx5e_dma_info *di = wi->di;
1526         u16 rx_headroom = rq->buff.headroom;
1527         struct bpf_prog *prog;
1528         struct sk_buff *skb;
1529         u32 metasize = 0;
1530         void *va, *data;
1531         u32 frag_size;
1532
1533         va             = page_address(di->page) + wi->offset;
1534         data           = va + rx_headroom;
1535         frag_size      = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
1536
1537         dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset,
1538                                       frag_size, DMA_FROM_DEVICE);
1539         net_prefetch(data);
1540
1541         prog = rcu_dereference(rq->xdp_prog);
1542         if (prog) {
1543                 struct xdp_buff xdp;
1544
1545                 net_prefetchw(va); /* xdp_frame data area */
1546                 mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp);
1547                 if (mlx5e_xdp_handle(rq, di, prog, &xdp))
1548                         return NULL; /* page/packet was consumed by XDP */
1549
1550                 rx_headroom = xdp.data - xdp.data_hard_start;
1551                 metasize = xdp.data - xdp.data_meta;
1552                 cqe_bcnt = xdp.data_end - xdp.data;
1553         }
1554         frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
1555         skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt, metasize);
1556         if (unlikely(!skb))
1557                 return NULL;
1558
1559         /* queue up for recycling/reuse */
1560         page_ref_inc(di->page);
1561
1562         return skb;
1563 }
1564
1565 static struct sk_buff *
1566 mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
1567                              struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
1568 {
1569         struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
1570         u16 rx_headroom = rq->buff.headroom;
1571         struct mlx5e_dma_info *di = wi->di;
1572         u32 frag_consumed_bytes;
1573         u32 first_frag_size;
1574         struct sk_buff *skb;
1575         void *va;
1576
1577         va = page_address(di->page) + wi->offset;
1578         frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt);
1579         first_frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + frag_consumed_bytes);
1580
1581         dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset,
1582                                       first_frag_size, DMA_FROM_DEVICE);
1583         net_prefetch(va + rx_headroom);
1584
1585         /* XDP is not supported in this configuration, as incoming packets
1586          * might spread among multiple pages.
1587          */
1588         skb = mlx5e_build_linear_skb(rq, va, first_frag_size, rx_headroom,
1589                                      frag_consumed_bytes, 0);
1590         if (unlikely(!skb))
1591                 return NULL;
1592
1593         page_ref_inc(di->page);
1594
1595         cqe_bcnt -= frag_consumed_bytes;
1596         frag_info++;
1597         wi++;
1598
1599         while (cqe_bcnt) {
1600                 frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt);
1601
1602                 mlx5e_add_skb_frag(rq, skb, wi->di, wi->offset,
1603                                    frag_consumed_bytes, frag_info->frag_stride);
1604                 cqe_bcnt -= frag_consumed_bytes;
1605                 frag_info++;
1606                 wi++;
1607         }
1608
1609         return skb;
1610 }
1611
1612 static void trigger_report(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1613 {
1614         struct mlx5_err_cqe *err_cqe = (struct mlx5_err_cqe *)cqe;
1615         struct mlx5e_priv *priv = rq->priv;
1616
1617         if (cqe_syndrome_needs_recover(err_cqe->syndrome) &&
1618             !test_and_set_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state)) {
1619                 mlx5e_dump_error_cqe(&rq->cq, rq->rqn, err_cqe);
1620                 queue_work(priv->wq, &rq->recover_work);
1621         }
1622 }
1623
1624 static void mlx5e_handle_rx_err_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1625 {
1626         trigger_report(rq, cqe);
1627         rq->stats->wqe_err++;
1628 }
1629
1630 static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1631 {
1632         struct mlx5_wq_cyc *wq = &rq->wqe.wq;
1633         struct mlx5e_wqe_frag_info *wi;
1634         struct sk_buff *skb;
1635         u32 cqe_bcnt;
1636         u16 ci;
1637
1638         ci       = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
1639         wi       = get_frag(rq, ci);
1640         cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
1641
1642         if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1643                 mlx5e_handle_rx_err_cqe(rq, cqe);
1644                 goto free_wqe;
1645         }
1646
1647         skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
1648                               mlx5e_skb_from_cqe_linear,
1649                               mlx5e_skb_from_cqe_nonlinear,
1650                               rq, cqe, wi, cqe_bcnt);
1651         if (!skb) {
1652                 /* probably for XDP */
1653                 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
1654                         /* do not return page to cache,
1655                          * it will be returned on XDP_TX completion.
1656                          */
1657                         goto wq_cyc_pop;
1658                 }
1659                 goto free_wqe;
1660         }
1661
1662         mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1663
1664         if (mlx5e_cqe_regb_chain(cqe))
1665                 if (!mlx5e_tc_update_skb(cqe, skb)) {
1666                         dev_kfree_skb_any(skb);
1667                         goto free_wqe;
1668                 }
1669
1670         napi_gro_receive(rq->cq.napi, skb);
1671
1672 free_wqe:
1673         mlx5e_free_rx_wqe(rq, wi, true);
1674 wq_cyc_pop:
1675         mlx5_wq_cyc_pop(wq);
1676 }
1677
1678 #ifdef CONFIG_MLX5_ESWITCH
1679 static void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1680 {
1681         struct net_device *netdev = rq->netdev;
1682         struct mlx5e_priv *priv = netdev_priv(netdev);
1683         struct mlx5e_rep_priv *rpriv  = priv->ppriv;
1684         struct mlx5_eswitch_rep *rep = rpriv->rep;
1685         struct mlx5_wq_cyc *wq = &rq->wqe.wq;
1686         struct mlx5e_wqe_frag_info *wi;
1687         struct sk_buff *skb;
1688         u32 cqe_bcnt;
1689         u16 ci;
1690
1691         ci       = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
1692         wi       = get_frag(rq, ci);
1693         cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
1694
1695         if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1696                 mlx5e_handle_rx_err_cqe(rq, cqe);
1697                 goto free_wqe;
1698         }
1699
1700         skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
1701                               mlx5e_skb_from_cqe_linear,
1702                               mlx5e_skb_from_cqe_nonlinear,
1703                               rq, cqe, wi, cqe_bcnt);
1704         if (!skb) {
1705                 /* probably for XDP */
1706                 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
1707                         /* do not return page to cache,
1708                          * it will be returned on XDP_TX completion.
1709                          */
1710                         goto wq_cyc_pop;
1711                 }
1712                 goto free_wqe;
1713         }
1714
1715         mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1716
1717         if (rep->vlan && skb_vlan_tag_present(skb))
1718                 skb_vlan_pop(skb);
1719
1720         mlx5e_rep_tc_receive(cqe, rq, skb);
1721
1722 free_wqe:
1723         mlx5e_free_rx_wqe(rq, wi, true);
1724 wq_cyc_pop:
1725         mlx5_wq_cyc_pop(wq);
1726 }
1727
1728 static void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1729 {
1730         u16 cstrides       = mpwrq_get_cqe_consumed_strides(cqe);
1731         u16 wqe_id         = be16_to_cpu(cqe->wqe_id);
1732         struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id];
1733         u16 stride_ix      = mpwrq_get_cqe_stride_index(cqe);
1734         u32 wqe_offset     = stride_ix << rq->mpwqe.log_stride_sz;
1735         u32 head_offset    = wqe_offset & (PAGE_SIZE - 1);
1736         u32 page_idx       = wqe_offset >> PAGE_SHIFT;
1737         struct mlx5e_rx_wqe_ll *wqe;
1738         struct mlx5_wq_ll *wq;
1739         struct sk_buff *skb;
1740         u16 cqe_bcnt;
1741
1742         wi->consumed_strides += cstrides;
1743
1744         if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1745                 mlx5e_handle_rx_err_cqe(rq, cqe);
1746                 goto mpwrq_cqe_out;
1747         }
1748
1749         if (unlikely(mpwrq_is_filler_cqe(cqe))) {
1750                 struct mlx5e_rq_stats *stats = rq->stats;
1751
1752                 stats->mpwqe_filler_cqes++;
1753                 stats->mpwqe_filler_strides += cstrides;
1754                 goto mpwrq_cqe_out;
1755         }
1756
1757         cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe);
1758
1759         skb = INDIRECT_CALL_2(rq->mpwqe.skb_from_cqe_mpwrq,
1760                               mlx5e_skb_from_cqe_mpwrq_linear,
1761                               mlx5e_skb_from_cqe_mpwrq_nonlinear,
1762                               rq, wi, cqe_bcnt, head_offset, page_idx);
1763         if (!skb)
1764                 goto mpwrq_cqe_out;
1765
1766         mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1767
1768         mlx5e_rep_tc_receive(cqe, rq, skb);
1769
1770 mpwrq_cqe_out:
1771         if (likely(wi->consumed_strides < rq->mpwqe.num_strides))
1772                 return;
1773
1774         wq  = &rq->mpwqe.wq;
1775         wqe = mlx5_wq_ll_get_wqe(wq, wqe_id);
1776         mlx5e_free_rx_mpwqe(rq, wi, true);
1777         mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
1778 }
1779
1780 const struct mlx5e_rx_handlers mlx5e_rx_handlers_rep = {
1781         .handle_rx_cqe       = mlx5e_handle_rx_cqe_rep,
1782         .handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq_rep,
1783 };
1784 #endif
1785
1786 static void
1787 mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
1788                     u32 data_bcnt, u32 data_offset)
1789 {
1790         net_prefetchw(skb->data);
1791
1792         while (data_bcnt) {
1793                 u32 pg_consumed_bytes = min_t(u32, PAGE_SIZE - data_offset, data_bcnt);
1794                 unsigned int truesize;
1795
1796                 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
1797                         truesize = pg_consumed_bytes;
1798                 else
1799                         truesize = ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz));
1800
1801                 mlx5e_add_skb_frag(rq, skb, di, data_offset,
1802                                    pg_consumed_bytes, truesize);
1803
1804                 data_bcnt -= pg_consumed_bytes;
1805                 data_offset = 0;
1806                 di++;
1807         }
1808 }
1809
1810 static struct sk_buff *
1811 mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
1812                                    u16 cqe_bcnt, u32 head_offset, u32 page_idx)
1813 {
1814         u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt);
1815         struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
1816         u32 frag_offset    = head_offset + headlen;
1817         u32 byte_cnt       = cqe_bcnt - headlen;
1818         struct mlx5e_dma_info *head_di = di;
1819         struct sk_buff *skb;
1820
1821         skb = napi_alloc_skb(rq->cq.napi,
1822                              ALIGN(MLX5E_RX_MAX_HEAD, sizeof(long)));
1823         if (unlikely(!skb)) {
1824                 rq->stats->buff_alloc_err++;
1825                 return NULL;
1826         }
1827
1828         net_prefetchw(skb->data);
1829
1830         if (unlikely(frag_offset >= PAGE_SIZE)) {
1831                 di++;
1832                 frag_offset -= PAGE_SIZE;
1833         }
1834
1835         mlx5e_fill_skb_data(skb, rq, di, byte_cnt, frag_offset);
1836         /* copy header */
1837         mlx5e_copy_skb_header(rq->pdev, skb, head_di, head_offset, head_offset, headlen);
1838         /* skb linear part was allocated with headlen and aligned to long */
1839         skb->tail += headlen;
1840         skb->len  += headlen;
1841
1842         return skb;
1843 }
1844
1845 static struct sk_buff *
1846 mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
1847                                 u16 cqe_bcnt, u32 head_offset, u32 page_idx)
1848 {
1849         struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
1850         u16 rx_headroom = rq->buff.headroom;
1851         u32 cqe_bcnt32 = cqe_bcnt;
1852         struct bpf_prog *prog;
1853         struct sk_buff *skb;
1854         u32 metasize = 0;
1855         void *va, *data;
1856         u32 frag_size;
1857
1858         /* Check packet size. Note LRO doesn't use linear SKB */
1859         if (unlikely(cqe_bcnt > rq->hw_mtu)) {
1860                 rq->stats->oversize_pkts_sw_drop++;
1861                 return NULL;
1862         }
1863
1864         va             = page_address(di->page) + head_offset;
1865         data           = va + rx_headroom;
1866         frag_size      = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt32);
1867
1868         dma_sync_single_range_for_cpu(rq->pdev, di->addr, head_offset,
1869                                       frag_size, DMA_FROM_DEVICE);
1870         net_prefetch(data);
1871
1872         prog = rcu_dereference(rq->xdp_prog);
1873         if (prog) {
1874                 struct xdp_buff xdp;
1875
1876                 net_prefetchw(va); /* xdp_frame data area */
1877                 mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt32, &xdp);
1878                 if (mlx5e_xdp_handle(rq, di, prog, &xdp)) {
1879                         if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags))
1880                                 __set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */
1881                         return NULL; /* page/packet was consumed by XDP */
1882                 }
1883
1884                 rx_headroom = xdp.data - xdp.data_hard_start;
1885                 metasize = xdp.data - xdp.data_meta;
1886                 cqe_bcnt32 = xdp.data_end - xdp.data;
1887         }
1888         frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt32);
1889         skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt32, metasize);
1890         if (unlikely(!skb))
1891                 return NULL;
1892
1893         /* queue up for recycling/reuse */
1894         page_ref_inc(di->page);
1895
1896         return skb;
1897 }
1898
1899 static struct sk_buff *
1900 mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
1901                           struct mlx5_cqe64 *cqe, u16 header_index)
1902 {
1903         struct mlx5e_dma_info *head = &rq->mpwqe.shampo->info[header_index];
1904         u16 head_offset = head->addr & (PAGE_SIZE - 1);
1905         u16 head_size = cqe->shampo.header_size;
1906         u16 rx_headroom = rq->buff.headroom;
1907         struct sk_buff *skb = NULL;
1908         void *hdr, *data;
1909         u32 frag_size;
1910
1911         hdr             = page_address(head->page) + head_offset;
1912         data            = hdr + rx_headroom;
1913         frag_size       = MLX5_SKB_FRAG_SZ(rx_headroom + head_size);
1914
1915         if (likely(frag_size <= BIT(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE))) {
1916                 /* build SKB around header */
1917                 dma_sync_single_range_for_cpu(rq->pdev, head->addr, 0, frag_size, DMA_FROM_DEVICE);
1918                 prefetchw(hdr);
1919                 prefetch(data);
1920                 skb = mlx5e_build_linear_skb(rq, hdr, frag_size, rx_headroom, head_size, 0);
1921
1922                 if (unlikely(!skb))
1923                         return NULL;
1924
1925                 /* queue up for recycling/reuse */
1926                 page_ref_inc(head->page);
1927
1928         } else {
1929                 /* allocate SKB and copy header for large header */
1930                 rq->stats->gro_large_hds++;
1931                 skb = napi_alloc_skb(rq->cq.napi,
1932                                      ALIGN(head_size, sizeof(long)));
1933                 if (unlikely(!skb)) {
1934                         rq->stats->buff_alloc_err++;
1935                         return NULL;
1936                 }
1937
1938                 prefetchw(skb->data);
1939                 mlx5e_copy_skb_header(rq->pdev, skb, head,
1940                                       head_offset + rx_headroom,
1941                                       rx_headroom, head_size);
1942                 /* skb linear part was allocated with headlen and aligned to long */
1943                 skb->tail += head_size;
1944                 skb->len  += head_size;
1945         }
1946         return skb;
1947 }
1948
1949 static void
1950 mlx5e_shampo_align_fragment(struct sk_buff *skb, u8 log_stride_sz)
1951 {
1952         skb_frag_t *last_frag = &skb_shinfo(skb)->frags[skb_shinfo(skb)->nr_frags - 1];
1953         unsigned int frag_size = skb_frag_size(last_frag);
1954         unsigned int frag_truesize;
1955
1956         frag_truesize = ALIGN(frag_size, BIT(log_stride_sz));
1957         skb->truesize += frag_truesize - frag_size;
1958 }
1959
1960 static void
1961 mlx5e_shampo_flush_skb(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, bool match)
1962 {
1963         struct sk_buff *skb = rq->hw_gro_data->skb;
1964         struct mlx5e_rq_stats *stats = rq->stats;
1965
1966         stats->gro_skbs++;
1967         if (likely(skb_shinfo(skb)->nr_frags))
1968                 mlx5e_shampo_align_fragment(skb, rq->mpwqe.log_stride_sz);
1969         if (NAPI_GRO_CB(skb)->count > 1)
1970                 mlx5e_shampo_update_hdr(rq, cqe, match);
1971         napi_gro_receive(rq->cq.napi, skb);
1972         rq->hw_gro_data->skb = NULL;
1973 }
1974
1975 static bool
1976 mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb, u16 data_bcnt)
1977 {
1978         int nr_frags = skb_shinfo(skb)->nr_frags;
1979
1980         return PAGE_SIZE * nr_frags + data_bcnt <= GSO_MAX_SIZE;
1981 }
1982
1983 static void
1984 mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index)
1985 {
1986         struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
1987         u64 addr = shampo->info[header_index].addr;
1988
1989         if (((header_index + 1) & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) == 0) {
1990                 shampo->info[header_index].addr = ALIGN_DOWN(addr, PAGE_SIZE);
1991                 mlx5e_page_release(rq, &shampo->info[header_index], true);
1992         }
1993         bitmap_clear(shampo->bitmap, header_index, 1);
1994 }
1995
1996 static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1997 {
1998         u16 data_bcnt           = mpwrq_get_cqe_byte_cnt(cqe) - cqe->shampo.header_size;
1999         u16 header_index        = mlx5e_shampo_get_cqe_header_index(rq, cqe);
2000         u32 wqe_offset          = be32_to_cpu(cqe->shampo.data_offset);
2001         u16 cstrides            = mpwrq_get_cqe_consumed_strides(cqe);
2002         u32 data_offset         = wqe_offset & (PAGE_SIZE - 1);
2003         u32 cqe_bcnt            = mpwrq_get_cqe_byte_cnt(cqe);
2004         u16 wqe_id              = be16_to_cpu(cqe->wqe_id);
2005         u32 page_idx            = wqe_offset >> PAGE_SHIFT;
2006         u16 head_size           = cqe->shampo.header_size;
2007         struct sk_buff **skb    = &rq->hw_gro_data->skb;
2008         bool flush              = cqe->shampo.flush;
2009         bool match              = cqe->shampo.match;
2010         struct mlx5e_rq_stats *stats = rq->stats;
2011         struct mlx5e_rx_wqe_ll *wqe;
2012         struct mlx5e_dma_info *di;
2013         struct mlx5e_mpw_info *wi;
2014         struct mlx5_wq_ll *wq;
2015
2016         wi = &rq->mpwqe.info[wqe_id];
2017         wi->consumed_strides += cstrides;
2018
2019         if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
2020                 mlx5e_handle_rx_err_cqe(rq, cqe);
2021                 goto mpwrq_cqe_out;
2022         }
2023
2024         if (unlikely(mpwrq_is_filler_cqe(cqe))) {
2025                 stats->mpwqe_filler_cqes++;
2026                 stats->mpwqe_filler_strides += cstrides;
2027                 goto mpwrq_cqe_out;
2028         }
2029
2030         stats->gro_match_packets += match;
2031
2032         if (*skb && (!match || !(mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt)))) {
2033                 match = false;
2034                 mlx5e_shampo_flush_skb(rq, cqe, match);
2035         }
2036
2037         if (!*skb) {
2038                 if (likely(head_size))
2039                         *skb = mlx5e_skb_from_cqe_shampo(rq, wi, cqe, header_index);
2040                 else
2041                         *skb = mlx5e_skb_from_cqe_mpwrq_nonlinear(rq, wi, cqe_bcnt, data_offset,
2042                                                                   page_idx);
2043                 if (unlikely(!*skb))
2044                         goto free_hd_entry;
2045
2046                 NAPI_GRO_CB(*skb)->count = 1;
2047                 skb_shinfo(*skb)->gso_size = cqe_bcnt - head_size;
2048         } else {
2049                 NAPI_GRO_CB(*skb)->count++;
2050                 if (NAPI_GRO_CB(*skb)->count == 2 &&
2051                     rq->hw_gro_data->fk.basic.n_proto == htons(ETH_P_IP)) {
2052                         void *hd_addr = mlx5e_shampo_get_packet_hd(rq, header_index);
2053                         int nhoff = ETH_HLEN + rq->hw_gro_data->fk.control.thoff -
2054                                     sizeof(struct iphdr);
2055                         struct iphdr *iph = (struct iphdr *)(hd_addr + nhoff);
2056
2057                         rq->hw_gro_data->second_ip_id = ntohs(iph->id);
2058                 }
2059         }
2060
2061         if (likely(head_size)) {
2062                 di = &wi->umr.dma_info[page_idx];
2063                 mlx5e_fill_skb_data(*skb, rq, di, data_bcnt, data_offset);
2064         }
2065
2066         mlx5e_shampo_complete_rx_cqe(rq, cqe, cqe_bcnt, *skb);
2067         if (flush)
2068                 mlx5e_shampo_flush_skb(rq, cqe, match);
2069 free_hd_entry:
2070         mlx5e_free_rx_shampo_hd_entry(rq, header_index);
2071 mpwrq_cqe_out:
2072         if (likely(wi->consumed_strides < rq->mpwqe.num_strides))
2073                 return;
2074
2075         wq  = &rq->mpwqe.wq;
2076         wqe = mlx5_wq_ll_get_wqe(wq, wqe_id);
2077         mlx5e_free_rx_mpwqe(rq, wi, true);
2078         mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
2079 }
2080
2081 static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
2082 {
2083         u16 cstrides       = mpwrq_get_cqe_consumed_strides(cqe);
2084         u16 wqe_id         = be16_to_cpu(cqe->wqe_id);
2085         struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id];
2086         u16 stride_ix      = mpwrq_get_cqe_stride_index(cqe);
2087         u32 wqe_offset     = stride_ix << rq->mpwqe.log_stride_sz;
2088         u32 head_offset    = wqe_offset & (PAGE_SIZE - 1);
2089         u32 page_idx       = wqe_offset >> PAGE_SHIFT;
2090         struct mlx5e_rx_wqe_ll *wqe;
2091         struct mlx5_wq_ll *wq;
2092         struct sk_buff *skb;
2093         u16 cqe_bcnt;
2094
2095         wi->consumed_strides += cstrides;
2096
2097         if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
2098                 mlx5e_handle_rx_err_cqe(rq, cqe);
2099                 goto mpwrq_cqe_out;
2100         }
2101
2102         if (unlikely(mpwrq_is_filler_cqe(cqe))) {
2103                 struct mlx5e_rq_stats *stats = rq->stats;
2104
2105                 stats->mpwqe_filler_cqes++;
2106                 stats->mpwqe_filler_strides += cstrides;
2107                 goto mpwrq_cqe_out;
2108         }
2109
2110         cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe);
2111
2112         skb = INDIRECT_CALL_2(rq->mpwqe.skb_from_cqe_mpwrq,
2113                               mlx5e_skb_from_cqe_mpwrq_linear,
2114                               mlx5e_skb_from_cqe_mpwrq_nonlinear,
2115                               rq, wi, cqe_bcnt, head_offset, page_idx);
2116         if (!skb)
2117                 goto mpwrq_cqe_out;
2118
2119         mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
2120
2121         if (mlx5e_cqe_regb_chain(cqe))
2122                 if (!mlx5e_tc_update_skb(cqe, skb)) {
2123                         dev_kfree_skb_any(skb);
2124                         goto mpwrq_cqe_out;
2125                 }
2126
2127         napi_gro_receive(rq->cq.napi, skb);
2128
2129 mpwrq_cqe_out:
2130         if (likely(wi->consumed_strides < rq->mpwqe.num_strides))
2131                 return;
2132
2133         wq  = &rq->mpwqe.wq;
2134         wqe = mlx5_wq_ll_get_wqe(wq, wqe_id);
2135         mlx5e_free_rx_mpwqe(rq, wi, true);
2136         mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
2137 }
2138
2139 int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
2140 {
2141         struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq);
2142         struct mlx5_cqwq *cqwq = &cq->wq;
2143         struct mlx5_cqe64 *cqe;
2144         int work_done = 0;
2145
2146         if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
2147                 return 0;
2148
2149         if (rq->cqd.left) {
2150                 work_done += mlx5e_decompress_cqes_cont(rq, cqwq, 0, budget);
2151                 if (work_done >= budget)
2152                         goto out;
2153         }
2154
2155         cqe = mlx5_cqwq_get_cqe(cqwq);
2156         if (!cqe) {
2157                 if (unlikely(work_done))
2158                         goto out;
2159                 return 0;
2160         }
2161
2162         do {
2163                 if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) {
2164                         work_done +=
2165                                 mlx5e_decompress_cqes_start(rq, cqwq,
2166                                                             budget - work_done);
2167                         continue;
2168                 }
2169
2170                 mlx5_cqwq_pop(cqwq);
2171
2172                 INDIRECT_CALL_3(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
2173                                 mlx5e_handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq_shampo,
2174                                 rq, cqe);
2175         } while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(cqwq)));
2176
2177 out:
2178         if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) && rq->hw_gro_data->skb)
2179                 mlx5e_shampo_flush_skb(rq, NULL, false);
2180
2181         if (rcu_access_pointer(rq->xdp_prog))
2182                 mlx5e_xdp_rx_poll_complete(rq);
2183
2184         mlx5_cqwq_update_db_record(cqwq);
2185
2186         /* ensure cq space is freed before enabling more cqes */
2187         wmb();
2188
2189         return work_done;
2190 }
2191
2192 #ifdef CONFIG_MLX5_CORE_IPOIB
2193
2194 #define MLX5_IB_GRH_SGID_OFFSET 8
2195 #define MLX5_IB_GRH_DGID_OFFSET 24
2196 #define MLX5_GID_SIZE           16
2197
2198 static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq,
2199                                          struct mlx5_cqe64 *cqe,
2200                                          u32 cqe_bcnt,
2201                                          struct sk_buff *skb)
2202 {
2203         struct hwtstamp_config *tstamp;
2204         struct mlx5e_rq_stats *stats;
2205         struct net_device *netdev;
2206         struct mlx5e_priv *priv;
2207         char *pseudo_header;
2208         u32 flags_rqpn;
2209         u32 qpn;
2210         u8 *dgid;
2211         u8 g;
2212
2213         qpn = be32_to_cpu(cqe->sop_drop_qpn) & 0xffffff;
2214         netdev = mlx5i_pkey_get_netdev(rq->netdev, qpn);
2215
2216         /* No mapping present, cannot process SKB. This might happen if a child
2217          * interface is going down while having unprocessed CQEs on parent RQ
2218          */
2219         if (unlikely(!netdev)) {
2220                 /* TODO: add drop counters support */
2221                 skb->dev = NULL;
2222                 pr_warn_once("Unable to map QPN %u to dev - dropping skb\n", qpn);
2223                 return;
2224         }
2225
2226         priv = mlx5i_epriv(netdev);
2227         tstamp = &priv->tstamp;
2228         stats = rq->stats;
2229
2230         flags_rqpn = be32_to_cpu(cqe->flags_rqpn);
2231         g = (flags_rqpn >> 28) & 3;
2232         dgid = skb->data + MLX5_IB_GRH_DGID_OFFSET;
2233         if ((!g) || dgid[0] != 0xff)
2234                 skb->pkt_type = PACKET_HOST;
2235         else if (memcmp(dgid, netdev->broadcast + 4, MLX5_GID_SIZE) == 0)
2236                 skb->pkt_type = PACKET_BROADCAST;
2237         else
2238                 skb->pkt_type = PACKET_MULTICAST;
2239
2240         /* Drop packets that this interface sent, ie multicast packets
2241          * that the HCA has replicated.
2242          */
2243         if (g && (qpn == (flags_rqpn & 0xffffff)) &&
2244             (memcmp(netdev->dev_addr + 4, skb->data + MLX5_IB_GRH_SGID_OFFSET,
2245                     MLX5_GID_SIZE) == 0)) {
2246                 skb->dev = NULL;
2247                 return;
2248         }
2249
2250         skb_pull(skb, MLX5_IB_GRH_BYTES);
2251
2252         skb->protocol = *((__be16 *)(skb->data));
2253
2254         if (netdev->features & NETIF_F_RXCSUM) {
2255                 skb->ip_summed = CHECKSUM_COMPLETE;
2256                 skb->csum = csum_unfold((__force __sum16)cqe->check_sum);
2257                 stats->csum_complete++;
2258         } else {
2259                 skb->ip_summed = CHECKSUM_NONE;
2260                 stats->csum_none++;
2261         }
2262
2263         if (unlikely(mlx5e_rx_hw_stamp(tstamp)))
2264                 skb_hwtstamps(skb)->hwtstamp = mlx5e_cqe_ts_to_ns(rq->ptp_cyc2time,
2265                                                                   rq->clock, get_cqe_ts(cqe));
2266         skb_record_rx_queue(skb, rq->ix);
2267
2268         if (likely(netdev->features & NETIF_F_RXHASH))
2269                 mlx5e_skb_set_hash(cqe, skb);
2270
2271         /* 20 bytes of ipoib header and 4 for encap existing */
2272         pseudo_header = skb_push(skb, MLX5_IPOIB_PSEUDO_LEN);
2273         memset(pseudo_header, 0, MLX5_IPOIB_PSEUDO_LEN);
2274         skb_reset_mac_header(skb);
2275         skb_pull(skb, MLX5_IPOIB_HARD_LEN);
2276
2277         skb->dev = netdev;
2278
2279         stats->packets++;
2280         stats->bytes += cqe_bcnt;
2281 }
2282
2283 static void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
2284 {
2285         struct mlx5_wq_cyc *wq = &rq->wqe.wq;
2286         struct mlx5e_wqe_frag_info *wi;
2287         struct sk_buff *skb;
2288         u32 cqe_bcnt;
2289         u16 ci;
2290
2291         ci       = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
2292         wi       = get_frag(rq, ci);
2293         cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
2294
2295         if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
2296                 rq->stats->wqe_err++;
2297                 goto wq_free_wqe;
2298         }
2299
2300         skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
2301                               mlx5e_skb_from_cqe_linear,
2302                               mlx5e_skb_from_cqe_nonlinear,
2303                               rq, cqe, wi, cqe_bcnt);
2304         if (!skb)
2305                 goto wq_free_wqe;
2306
2307         mlx5i_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
2308         if (unlikely(!skb->dev)) {
2309                 dev_kfree_skb_any(skb);
2310                 goto wq_free_wqe;
2311         }
2312         napi_gro_receive(rq->cq.napi, skb);
2313
2314 wq_free_wqe:
2315         mlx5e_free_rx_wqe(rq, wi, true);
2316         mlx5_wq_cyc_pop(wq);
2317 }
2318
2319 const struct mlx5e_rx_handlers mlx5i_rx_handlers = {
2320         .handle_rx_cqe       = mlx5i_handle_rx_cqe,
2321         .handle_rx_cqe_mpwqe = NULL, /* Not supported */
2322 };
2323 #endif /* CONFIG_MLX5_CORE_IPOIB */
2324
2325 #ifdef CONFIG_MLX5_EN_IPSEC
2326
2327 static void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
2328 {
2329         struct mlx5_wq_cyc *wq = &rq->wqe.wq;
2330         struct mlx5e_wqe_frag_info *wi;
2331         struct sk_buff *skb;
2332         u32 cqe_bcnt;
2333         u16 ci;
2334
2335         ci       = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
2336         wi       = get_frag(rq, ci);
2337         cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
2338
2339         if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
2340                 rq->stats->wqe_err++;
2341                 goto wq_free_wqe;
2342         }
2343
2344         skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
2345                               mlx5e_skb_from_cqe_linear,
2346                               mlx5e_skb_from_cqe_nonlinear,
2347                               rq, cqe, wi, cqe_bcnt);
2348         if (unlikely(!skb)) /* a DROP, save the page-reuse checks */
2349                 goto wq_free_wqe;
2350
2351         skb = mlx5e_ipsec_handle_rx_skb(rq->netdev, skb, &cqe_bcnt);
2352         if (unlikely(!skb))
2353                 goto wq_free_wqe;
2354
2355         mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
2356         napi_gro_receive(rq->cq.napi, skb);
2357
2358 wq_free_wqe:
2359         mlx5e_free_rx_wqe(rq, wi, true);
2360         mlx5_wq_cyc_pop(wq);
2361 }
2362
2363 #endif /* CONFIG_MLX5_EN_IPSEC */
2364
2365 int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool xsk)
2366 {
2367         struct net_device *netdev = rq->netdev;
2368         struct mlx5_core_dev *mdev = rq->mdev;
2369         struct mlx5e_priv *priv = rq->priv;
2370
2371         switch (rq->wq_type) {
2372         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
2373                 rq->mpwqe.skb_from_cqe_mpwrq = xsk ?
2374                         mlx5e_xsk_skb_from_cqe_mpwrq_linear :
2375                         mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ?
2376                                 mlx5e_skb_from_cqe_mpwrq_linear :
2377                                 mlx5e_skb_from_cqe_mpwrq_nonlinear;
2378                 rq->post_wqes = mlx5e_post_rx_mpwqes;
2379                 rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
2380
2381                 if (mlx5_fpga_is_ipsec_device(mdev)) {
2382                         netdev_err(netdev, "MPWQE RQ with Innova IPSec offload not supported\n");
2383                         return -EINVAL;
2384                 }
2385                 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) {
2386                         rq->handle_rx_cqe = priv->profile->rx_handlers->handle_rx_cqe_mpwqe_shampo;
2387                         if (!rq->handle_rx_cqe) {
2388                                 netdev_err(netdev, "RX handler of SHAMPO MPWQE RQ is not set\n");
2389                                 return -EINVAL;
2390                         }
2391                 } else {
2392                         rq->handle_rx_cqe = priv->profile->rx_handlers->handle_rx_cqe_mpwqe;
2393                         if (!rq->handle_rx_cqe) {
2394                                 netdev_err(netdev, "RX handler of MPWQE RQ is not set\n");
2395                                 return -EINVAL;
2396                         }
2397                 }
2398
2399                 break;
2400         default: /* MLX5_WQ_TYPE_CYCLIC */
2401                 rq->wqe.skb_from_cqe = xsk ?
2402                         mlx5e_xsk_skb_from_cqe_linear :
2403                         mlx5e_rx_is_linear_skb(params, NULL) ?
2404                                 mlx5e_skb_from_cqe_linear :
2405                                 mlx5e_skb_from_cqe_nonlinear;
2406                 rq->post_wqes = mlx5e_post_rx_wqes;
2407                 rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
2408
2409 #ifdef CONFIG_MLX5_EN_IPSEC
2410                 if ((mlx5_fpga_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) &&
2411                     priv->ipsec)
2412                         rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe;
2413                 else
2414 #endif
2415                         rq->handle_rx_cqe = priv->profile->rx_handlers->handle_rx_cqe;
2416                 if (!rq->handle_rx_cqe) {
2417                         netdev_err(netdev, "RX handler of RQ is not set\n");
2418                         return -EINVAL;
2419                 }
2420         }
2421
2422         return 0;
2423 }
2424
2425 static void mlx5e_trap_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
2426 {
2427         struct mlx5e_priv *priv = netdev_priv(rq->netdev);
2428         struct mlx5_wq_cyc *wq = &rq->wqe.wq;
2429         struct mlx5e_wqe_frag_info *wi;
2430         struct devlink_port *dl_port;
2431         struct sk_buff *skb;
2432         u32 cqe_bcnt;
2433         u16 trap_id;
2434         u16 ci;
2435
2436         trap_id  = get_cqe_flow_tag(cqe);
2437         ci       = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
2438         wi       = get_frag(rq, ci);
2439         cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
2440
2441         if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
2442                 rq->stats->wqe_err++;
2443                 goto free_wqe;
2444         }
2445
2446         skb = mlx5e_skb_from_cqe_nonlinear(rq, cqe, wi, cqe_bcnt);
2447         if (!skb)
2448                 goto free_wqe;
2449
2450         mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
2451         skb_push(skb, ETH_HLEN);
2452
2453         dl_port = mlx5e_devlink_get_dl_port(priv);
2454         mlx5_devlink_trap_report(rq->mdev, trap_id, skb, dl_port);
2455         dev_kfree_skb_any(skb);
2456
2457 free_wqe:
2458         mlx5e_free_rx_wqe(rq, wi, false);
2459         mlx5_wq_cyc_pop(wq);
2460 }
2461
2462 void mlx5e_rq_set_trap_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params)
2463 {
2464         rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(params, NULL) ?
2465                                mlx5e_skb_from_cqe_linear :
2466                                mlx5e_skb_from_cqe_nonlinear;
2467         rq->post_wqes = mlx5e_post_rx_wqes;
2468         rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
2469         rq->handle_rx_cqe = mlx5e_trap_handle_rx_cqe;
2470 }