drivers/net/ethernet/google/gve/gve_rx.c

   1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
   2 /* Google virtual Ethernet (gve) driver
   3  *
   4  * Copyright (C) 2015-2021 Google, Inc.
   5  */
   6
   7 #include "gve.h"
   8 #include "gve_adminq.h"
   9 #include "gve_utils.h"
  10 #include <linux/etherdevice.h>
  11
  12 static void gve_rx_free_buffer(struct device *dev,
  13                                struct gve_rx_slot_page_info *page_info,
  14                                union gve_rx_data_slot *data_slot)
  15 {
  16         dma_addr_t dma = (dma_addr_t)(be64_to_cpu(data_slot->addr) &
  17                                       GVE_DATA_SLOT_ADDR_PAGE_MASK);
  18
  19         page_ref_sub(page_info->page, page_info->pagecnt_bias - 1);
  20         gve_free_page(dev, page_info->page, dma, DMA_FROM_DEVICE);
  21 }
  22
  23 static void gve_rx_unfill_pages(struct gve_priv *priv, struct gve_rx_ring *rx)
  24 {
  25         u32 slots = rx->mask + 1;
  26         int i;
  27
  28         if (rx->data.raw_addressing) {
  29                 for (i = 0; i < slots; i++)
  30                         gve_rx_free_buffer(&priv->pdev->dev, &rx->data.page_info[i],
  31                                            &rx->data.data_ring[i]);
  32         } else {
  33                 for (i = 0; i < slots; i++)
  34                         page_ref_sub(rx->data.page_info[i].page,
  35                                      rx->data.page_info[i].pagecnt_bias - 1);
  36                 gve_unassign_qpl(priv, rx->data.qpl->id);
  37                 rx->data.qpl = NULL;
  38         }
  39         kvfree(rx->data.page_info);
  40         rx->data.page_info = NULL;
  41 }
  42
  43 static void gve_rx_free_ring(struct gve_priv *priv, int idx)
  44 {
  45         struct gve_rx_ring *rx = &priv->rx[idx];
  46         struct device *dev = &priv->pdev->dev;
  47         u32 slots = rx->mask + 1;
  48         size_t bytes;
  49
  50         gve_rx_remove_from_block(priv, idx);
  51
  52         bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
  53         dma_free_coherent(dev, bytes, rx->desc.desc_ring, rx->desc.bus);
  54         rx->desc.desc_ring = NULL;
  55
  56         dma_free_coherent(dev, sizeof(*rx->q_resources),
  57                           rx->q_resources, rx->q_resources_bus);
  58         rx->q_resources = NULL;
  59
  60         gve_rx_unfill_pages(priv, rx);
  61
  62         bytes = sizeof(*rx->data.data_ring) * slots;
  63         dma_free_coherent(dev, bytes, rx->data.data_ring,
  64                           rx->data.data_bus);
  65         rx->data.data_ring = NULL;
  66         netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx);
  67 }
  68
  69 static void gve_setup_rx_buffer(struct gve_rx_slot_page_info *page_info,
  70                              dma_addr_t addr, struct page *page, __be64 *slot_addr)
  71 {
  72         page_info->page = page;
  73         page_info->page_offset = 0;
  74         page_info->page_address = page_address(page);
  75         *slot_addr = cpu_to_be64(addr);
  76         /* The page already has 1 ref */
  77         page_ref_add(page, INT_MAX - 1);
  78         page_info->pagecnt_bias = INT_MAX;
  79 }
  80
  81 static int gve_rx_alloc_buffer(struct gve_priv *priv, struct device *dev,
  82                                struct gve_rx_slot_page_info *page_info,
  83                                union gve_rx_data_slot *data_slot)
  84 {
  85         struct page *page;
  86         dma_addr_t dma;
  87         int err;
  88
  89         err = gve_alloc_page(priv, dev, &page, &dma, DMA_FROM_DEVICE);
  90         if (err)
  91                 return err;
  92
  93         gve_setup_rx_buffer(page_info, dma, page, &data_slot->addr);
  94         return 0;
  95 }
  96
  97 static int gve_prefill_rx_pages(struct gve_rx_ring *rx)
  98 {
  99         struct gve_priv *priv = rx->gve;
 100         u32 slots;
 101         int err;
 102         int i;
 103
 104         /* Allocate one page per Rx queue slot. Each page is split into two
 105          * packet buffers, when possible we "page flip" between the two.
 106          */
 107         slots = rx->mask + 1;
 108
 109         rx->data.page_info = kvzalloc(slots *
 110                                       sizeof(*rx->data.page_info), GFP_KERNEL);
 111         if (!rx->data.page_info)
 112                 return -ENOMEM;
 113
 114         if (!rx->data.raw_addressing) {
 115                 rx->data.qpl = gve_assign_rx_qpl(priv);
 116                 if (!rx->data.qpl) {
 117                         kvfree(rx->data.page_info);
 118                         rx->data.page_info = NULL;
 119                         return -ENOMEM;
 120                 }
 121         }
 122         for (i = 0; i < slots; i++) {
 123                 if (!rx->data.raw_addressing) {
 124                         struct page *page = rx->data.qpl->pages[i];
 125                         dma_addr_t addr = i * PAGE_SIZE;
 126
 127                         gve_setup_rx_buffer(&rx->data.page_info[i], addr, page,
 128                                             &rx->data.data_ring[i].qpl_offset);
 129                         continue;
 130                 }
 131                 err = gve_rx_alloc_buffer(priv, &priv->pdev->dev, &rx->data.page_info[i],
 132                                           &rx->data.data_ring[i]);
 133                 if (err)
 134                         goto alloc_err;
 135         }
 136
 137         return slots;
 138 alloc_err:
 139         while (i--)
 140                 gve_rx_free_buffer(&priv->pdev->dev,
 141                                    &rx->data.page_info[i],
 142                                    &rx->data.data_ring[i]);
 143         return err;
 144 }
 145
 146 static void gve_rx_ctx_clear(struct gve_rx_ctx *ctx)
 147 {
 148         ctx->curr_frag_cnt = 0;
 149         ctx->total_expected_size = 0;
 150         ctx->expected_frag_cnt = 0;
 151         ctx->skb_head = NULL;
 152         ctx->skb_tail = NULL;
 153         ctx->reuse_frags = false;
 154 }
 155
 156 static int gve_rx_alloc_ring(struct gve_priv *priv, int idx)
 157 {
 158         struct gve_rx_ring *rx = &priv->rx[idx];
 159         struct device *hdev = &priv->pdev->dev;
 160         u32 slots, npages;
 161         int filled_pages;
 162         size_t bytes;
 163         int err;
 164
 165         netif_dbg(priv, drv, priv->dev, "allocating rx ring\n");
 166         /* Make sure everything is zeroed to start with */
 167         memset(rx, 0, sizeof(*rx));
 168
 169         rx->gve = priv;
 170         rx->q_num = idx;
 171
 172         slots = priv->rx_data_slot_cnt;
 173         rx->mask = slots - 1;
 174         rx->data.raw_addressing = priv->queue_format == GVE_GQI_RDA_FORMAT;
 175
 176         /* alloc rx data ring */
 177         bytes = sizeof(*rx->data.data_ring) * slots;
 178         rx->data.data_ring = dma_alloc_coherent(hdev, bytes,
 179                                                 &rx->data.data_bus,
 180                                                 GFP_KERNEL);
 181         if (!rx->data.data_ring)
 182                 return -ENOMEM;
 183         filled_pages = gve_prefill_rx_pages(rx);
 184         if (filled_pages < 0) {
 185                 err = -ENOMEM;
 186                 goto abort_with_slots;
 187         }
 188         rx->fill_cnt = filled_pages;
 189         /* Ensure data ring slots (packet buffers) are visible. */
 190         dma_wmb();
 191
 192         /* Alloc gve_queue_resources */
 193         rx->q_resources =
 194                 dma_alloc_coherent(hdev,
 195                                    sizeof(*rx->q_resources),
 196                                    &rx->q_resources_bus,
 197                                    GFP_KERNEL);
 198         if (!rx->q_resources) {
 199                 err = -ENOMEM;
 200                 goto abort_filled;
 201         }
 202         netif_dbg(priv, drv, priv->dev, "rx[%d]->data.data_bus=%lx\n", idx,
 203                   (unsigned long)rx->data.data_bus);
 204
 205         /* alloc rx desc ring */
 206         bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
 207         npages = bytes / PAGE_SIZE;
 208         if (npages * PAGE_SIZE != bytes) {
 209                 err = -EIO;
 210                 goto abort_with_q_resources;
 211         }
 212
 213         rx->desc.desc_ring = dma_alloc_coherent(hdev, bytes, &rx->desc.bus,
 214                                                 GFP_KERNEL);
 215         if (!rx->desc.desc_ring) {
 216                 err = -ENOMEM;
 217                 goto abort_with_q_resources;
 218         }
 219         rx->cnt = 0;
 220         rx->db_threshold = priv->rx_desc_cnt / 2;
 221         rx->desc.seqno = 1;
 222
 223         /* Allocating half-page buffers allows page-flipping which is faster
 224          * than copying or allocating new pages.
 225          */
 226         rx->packet_buffer_size = PAGE_SIZE / 2;
 227         gve_rx_ctx_clear(&rx->ctx);
 228         gve_rx_add_to_block(priv, idx);
 229
 230         return 0;
 231
 232 abort_with_q_resources:
 233         dma_free_coherent(hdev, sizeof(*rx->q_resources),
 234                           rx->q_resources, rx->q_resources_bus);
 235         rx->q_resources = NULL;
 236 abort_filled:
 237         gve_rx_unfill_pages(priv, rx);
 238 abort_with_slots:
 239         bytes = sizeof(*rx->data.data_ring) * slots;
 240         dma_free_coherent(hdev, bytes, rx->data.data_ring, rx->data.data_bus);
 241         rx->data.data_ring = NULL;
 242
 243         return err;
 244 }
 245
 246 int gve_rx_alloc_rings(struct gve_priv *priv)
 247 {
 248         int err = 0;
 249         int i;
 250
 251         for (i = 0; i < priv->rx_cfg.num_queues; i++) {
 252                 err = gve_rx_alloc_ring(priv, i);
 253                 if (err) {
 254                         netif_err(priv, drv, priv->dev,
 255                                   "Failed to alloc rx ring=%d: err=%d\n",
 256                                   i, err);
 257                         break;
 258                 }
 259         }
 260         /* Unallocate if there was an error */
 261         if (err) {
 262                 int j;
 263
 264                 for (j = 0; j < i; j++)
 265                         gve_rx_free_ring(priv, j);
 266         }
 267         return err;
 268 }
 269
 270 void gve_rx_free_rings_gqi(struct gve_priv *priv)
 271 {
 272         int i;
 273
 274         for (i = 0; i < priv->rx_cfg.num_queues; i++)
 275                 gve_rx_free_ring(priv, i);
 276 }
 277
 278 void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx)
 279 {
 280         u32 db_idx = be32_to_cpu(rx->q_resources->db_index);
 281
 282         iowrite32be(rx->fill_cnt, &priv->db_bar2[db_idx]);
 283 }
 284
 285 static enum pkt_hash_types gve_rss_type(__be16 pkt_flags)
 286 {
 287         if (likely(pkt_flags & (GVE_RXF_TCP | GVE_RXF_UDP)))
 288                 return PKT_HASH_TYPE_L4;
 289         if (pkt_flags & (GVE_RXF_IPV4 | GVE_RXF_IPV6))
 290                 return PKT_HASH_TYPE_L3;
 291         return PKT_HASH_TYPE_L2;
 292 }
 293
 294 static u16 gve_rx_ctx_padding(struct gve_rx_ctx *ctx)
 295 {
 296         return (ctx->curr_frag_cnt == 0) ? GVE_RX_PAD : 0;
 297 }
 298
 299 static struct sk_buff *gve_rx_add_frags(struct napi_struct *napi,
 300                                         struct gve_rx_slot_page_info *page_info,
 301                                         u16 packet_buffer_size, u16 len,
 302                                         struct gve_rx_ctx *ctx)
 303 {
 304         u32 offset = page_info->page_offset +  gve_rx_ctx_padding(ctx);
 305         struct sk_buff *skb;
 306
 307         if (!ctx->skb_head)
 308                 ctx->skb_head = napi_get_frags(napi);
 309
 310         if (unlikely(!ctx->skb_head))
 311                 return NULL;
 312
 313         skb = ctx->skb_head;
 314         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page_info->page,
 315                         offset, len, packet_buffer_size);
 316
 317         return skb;
 318 }
 319
 320 static void gve_rx_flip_buff(struct gve_rx_slot_page_info *page_info, __be64 *slot_addr)
 321 {
 322         const __be64 offset = cpu_to_be64(PAGE_SIZE / 2);
 323
 324         /* "flip" to other packet buffer on this page */
 325         page_info->page_offset ^= PAGE_SIZE / 2;
 326         *(slot_addr) ^= offset;
 327 }
 328
 329 static int gve_rx_can_recycle_buffer(struct gve_rx_slot_page_info *page_info)
 330 {
 331         int pagecount = page_count(page_info->page);
 332
 333         /* This page is not being used by any SKBs - reuse */
 334         if (pagecount == page_info->pagecnt_bias)
 335                 return 1;
 336         /* This page is still being used by an SKB - we can't reuse */
 337         else if (pagecount > page_info->pagecnt_bias)
 338                 return 0;
 339         WARN(pagecount < page_info->pagecnt_bias,
 340              "Pagecount should never be less than the bias.");
 341         return -1;
 342 }
 343
 344 static struct sk_buff *
 345 gve_rx_raw_addressing(struct device *dev, struct net_device *netdev,
 346                       struct gve_rx_slot_page_info *page_info, u16 len,
 347                       struct napi_struct *napi,
 348                       union gve_rx_data_slot *data_slot,
 349                       u16 packet_buffer_size, struct gve_rx_ctx *ctx)
 350 {
 351         struct sk_buff *skb = gve_rx_add_frags(napi, page_info, packet_buffer_size, len, ctx);
 352
 353         if (!skb)
 354                 return NULL;
 355
 356         /* Optimistically stop the kernel from freeing the page.
 357          * We will check again in refill to determine if we need to alloc a
 358          * new page.
 359          */
 360         gve_dec_pagecnt_bias(page_info);
 361
 362         return skb;
 363 }
 364
 365 static struct sk_buff *
 366 gve_rx_qpl(struct device *dev, struct net_device *netdev,
 367            struct gve_rx_ring *rx, struct gve_rx_slot_page_info *page_info,
 368            u16 len, struct napi_struct *napi,
 369            union gve_rx_data_slot *data_slot)
 370 {
 371         struct gve_rx_ctx *ctx = &rx->ctx;
 372         struct sk_buff *skb;
 373
 374         /* if raw_addressing mode is not enabled gvnic can only receive into
 375          * registered segments. If the buffer can't be recycled, our only
 376          * choice is to copy the data out of it so that we can return it to the
 377          * device.
 378          */
 379         if (ctx->reuse_frags) {
 380                 skb = gve_rx_add_frags(napi, page_info, rx->packet_buffer_size, len, ctx);
 381                 /* No point in recycling if we didn't get the skb */
 382                 if (skb) {
 383                         /* Make sure that the page isn't freed. */
 384                         gve_dec_pagecnt_bias(page_info);
 385                         gve_rx_flip_buff(page_info, &data_slot->qpl_offset);
 386                 }
 387         } else {
 388                 const u16 padding = gve_rx_ctx_padding(ctx);
 389
 390                 skb = gve_rx_copy(netdev, napi, page_info, len, padding, ctx);
 391                 if (skb) {
 392                         u64_stats_update_begin(&rx->statss);
 393                         rx->rx_frag_copy_cnt++;
 394                         u64_stats_update_end(&rx->statss);
 395                 }
 396         }
 397         return skb;
 398 }
 399
 400 #define GVE_PKTCONT_BIT_IS_SET(x) (GVE_RXF_PKT_CONT & (x))
 401 static u16 gve_rx_get_fragment_size(struct gve_rx_ctx *ctx, struct gve_rx_desc *desc)
 402 {
 403         return be16_to_cpu(desc->len) - gve_rx_ctx_padding(ctx);
 404 }
 405
 406 static bool gve_rx_ctx_init(struct gve_rx_ctx *ctx, struct gve_rx_ring *rx)
 407 {
 408         bool qpl_mode = !rx->data.raw_addressing, packet_size_error = false;
 409         bool buffer_error = false, desc_error = false, seqno_error = false;
 410         struct gve_rx_slot_page_info *page_info;
 411         struct gve_priv *priv = rx->gve;
 412         u32 idx = rx->cnt & rx->mask;
 413         bool reuse_frags, can_flip;
 414         struct gve_rx_desc *desc;
 415         u16 packet_size = 0;
 416         u16 n_frags = 0;
 417         int recycle;
 418
 419         /** In QPL mode, we only flip buffers when all buffers containing the packet
 420          * can be flipped. RDA can_flip decisions will be made later, per frag.
 421          */
 422         can_flip = qpl_mode;
 423         reuse_frags = can_flip;
 424         do {
 425                 u16 frag_size;
 426
 427                 n_frags++;
 428                 desc = &rx->desc.desc_ring[idx];
 429                 desc_error = unlikely(desc->flags_seq & GVE_RXF_ERR) || desc_error;
 430                 if (GVE_SEQNO(desc->flags_seq) != rx->desc.seqno) {
 431                         seqno_error = true;
 432                         netdev_warn(priv->dev,
 433                                     "RX seqno error: want=%d, got=%d, dropping packet and scheduling reset.",
 434                                     rx->desc.seqno, GVE_SEQNO(desc->flags_seq));
 435                 }
 436                 frag_size = be16_to_cpu(desc->len);
 437                 packet_size += frag_size;
 438                 if (frag_size > rx->packet_buffer_size) {
 439                         packet_size_error = true;
 440                         netdev_warn(priv->dev,
 441                                     "RX fragment error: packet_buffer_size=%d, frag_size=%d, droping packet.",
 442                                     rx->packet_buffer_size, be16_to_cpu(desc->len));
 443                 }
 444                 page_info = &rx->data.page_info[idx];
 445                 if (can_flip) {
 446                         recycle = gve_rx_can_recycle_buffer(page_info);
 447                         reuse_frags = reuse_frags && recycle > 0;
 448                         buffer_error = buffer_error || unlikely(recycle < 0);
 449                 }
 450                 idx = (idx + 1) & rx->mask;
 451                 rx->desc.seqno = gve_next_seqno(rx->desc.seqno);
 452         } while (GVE_PKTCONT_BIT_IS_SET(desc->flags_seq));
 453
 454         prefetch(rx->desc.desc_ring + idx);
 455
 456         ctx->curr_frag_cnt = 0;
 457         ctx->total_expected_size = packet_size - GVE_RX_PAD;
 458         ctx->expected_frag_cnt = n_frags;
 459         ctx->skb_head = NULL;
 460         ctx->reuse_frags = reuse_frags;
 461
 462         if (ctx->expected_frag_cnt > 1) {
 463                 u64_stats_update_begin(&rx->statss);
 464                 rx->rx_cont_packet_cnt++;
 465                 u64_stats_update_end(&rx->statss);
 466         }
 467         if (ctx->total_expected_size > priv->rx_copybreak && !ctx->reuse_frags && qpl_mode) {
 468                 u64_stats_update_begin(&rx->statss);
 469                 rx->rx_copied_pkt++;
 470                 u64_stats_update_end(&rx->statss);
 471         }
 472
 473         if (unlikely(buffer_error || seqno_error || packet_size_error)) {
 474                 gve_schedule_reset(priv);
 475                 return false;
 476         }
 477
 478         if (unlikely(desc_error)) {
 479                 u64_stats_update_begin(&rx->statss);
 480                 rx->rx_desc_err_dropped_pkt++;
 481                 u64_stats_update_end(&rx->statss);
 482                 return false;
 483         }
 484         return true;
 485 }
 486
 487 static struct sk_buff *gve_rx_skb(struct gve_priv *priv, struct gve_rx_ring *rx,
 488                                   struct gve_rx_slot_page_info *page_info, struct napi_struct *napi,
 489                                   u16 len, union gve_rx_data_slot *data_slot)
 490 {
 491         struct net_device *netdev = priv->dev;
 492         struct gve_rx_ctx *ctx = &rx->ctx;
 493         struct sk_buff *skb = NULL;
 494
 495         if (len <= priv->rx_copybreak && ctx->expected_frag_cnt == 1) {
 496                 /* Just copy small packets */
 497                 skb = gve_rx_copy(netdev, napi, page_info, len, GVE_RX_PAD, ctx);
 498                 if (skb) {
 499                         u64_stats_update_begin(&rx->statss);
 500                         rx->rx_copied_pkt++;
 501                         rx->rx_frag_copy_cnt++;
 502                         rx->rx_copybreak_pkt++;
 503                         u64_stats_update_end(&rx->statss);
 504                 }
 505         } else {
 506                 if (rx->data.raw_addressing) {
 507                         int recycle = gve_rx_can_recycle_buffer(page_info);
 508
 509                         if (unlikely(recycle < 0)) {
 510                                 gve_schedule_reset(priv);
 511                                 return NULL;
 512                         }
 513                         page_info->can_flip = recycle;
 514                         if (page_info->can_flip) {
 515                                 u64_stats_update_begin(&rx->statss);
 516                                 rx->rx_frag_flip_cnt++;
 517                                 u64_stats_update_end(&rx->statss);
 518                         }
 519                         skb = gve_rx_raw_addressing(&priv->pdev->dev, netdev,
 520                                                     page_info, len, napi,
 521                                                     data_slot,
 522                                                     rx->packet_buffer_size, ctx);
 523                 } else {
 524                         if (ctx->reuse_frags) {
 525                                 u64_stats_update_begin(&rx->statss);
 526                                 rx->rx_frag_flip_cnt++;
 527                                 u64_stats_update_end(&rx->statss);
 528                         }
 529                         skb = gve_rx_qpl(&priv->pdev->dev, netdev, rx,
 530                                          page_info, len, napi, data_slot);
 531                 }
 532         }
 533         return skb;
 534 }
 535
 536 static bool gve_rx(struct gve_rx_ring *rx, netdev_features_t feat,
 537                    u64 *packet_size_bytes, u32 *work_done)
 538 {
 539         struct gve_rx_slot_page_info *page_info;
 540         struct gve_rx_ctx *ctx = &rx->ctx;
 541         union gve_rx_data_slot *data_slot;
 542         struct gve_priv *priv = rx->gve;
 543         struct gve_rx_desc *first_desc;
 544         struct sk_buff *skb = NULL;
 545         struct gve_rx_desc *desc;
 546         struct napi_struct *napi;
 547         dma_addr_t page_bus;
 548         u32 work_cnt = 0;
 549         void *va;
 550         u32 idx;
 551         u16 len;
 552
 553         idx = rx->cnt & rx->mask;
 554         first_desc = &rx->desc.desc_ring[idx];
 555         desc = first_desc;
 556         napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
 557
 558         if (unlikely(!gve_rx_ctx_init(ctx, rx)))
 559                 goto skb_alloc_fail;
 560
 561         while (ctx->curr_frag_cnt < ctx->expected_frag_cnt) {
 562                 /* Prefetch two packet buffers ahead, we will need it soon. */
 563                 page_info = &rx->data.page_info[(idx + 2) & rx->mask];
 564                 va = page_info->page_address + page_info->page_offset;
 565
 566                 prefetch(page_info->page); /* Kernel page struct. */
 567                 prefetch(va);              /* Packet header. */
 568                 prefetch(va + 64);         /* Next cacheline too. */
 569
 570                 len = gve_rx_get_fragment_size(ctx, desc);
 571
 572                 page_info = &rx->data.page_info[idx];
 573                 data_slot = &rx->data.data_ring[idx];
 574                 page_bus = rx->data.raw_addressing ?
 575                            be64_to_cpu(data_slot->addr) - page_info->page_offset :
 576                            rx->data.qpl->page_buses[idx];
 577                 dma_sync_single_for_cpu(&priv->pdev->dev, page_bus, PAGE_SIZE, DMA_FROM_DEVICE);
 578
 579                 skb = gve_rx_skb(priv, rx, page_info, napi, len, data_slot);
 580                 if (!skb) {
 581                         u64_stats_update_begin(&rx->statss);
 582                         rx->rx_skb_alloc_fail++;
 583                         u64_stats_update_end(&rx->statss);
 584                         goto skb_alloc_fail;
 585                 }
 586
 587                 ctx->curr_frag_cnt++;
 588                 rx->cnt++;
 589                 idx = rx->cnt & rx->mask;
 590                 work_cnt++;
 591                 desc = &rx->desc.desc_ring[idx];
 592         }
 593
 594         if (likely(feat & NETIF_F_RXCSUM)) {
 595                 /* NIC passes up the partial sum */
 596                 if (first_desc->csum)
 597                         skb->ip_summed = CHECKSUM_COMPLETE;
 598                 else
 599                         skb->ip_summed = CHECKSUM_NONE;
 600                 skb->csum = csum_unfold(first_desc->csum);
 601         }
 602
 603         /* parse flags & pass relevant info up */
 604         if (likely(feat & NETIF_F_RXHASH) &&
 605             gve_needs_rss(first_desc->flags_seq))
 606                 skb_set_hash(skb, be32_to_cpu(first_desc->rss_hash),
 607                              gve_rss_type(first_desc->flags_seq));
 608
 609         *packet_size_bytes = skb->len + (skb->protocol ? ETH_HLEN : 0);
 610         *work_done = work_cnt;
 611         if (skb_is_nonlinear(skb))
 612                 napi_gro_frags(napi);
 613         else
 614                 napi_gro_receive(napi, skb);
 615
 616         gve_rx_ctx_clear(ctx);
 617         return true;
 618
 619 skb_alloc_fail:
 620         if (napi->skb)
 621                 napi_free_frags(napi);
 622         *packet_size_bytes = 0;
 623         *work_done = ctx->expected_frag_cnt;
 624         while (ctx->curr_frag_cnt < ctx->expected_frag_cnt) {
 625                 rx->cnt++;
 626                 ctx->curr_frag_cnt++;
 627         }
 628         gve_rx_ctx_clear(ctx);
 629         return false;
 630 }
 631
 632 bool gve_rx_work_pending(struct gve_rx_ring *rx)
 633 {
 634         struct gve_rx_desc *desc;
 635         __be16 flags_seq;
 636         u32 next_idx;
 637
 638         next_idx = rx->cnt & rx->mask;
 639         desc = rx->desc.desc_ring + next_idx;
 640
 641         flags_seq = desc->flags_seq;
 642
 643         return (GVE_SEQNO(flags_seq) == rx->desc.seqno);
 644 }
 645
 646 static bool gve_rx_refill_buffers(struct gve_priv *priv, struct gve_rx_ring *rx)
 647 {
 648         int refill_target = rx->mask + 1;
 649         u32 fill_cnt = rx->fill_cnt;
 650
 651         while (fill_cnt - rx->cnt < refill_target) {
 652                 struct gve_rx_slot_page_info *page_info;
 653                 u32 idx = fill_cnt & rx->mask;
 654
 655                 page_info = &rx->data.page_info[idx];
 656                 if (page_info->can_flip) {
 657                         /* The other half of the page is free because it was
 658                          * free when we processed the descriptor. Flip to it.
 659                          */
 660                         union gve_rx_data_slot *data_slot =
 661                                                 &rx->data.data_ring[idx];
 662
 663                         gve_rx_flip_buff(page_info, &data_slot->addr);
 664                         page_info->can_flip = 0;
 665                 } else {
 666                         /* It is possible that the networking stack has already
 667                          * finished processing all outstanding packets in the buffer
 668                          * and it can be reused.
 669                          * Flipping is unnecessary here - if the networking stack still
 670                          * owns half the page it is impossible to tell which half. Either
 671                          * the whole page is free or it needs to be replaced.
 672                          */
 673                         int recycle = gve_rx_can_recycle_buffer(page_info);
 674
 675                         if (recycle < 0) {
 676                                 if (!rx->data.raw_addressing)
 677                                         gve_schedule_reset(priv);
 678                                 return false;
 679                         }
 680                         if (!recycle) {
 681                                 /* We can't reuse the buffer - alloc a new one*/
 682                                 union gve_rx_data_slot *data_slot =
 683                                                 &rx->data.data_ring[idx];
 684                                 struct device *dev = &priv->pdev->dev;
 685                                 gve_rx_free_buffer(dev, page_info, data_slot);
 686                                 page_info->page = NULL;
 687                                 if (gve_rx_alloc_buffer(priv, dev, page_info,
 688                                                         data_slot)) {
 689                                         u64_stats_update_begin(&rx->statss);
 690                                         rx->rx_buf_alloc_fail++;
 691                                         u64_stats_update_end(&rx->statss);
 692                                         break;
 693                                 }
 694                         }
 695                 }
 696                 fill_cnt++;
 697         }
 698         rx->fill_cnt = fill_cnt;
 699         return true;
 700 }
 701
 702 static int gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
 703                              netdev_features_t feat)
 704 {
 705         u32 work_done = 0, total_packet_cnt = 0, ok_packet_cnt = 0;
 706         struct gve_priv *priv = rx->gve;
 707         u32 idx = rx->cnt & rx->mask;
 708         struct gve_rx_desc *desc;
 709         u64 bytes = 0;
 710
 711         desc = &rx->desc.desc_ring[idx];
 712         while ((GVE_SEQNO(desc->flags_seq) == rx->desc.seqno) &&
 713                work_done < budget) {
 714                 u64 packet_size_bytes = 0;
 715                 u32 work_cnt = 0;
 716                 bool dropped;
 717
 718                 netif_info(priv, rx_status, priv->dev,
 719                            "[%d] idx=%d desc=%p desc->flags_seq=0x%x\n",
 720                            rx->q_num, idx, desc, desc->flags_seq);
 721                 netif_info(priv, rx_status, priv->dev,
 722                            "[%d] seqno=%d rx->desc.seqno=%d\n",
 723                            rx->q_num, GVE_SEQNO(desc->flags_seq),
 724                            rx->desc.seqno);
 725
 726                 dropped = !gve_rx(rx, feat, &packet_size_bytes, &work_cnt);
 727                 if (!dropped) {
 728                         bytes += packet_size_bytes;
 729                         ok_packet_cnt++;
 730                 }
 731                 total_packet_cnt++;
 732                 idx = rx->cnt & rx->mask;
 733                 desc = &rx->desc.desc_ring[idx];
 734                 work_done += work_cnt;
 735         }
 736
 737         if (!work_done && rx->fill_cnt - rx->cnt > rx->db_threshold)
 738                 return 0;
 739
 740         if (work_done) {
 741                 u64_stats_update_begin(&rx->statss);
 742                 rx->rpackets += ok_packet_cnt;
 743                 rx->rbytes += bytes;
 744                 u64_stats_update_end(&rx->statss);
 745         }
 746
 747         /* restock ring slots */
 748         if (!rx->data.raw_addressing) {
 749                 /* In QPL mode buffs are refilled as the desc are processed */
 750                 rx->fill_cnt += work_done;
 751         } else if (rx->fill_cnt - rx->cnt <= rx->db_threshold) {
 752                 /* In raw addressing mode buffs are only refilled if the avail
 753                  * falls below a threshold.
 754                  */
 755                 if (!gve_rx_refill_buffers(priv, rx))
 756                         return 0;
 757
 758                 /* If we were not able to completely refill buffers, we'll want
 759                  * to schedule this queue for work again to refill buffers.
 760                  */
 761                 if (rx->fill_cnt - rx->cnt <= rx->db_threshold) {
 762                         gve_rx_write_doorbell(priv, rx);
 763                         return budget;
 764                 }
 765         }
 766
 767         gve_rx_write_doorbell(priv, rx);
 768         return total_packet_cnt;
 769 }
 770
 771 int gve_rx_poll(struct gve_notify_block *block, int budget)
 772 {
 773         struct gve_rx_ring *rx = block->rx;
 774         netdev_features_t feat;
 775         int work_done = 0;
 776
 777         feat = block->napi.dev->features;
 778
 779         /* If budget is 0, do all the work */
 780         if (budget == 0)
 781                 budget = INT_MAX;
 782
 783         if (budget > 0)
 784                 work_done = gve_clean_rx_done(rx, budget, feat);
 785
 786         return work_done;
 787 }