Merge tag 'icc-6.7-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc...
[linux-2.6-microblaze.git] / drivers / net / ethernet / aquantia / atlantic / aq_ring.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Atlantic Network Driver
3  *
4  * Copyright (C) 2014-2019 aQuantia Corporation
5  * Copyright (C) 2019-2020 Marvell International Ltd.
6  */
7
8 /* File aq_ring.c: Definition of functions for Rx/Tx rings. */
9
10 #include "aq_nic.h"
11 #include "aq_hw.h"
12 #include "aq_hw_utils.h"
13 #include "aq_ptp.h"
14 #include "aq_vec.h"
15 #include "aq_main.h"
16
17 #include <net/xdp.h>
18 #include <linux/filter.h>
19 #include <linux/bpf_trace.h>
20 #include <linux/netdevice.h>
21 #include <linux/etherdevice.h>
22
23 static void aq_get_rxpages_xdp(struct aq_ring_buff_s *buff,
24                                struct xdp_buff *xdp)
25 {
26         struct skb_shared_info *sinfo;
27         int i;
28
29         if (xdp_buff_has_frags(xdp)) {
30                 sinfo = xdp_get_shared_info_from_buff(xdp);
31
32                 for (i = 0; i < sinfo->nr_frags; i++) {
33                         skb_frag_t *frag = &sinfo->frags[i];
34
35                         page_ref_inc(skb_frag_page(frag));
36                 }
37         }
38         page_ref_inc(buff->rxdata.page);
39 }
40
41 static inline void aq_free_rxpage(struct aq_rxpage *rxpage, struct device *dev)
42 {
43         unsigned int len = PAGE_SIZE << rxpage->order;
44
45         dma_unmap_page(dev, rxpage->daddr, len, DMA_FROM_DEVICE);
46
47         /* Drop the ref for being in the ring. */
48         __free_pages(rxpage->page, rxpage->order);
49         rxpage->page = NULL;
50 }
51
52 static int aq_alloc_rxpages(struct aq_rxpage *rxpage, struct aq_ring_s *rx_ring)
53 {
54         struct device *dev = aq_nic_get_dev(rx_ring->aq_nic);
55         unsigned int order = rx_ring->page_order;
56         struct page *page;
57         int ret = -ENOMEM;
58         dma_addr_t daddr;
59
60         page = dev_alloc_pages(order);
61         if (unlikely(!page))
62                 goto err_exit;
63
64         daddr = dma_map_page(dev, page, 0, PAGE_SIZE << order,
65                              DMA_FROM_DEVICE);
66
67         if (unlikely(dma_mapping_error(dev, daddr)))
68                 goto free_page;
69
70         rxpage->page = page;
71         rxpage->daddr = daddr;
72         rxpage->order = order;
73         rxpage->pg_off = rx_ring->page_offset;
74
75         return 0;
76
77 free_page:
78         __free_pages(page, order);
79
80 err_exit:
81         return ret;
82 }
83
84 static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf)
85 {
86         unsigned int order = self->page_order;
87         u16 page_offset = self->page_offset;
88         u16 frame_max = self->frame_max;
89         u16 tail_size = self->tail_size;
90         int ret;
91
92         if (rxbuf->rxdata.page) {
93                 /* One means ring is the only user and can reuse */
94                 if (page_ref_count(rxbuf->rxdata.page) > 1) {
95                         /* Try reuse buffer */
96                         rxbuf->rxdata.pg_off += frame_max + page_offset +
97                                                 tail_size;
98                         if (rxbuf->rxdata.pg_off + frame_max + tail_size <=
99                             (PAGE_SIZE << order)) {
100                                 u64_stats_update_begin(&self->stats.rx.syncp);
101                                 self->stats.rx.pg_flips++;
102                                 u64_stats_update_end(&self->stats.rx.syncp);
103
104                         } else {
105                                 /* Buffer exhausted. We have other users and
106                                  * should release this page and realloc
107                                  */
108                                 aq_free_rxpage(&rxbuf->rxdata,
109                                                aq_nic_get_dev(self->aq_nic));
110                                 u64_stats_update_begin(&self->stats.rx.syncp);
111                                 self->stats.rx.pg_losts++;
112                                 u64_stats_update_end(&self->stats.rx.syncp);
113                         }
114                 } else {
115                         rxbuf->rxdata.pg_off = page_offset;
116                         u64_stats_update_begin(&self->stats.rx.syncp);
117                         self->stats.rx.pg_reuses++;
118                         u64_stats_update_end(&self->stats.rx.syncp);
119                 }
120         }
121
122         if (!rxbuf->rxdata.page) {
123                 ret = aq_alloc_rxpages(&rxbuf->rxdata, self);
124                 if (ret) {
125                         u64_stats_update_begin(&self->stats.rx.syncp);
126                         self->stats.rx.alloc_fails++;
127                         u64_stats_update_end(&self->stats.rx.syncp);
128                 }
129                 return ret;
130         }
131
132         return 0;
133 }
134
135 static struct aq_ring_s *aq_ring_alloc(struct aq_ring_s *self,
136                                        struct aq_nic_s *aq_nic)
137 {
138         int err = 0;
139
140         self->buff_ring =
141                 kcalloc(self->size, sizeof(struct aq_ring_buff_s), GFP_KERNEL);
142
143         if (!self->buff_ring) {
144                 err = -ENOMEM;
145                 goto err_exit;
146         }
147
148         self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic),
149                                            self->size * self->dx_size,
150                                            &self->dx_ring_pa, GFP_KERNEL);
151         if (!self->dx_ring) {
152                 err = -ENOMEM;
153                 goto err_exit;
154         }
155
156 err_exit:
157         if (err < 0) {
158                 aq_ring_free(self);
159                 self = NULL;
160         }
161
162         return self;
163 }
164
165 struct aq_ring_s *aq_ring_tx_alloc(struct aq_ring_s *self,
166                                    struct aq_nic_s *aq_nic,
167                                    unsigned int idx,
168                                    struct aq_nic_cfg_s *aq_nic_cfg)
169 {
170         int err = 0;
171
172         self->aq_nic = aq_nic;
173         self->idx = idx;
174         self->size = aq_nic_cfg->txds;
175         self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size;
176
177         self = aq_ring_alloc(self, aq_nic);
178         if (!self) {
179                 err = -ENOMEM;
180                 goto err_exit;
181         }
182
183 err_exit:
184         if (err < 0) {
185                 aq_ring_free(self);
186                 self = NULL;
187         }
188
189         return self;
190 }
191
192 struct aq_ring_s *aq_ring_rx_alloc(struct aq_ring_s *self,
193                                    struct aq_nic_s *aq_nic,
194                                    unsigned int idx,
195                                    struct aq_nic_cfg_s *aq_nic_cfg)
196 {
197         int err = 0;
198
199         self->aq_nic = aq_nic;
200         self->idx = idx;
201         self->size = aq_nic_cfg->rxds;
202         self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size;
203         self->xdp_prog = aq_nic->xdp_prog;
204         self->frame_max = AQ_CFG_RX_FRAME_MAX;
205
206         /* Only order-2 is allowed if XDP is enabled */
207         if (READ_ONCE(self->xdp_prog)) {
208                 self->page_offset = AQ_XDP_HEADROOM;
209                 self->page_order = AQ_CFG_XDP_PAGEORDER;
210                 self->tail_size = AQ_XDP_TAILROOM;
211         } else {
212                 self->page_offset = 0;
213                 self->page_order = fls(self->frame_max / PAGE_SIZE +
214                                        (self->frame_max % PAGE_SIZE ? 1 : 0)) - 1;
215                 if (aq_nic_cfg->rxpageorder > self->page_order)
216                         self->page_order = aq_nic_cfg->rxpageorder;
217                 self->tail_size = 0;
218         }
219
220         self = aq_ring_alloc(self, aq_nic);
221         if (!self) {
222                 err = -ENOMEM;
223                 goto err_exit;
224         }
225
226 err_exit:
227         if (err < 0) {
228                 aq_ring_free(self);
229                 self = NULL;
230         }
231
232         return self;
233 }
234
235 struct aq_ring_s *
236 aq_ring_hwts_rx_alloc(struct aq_ring_s *self, struct aq_nic_s *aq_nic,
237                       unsigned int idx, unsigned int size, unsigned int dx_size)
238 {
239         struct device *dev = aq_nic_get_dev(aq_nic);
240         size_t sz = size * dx_size + AQ_CFG_RXDS_DEF;
241
242         memset(self, 0, sizeof(*self));
243
244         self->aq_nic = aq_nic;
245         self->idx = idx;
246         self->size = size;
247         self->dx_size = dx_size;
248
249         self->dx_ring = dma_alloc_coherent(dev, sz, &self->dx_ring_pa,
250                                            GFP_KERNEL);
251         if (!self->dx_ring) {
252                 aq_ring_free(self);
253                 return NULL;
254         }
255
256         return self;
257 }
258
259 int aq_ring_init(struct aq_ring_s *self, const enum atl_ring_type ring_type)
260 {
261         self->hw_head = 0;
262         self->sw_head = 0;
263         self->sw_tail = 0;
264         self->ring_type = ring_type;
265
266         if (self->ring_type == ATL_RING_RX)
267                 u64_stats_init(&self->stats.rx.syncp);
268         else
269                 u64_stats_init(&self->stats.tx.syncp);
270
271         return 0;
272 }
273
274 static inline bool aq_ring_dx_in_range(unsigned int h, unsigned int i,
275                                        unsigned int t)
276 {
277         return (h < t) ? ((h < i) && (i < t)) : ((h < i) || (i < t));
278 }
279
280 void aq_ring_update_queue_state(struct aq_ring_s *ring)
281 {
282         if (aq_ring_avail_dx(ring) <= AQ_CFG_SKB_FRAGS_MAX)
283                 aq_ring_queue_stop(ring);
284         else if (aq_ring_avail_dx(ring) > AQ_CFG_RESTART_DESC_THRES)
285                 aq_ring_queue_wake(ring);
286 }
287
288 void aq_ring_queue_wake(struct aq_ring_s *ring)
289 {
290         struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic);
291
292         if (__netif_subqueue_stopped(ndev,
293                                      AQ_NIC_RING2QMAP(ring->aq_nic,
294                                                       ring->idx))) {
295                 netif_wake_subqueue(ndev,
296                                     AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx));
297                 u64_stats_update_begin(&ring->stats.tx.syncp);
298                 ring->stats.tx.queue_restarts++;
299                 u64_stats_update_end(&ring->stats.tx.syncp);
300         }
301 }
302
303 void aq_ring_queue_stop(struct aq_ring_s *ring)
304 {
305         struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic);
306
307         if (!__netif_subqueue_stopped(ndev,
308                                       AQ_NIC_RING2QMAP(ring->aq_nic,
309                                                        ring->idx)))
310                 netif_stop_subqueue(ndev,
311                                     AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx));
312 }
313
314 bool aq_ring_tx_clean(struct aq_ring_s *self)
315 {
316         struct device *dev = aq_nic_get_dev(self->aq_nic);
317         unsigned int budget;
318
319         for (budget = AQ_CFG_TX_CLEAN_BUDGET;
320              budget && self->sw_head != self->hw_head; budget--) {
321                 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];
322
323                 if (likely(buff->is_mapped)) {
324                         if (unlikely(buff->is_sop)) {
325                                 if (!buff->is_eop &&
326                                     buff->eop_index != 0xffffU &&
327                                     (!aq_ring_dx_in_range(self->sw_head,
328                                                 buff->eop_index,
329                                                 self->hw_head)))
330                                         break;
331
332                                 dma_unmap_single(dev, buff->pa, buff->len,
333                                                  DMA_TO_DEVICE);
334                         } else {
335                                 dma_unmap_page(dev, buff->pa, buff->len,
336                                                DMA_TO_DEVICE);
337                         }
338                 }
339
340                 if (likely(!buff->is_eop))
341                         goto out;
342
343                 if (buff->skb) {
344                         u64_stats_update_begin(&self->stats.tx.syncp);
345                         ++self->stats.tx.packets;
346                         self->stats.tx.bytes += buff->skb->len;
347                         u64_stats_update_end(&self->stats.tx.syncp);
348                         dev_kfree_skb_any(buff->skb);
349                 } else if (buff->xdpf) {
350                         u64_stats_update_begin(&self->stats.tx.syncp);
351                         ++self->stats.tx.packets;
352                         self->stats.tx.bytes += xdp_get_frame_len(buff->xdpf);
353                         u64_stats_update_end(&self->stats.tx.syncp);
354                         xdp_return_frame_rx_napi(buff->xdpf);
355                 }
356
357 out:
358                 buff->skb = NULL;
359                 buff->xdpf = NULL;
360                 buff->pa = 0U;
361                 buff->eop_index = 0xffffU;
362                 self->sw_head = aq_ring_next_dx(self, self->sw_head);
363         }
364
365         return !!budget;
366 }
367
368 static void aq_rx_checksum(struct aq_ring_s *self,
369                            struct aq_ring_buff_s *buff,
370                            struct sk_buff *skb)
371 {
372         if (!(self->aq_nic->ndev->features & NETIF_F_RXCSUM))
373                 return;
374
375         if (unlikely(buff->is_cso_err)) {
376                 u64_stats_update_begin(&self->stats.rx.syncp);
377                 ++self->stats.rx.errors;
378                 u64_stats_update_end(&self->stats.rx.syncp);
379                 skb->ip_summed = CHECKSUM_NONE;
380                 return;
381         }
382         if (buff->is_ip_cso) {
383                 __skb_incr_checksum_unnecessary(skb);
384         } else {
385                 skb->ip_summed = CHECKSUM_NONE;
386         }
387
388         if (buff->is_udp_cso || buff->is_tcp_cso)
389                 __skb_incr_checksum_unnecessary(skb);
390 }
391
392 int aq_xdp_xmit(struct net_device *dev, int num_frames,
393                 struct xdp_frame **frames, u32 flags)
394 {
395         struct aq_nic_s *aq_nic = netdev_priv(dev);
396         unsigned int vec, i, drop = 0;
397         int cpu = smp_processor_id();
398         struct aq_nic_cfg_s *aq_cfg;
399         struct aq_ring_s *ring;
400
401         aq_cfg = aq_nic_get_cfg(aq_nic);
402         vec = cpu % aq_cfg->vecs;
403         ring = aq_nic->aq_ring_tx[AQ_NIC_CFG_TCVEC2RING(aq_cfg, 0, vec)];
404
405         for (i = 0; i < num_frames; i++) {
406                 struct xdp_frame *xdpf = frames[i];
407
408                 if (aq_nic_xmit_xdpf(aq_nic, ring, xdpf) == NETDEV_TX_BUSY)
409                         drop++;
410         }
411
412         return num_frames - drop;
413 }
414
415 static struct sk_buff *aq_xdp_build_skb(struct xdp_buff *xdp,
416                                         struct net_device *dev,
417                                         struct aq_ring_buff_s *buff)
418 {
419         struct xdp_frame *xdpf;
420         struct sk_buff *skb;
421
422         xdpf = xdp_convert_buff_to_frame(xdp);
423         if (unlikely(!xdpf))
424                 return NULL;
425
426         skb = xdp_build_skb_from_frame(xdpf, dev);
427         if (!skb)
428                 return NULL;
429
430         aq_get_rxpages_xdp(buff, xdp);
431         return skb;
432 }
433
434 static struct sk_buff *aq_xdp_run_prog(struct aq_nic_s *aq_nic,
435                                        struct xdp_buff *xdp,
436                                        struct aq_ring_s *rx_ring,
437                                        struct aq_ring_buff_s *buff)
438 {
439         int result = NETDEV_TX_BUSY;
440         struct aq_ring_s *tx_ring;
441         struct xdp_frame *xdpf;
442         struct bpf_prog *prog;
443         u32 act = XDP_ABORTED;
444         struct sk_buff *skb;
445
446         u64_stats_update_begin(&rx_ring->stats.rx.syncp);
447         ++rx_ring->stats.rx.packets;
448         rx_ring->stats.rx.bytes += xdp_get_buff_len(xdp);
449         u64_stats_update_end(&rx_ring->stats.rx.syncp);
450
451         prog = READ_ONCE(rx_ring->xdp_prog);
452         if (!prog)
453                 return aq_xdp_build_skb(xdp, aq_nic->ndev, buff);
454
455         prefetchw(xdp->data_hard_start); /* xdp_frame write */
456
457         /* single buffer XDP program, but packet is multi buffer, aborted */
458         if (xdp_buff_has_frags(xdp) && !prog->aux->xdp_has_frags)
459                 goto out_aborted;
460
461         act = bpf_prog_run_xdp(prog, xdp);
462         switch (act) {
463         case XDP_PASS:
464                 skb = aq_xdp_build_skb(xdp, aq_nic->ndev, buff);
465                 if (!skb)
466                         goto out_aborted;
467                 u64_stats_update_begin(&rx_ring->stats.rx.syncp);
468                 ++rx_ring->stats.rx.xdp_pass;
469                 u64_stats_update_end(&rx_ring->stats.rx.syncp);
470                 return skb;
471         case XDP_TX:
472                 xdpf = xdp_convert_buff_to_frame(xdp);
473                 if (unlikely(!xdpf))
474                         goto out_aborted;
475                 tx_ring = aq_nic->aq_ring_tx[rx_ring->idx];
476                 result = aq_nic_xmit_xdpf(aq_nic, tx_ring, xdpf);
477                 if (result == NETDEV_TX_BUSY)
478                         goto out_aborted;
479                 u64_stats_update_begin(&rx_ring->stats.rx.syncp);
480                 ++rx_ring->stats.rx.xdp_tx;
481                 u64_stats_update_end(&rx_ring->stats.rx.syncp);
482                 aq_get_rxpages_xdp(buff, xdp);
483                 break;
484         case XDP_REDIRECT:
485                 if (xdp_do_redirect(aq_nic->ndev, xdp, prog) < 0)
486                         goto out_aborted;
487                 xdp_do_flush();
488                 u64_stats_update_begin(&rx_ring->stats.rx.syncp);
489                 ++rx_ring->stats.rx.xdp_redirect;
490                 u64_stats_update_end(&rx_ring->stats.rx.syncp);
491                 aq_get_rxpages_xdp(buff, xdp);
492                 break;
493         default:
494                 fallthrough;
495         case XDP_ABORTED:
496 out_aborted:
497                 u64_stats_update_begin(&rx_ring->stats.rx.syncp);
498                 ++rx_ring->stats.rx.xdp_aborted;
499                 u64_stats_update_end(&rx_ring->stats.rx.syncp);
500                 trace_xdp_exception(aq_nic->ndev, prog, act);
501                 bpf_warn_invalid_xdp_action(aq_nic->ndev, prog, act);
502                 break;
503         case XDP_DROP:
504                 u64_stats_update_begin(&rx_ring->stats.rx.syncp);
505                 ++rx_ring->stats.rx.xdp_drop;
506                 u64_stats_update_end(&rx_ring->stats.rx.syncp);
507                 break;
508         }
509
510         return ERR_PTR(-result);
511 }
512
513 static bool aq_add_rx_fragment(struct device *dev,
514                                struct aq_ring_s *ring,
515                                struct aq_ring_buff_s *buff,
516                                struct xdp_buff *xdp)
517 {
518         struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
519         struct aq_ring_buff_s *buff_ = buff;
520
521         memset(sinfo, 0, sizeof(*sinfo));
522         do {
523                 skb_frag_t *frag;
524
525                 if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS))
526                         return true;
527
528                 frag = &sinfo->frags[sinfo->nr_frags++];
529                 buff_ = &ring->buff_ring[buff_->next];
530                 dma_sync_single_range_for_cpu(dev,
531                                               buff_->rxdata.daddr,
532                                               buff_->rxdata.pg_off,
533                                               buff_->len,
534                                               DMA_FROM_DEVICE);
535                 sinfo->xdp_frags_size += buff_->len;
536                 skb_frag_fill_page_desc(frag, buff_->rxdata.page,
537                                         buff_->rxdata.pg_off,
538                                         buff_->len);
539
540                 buff_->is_cleaned = 1;
541
542                 buff->is_ip_cso &= buff_->is_ip_cso;
543                 buff->is_udp_cso &= buff_->is_udp_cso;
544                 buff->is_tcp_cso &= buff_->is_tcp_cso;
545                 buff->is_cso_err |= buff_->is_cso_err;
546
547                 if (page_is_pfmemalloc(buff_->rxdata.page))
548                         xdp_buff_set_frag_pfmemalloc(xdp);
549
550         } while (!buff_->is_eop);
551
552         xdp_buff_set_frags_flag(xdp);
553
554         return false;
555 }
556
557 static int __aq_ring_rx_clean(struct aq_ring_s *self, struct napi_struct *napi,
558                               int *work_done, int budget)
559 {
560         struct net_device *ndev = aq_nic_get_ndev(self->aq_nic);
561         int err = 0;
562
563         for (; (self->sw_head != self->hw_head) && budget;
564                 self->sw_head = aq_ring_next_dx(self, self->sw_head),
565                 --budget, ++(*work_done)) {
566                 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];
567                 bool is_ptp_ring = aq_ptp_ring(self->aq_nic, self);
568                 struct aq_ring_buff_s *buff_ = NULL;
569                 struct sk_buff *skb = NULL;
570                 unsigned int next_ = 0U;
571                 unsigned int i = 0U;
572                 u16 hdr_len;
573
574                 if (buff->is_cleaned)
575                         continue;
576
577                 if (!buff->is_eop) {
578                         unsigned int frag_cnt = 0U;
579                         buff_ = buff;
580                         do {
581                                 bool is_rsc_completed = true;
582
583                                 if (buff_->next >= self->size) {
584                                         err = -EIO;
585                                         goto err_exit;
586                                 }
587
588                                 frag_cnt++;
589                                 next_ = buff_->next,
590                                 buff_ = &self->buff_ring[next_];
591                                 is_rsc_completed =
592                                         aq_ring_dx_in_range(self->sw_head,
593                                                             next_,
594                                                             self->hw_head);
595
596                                 if (unlikely(!is_rsc_completed) ||
597                                                 frag_cnt > MAX_SKB_FRAGS) {
598                                         err = 0;
599                                         goto err_exit;
600                                 }
601
602                                 buff->is_error |= buff_->is_error;
603                                 buff->is_cso_err |= buff_->is_cso_err;
604
605                         } while (!buff_->is_eop);
606
607                         if (buff->is_error ||
608                             (buff->is_lro && buff->is_cso_err)) {
609                                 buff_ = buff;
610                                 do {
611                                         if (buff_->next >= self->size) {
612                                                 err = -EIO;
613                                                 goto err_exit;
614                                         }
615                                         next_ = buff_->next,
616                                         buff_ = &self->buff_ring[next_];
617
618                                         buff_->is_cleaned = true;
619                                 } while (!buff_->is_eop);
620
621                                 u64_stats_update_begin(&self->stats.rx.syncp);
622                                 ++self->stats.rx.errors;
623                                 u64_stats_update_end(&self->stats.rx.syncp);
624                                 continue;
625                         }
626                 }
627
628                 if (buff->is_error) {
629                         u64_stats_update_begin(&self->stats.rx.syncp);
630                         ++self->stats.rx.errors;
631                         u64_stats_update_end(&self->stats.rx.syncp);
632                         continue;
633                 }
634
635                 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic),
636                                               buff->rxdata.daddr,
637                                               buff->rxdata.pg_off,
638                                               buff->len, DMA_FROM_DEVICE);
639
640                 skb = napi_alloc_skb(napi, AQ_CFG_RX_HDR_SIZE);
641                 if (unlikely(!skb)) {
642                         u64_stats_update_begin(&self->stats.rx.syncp);
643                         self->stats.rx.skb_alloc_fails++;
644                         u64_stats_update_end(&self->stats.rx.syncp);
645                         err = -ENOMEM;
646                         goto err_exit;
647                 }
648                 if (is_ptp_ring)
649                         buff->len -=
650                                 aq_ptp_extract_ts(self->aq_nic, skb_hwtstamps(skb),
651                                                   aq_buf_vaddr(&buff->rxdata),
652                                                   buff->len);
653
654                 hdr_len = buff->len;
655                 if (hdr_len > AQ_CFG_RX_HDR_SIZE)
656                         hdr_len = eth_get_headlen(skb->dev,
657                                                   aq_buf_vaddr(&buff->rxdata),
658                                                   AQ_CFG_RX_HDR_SIZE);
659
660                 memcpy(__skb_put(skb, hdr_len), aq_buf_vaddr(&buff->rxdata),
661                        ALIGN(hdr_len, sizeof(long)));
662
663                 if (buff->len - hdr_len > 0) {
664                         skb_add_rx_frag(skb, i++, buff->rxdata.page,
665                                         buff->rxdata.pg_off + hdr_len,
666                                         buff->len - hdr_len,
667                                         self->frame_max);
668                         page_ref_inc(buff->rxdata.page);
669                 }
670
671                 if (!buff->is_eop) {
672                         buff_ = buff;
673                         do {
674                                 next_ = buff_->next;
675                                 buff_ = &self->buff_ring[next_];
676
677                                 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic),
678                                                               buff_->rxdata.daddr,
679                                                               buff_->rxdata.pg_off,
680                                                               buff_->len,
681                                                               DMA_FROM_DEVICE);
682                                 skb_add_rx_frag(skb, i++,
683                                                 buff_->rxdata.page,
684                                                 buff_->rxdata.pg_off,
685                                                 buff_->len,
686                                                 self->frame_max);
687                                 page_ref_inc(buff_->rxdata.page);
688                                 buff_->is_cleaned = 1;
689
690                                 buff->is_ip_cso &= buff_->is_ip_cso;
691                                 buff->is_udp_cso &= buff_->is_udp_cso;
692                                 buff->is_tcp_cso &= buff_->is_tcp_cso;
693                                 buff->is_cso_err |= buff_->is_cso_err;
694
695                         } while (!buff_->is_eop);
696                 }
697
698                 if (buff->is_vlan)
699                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
700                                                buff->vlan_rx_tag);
701
702                 skb->protocol = eth_type_trans(skb, ndev);
703
704                 aq_rx_checksum(self, buff, skb);
705
706                 skb_set_hash(skb, buff->rss_hash,
707                              buff->is_hash_l4 ? PKT_HASH_TYPE_L4 :
708                              PKT_HASH_TYPE_NONE);
709                 /* Send all PTP traffic to 0 queue */
710                 skb_record_rx_queue(skb,
711                                     is_ptp_ring ? 0
712                                                 : AQ_NIC_RING2QMAP(self->aq_nic,
713                                                                    self->idx));
714
715                 u64_stats_update_begin(&self->stats.rx.syncp);
716                 ++self->stats.rx.packets;
717                 self->stats.rx.bytes += skb->len;
718                 u64_stats_update_end(&self->stats.rx.syncp);
719
720                 napi_gro_receive(napi, skb);
721         }
722
723 err_exit:
724         return err;
725 }
726
727 static int __aq_ring_xdp_clean(struct aq_ring_s *rx_ring,
728                                struct napi_struct *napi, int *work_done,
729                                int budget)
730 {
731         int frame_sz = rx_ring->page_offset + rx_ring->frame_max +
732                        rx_ring->tail_size;
733         struct aq_nic_s *aq_nic = rx_ring->aq_nic;
734         bool is_rsc_completed = true;
735         struct device *dev;
736         int err = 0;
737
738         dev = aq_nic_get_dev(aq_nic);
739         for (; (rx_ring->sw_head != rx_ring->hw_head) && budget;
740                 rx_ring->sw_head = aq_ring_next_dx(rx_ring, rx_ring->sw_head),
741                 --budget, ++(*work_done)) {
742                 struct aq_ring_buff_s *buff = &rx_ring->buff_ring[rx_ring->sw_head];
743                 bool is_ptp_ring = aq_ptp_ring(rx_ring->aq_nic, rx_ring);
744                 struct aq_ring_buff_s *buff_ = NULL;
745                 u16 ptp_hwtstamp_len = 0;
746                 struct skb_shared_hwtstamps shhwtstamps;
747                 struct sk_buff *skb = NULL;
748                 unsigned int next_ = 0U;
749                 struct xdp_buff xdp;
750                 void *hard_start;
751
752                 if (buff->is_cleaned)
753                         continue;
754
755                 if (!buff->is_eop) {
756                         buff_ = buff;
757                         do {
758                                 if (buff_->next >= rx_ring->size) {
759                                         err = -EIO;
760                                         goto err_exit;
761                                 }
762                                 next_ = buff_->next;
763                                 buff_ = &rx_ring->buff_ring[next_];
764                                 is_rsc_completed =
765                                         aq_ring_dx_in_range(rx_ring->sw_head,
766                                                             next_,
767                                                             rx_ring->hw_head);
768
769                                 if (unlikely(!is_rsc_completed))
770                                         break;
771
772                                 buff->is_error |= buff_->is_error;
773                                 buff->is_cso_err |= buff_->is_cso_err;
774                         } while (!buff_->is_eop);
775
776                         if (!is_rsc_completed) {
777                                 err = 0;
778                                 goto err_exit;
779                         }
780                         if (buff->is_error ||
781                             (buff->is_lro && buff->is_cso_err)) {
782                                 buff_ = buff;
783                                 do {
784                                         if (buff_->next >= rx_ring->size) {
785                                                 err = -EIO;
786                                                 goto err_exit;
787                                         }
788                                         next_ = buff_->next;
789                                         buff_ = &rx_ring->buff_ring[next_];
790
791                                         buff_->is_cleaned = true;
792                                 } while (!buff_->is_eop);
793
794                                 u64_stats_update_begin(&rx_ring->stats.rx.syncp);
795                                 ++rx_ring->stats.rx.errors;
796                                 u64_stats_update_end(&rx_ring->stats.rx.syncp);
797                                 continue;
798                         }
799                 }
800
801                 if (buff->is_error) {
802                         u64_stats_update_begin(&rx_ring->stats.rx.syncp);
803                         ++rx_ring->stats.rx.errors;
804                         u64_stats_update_end(&rx_ring->stats.rx.syncp);
805                         continue;
806                 }
807
808                 dma_sync_single_range_for_cpu(dev,
809                                               buff->rxdata.daddr,
810                                               buff->rxdata.pg_off,
811                                               buff->len, DMA_FROM_DEVICE);
812                 hard_start = page_address(buff->rxdata.page) +
813                              buff->rxdata.pg_off - rx_ring->page_offset;
814
815                 if (is_ptp_ring) {
816                         ptp_hwtstamp_len = aq_ptp_extract_ts(rx_ring->aq_nic, &shhwtstamps,
817                                                              aq_buf_vaddr(&buff->rxdata),
818                                                              buff->len);
819                         buff->len -= ptp_hwtstamp_len;
820                 }
821
822                 xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq);
823                 xdp_prepare_buff(&xdp, hard_start, rx_ring->page_offset,
824                                  buff->len, false);
825                 if (!buff->is_eop) {
826                         if (aq_add_rx_fragment(dev, rx_ring, buff, &xdp)) {
827                                 u64_stats_update_begin(&rx_ring->stats.rx.syncp);
828                                 ++rx_ring->stats.rx.packets;
829                                 rx_ring->stats.rx.bytes += xdp_get_buff_len(&xdp);
830                                 ++rx_ring->stats.rx.xdp_aborted;
831                                 u64_stats_update_end(&rx_ring->stats.rx.syncp);
832                                 continue;
833                         }
834                 }
835
836                 skb = aq_xdp_run_prog(aq_nic, &xdp, rx_ring, buff);
837                 if (IS_ERR(skb) || !skb)
838                         continue;
839
840                 if (ptp_hwtstamp_len > 0)
841                         *skb_hwtstamps(skb) = shhwtstamps;
842
843                 if (buff->is_vlan)
844                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
845                                                buff->vlan_rx_tag);
846
847                 aq_rx_checksum(rx_ring, buff, skb);
848
849                 skb_set_hash(skb, buff->rss_hash,
850                              buff->is_hash_l4 ? PKT_HASH_TYPE_L4 :
851                              PKT_HASH_TYPE_NONE);
852                 /* Send all PTP traffic to 0 queue */
853                 skb_record_rx_queue(skb,
854                                     is_ptp_ring ? 0
855                                                 : AQ_NIC_RING2QMAP(rx_ring->aq_nic,
856                                                                    rx_ring->idx));
857
858                 napi_gro_receive(napi, skb);
859         }
860
861 err_exit:
862         return err;
863 }
864
865 int aq_ring_rx_clean(struct aq_ring_s *self,
866                      struct napi_struct *napi,
867                      int *work_done,
868                      int budget)
869 {
870         if (static_branch_unlikely(&aq_xdp_locking_key))
871                 return __aq_ring_xdp_clean(self, napi, work_done, budget);
872         else
873                 return __aq_ring_rx_clean(self, napi, work_done, budget);
874 }
875
876 void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic)
877 {
878 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
879         while (self->sw_head != self->hw_head) {
880                 u64 ns;
881
882                 aq_nic->aq_hw_ops->extract_hwts(aq_nic->aq_hw,
883                                                 self->dx_ring +
884                                                 (self->sw_head * self->dx_size),
885                                                 self->dx_size, &ns);
886                 aq_ptp_tx_hwtstamp(aq_nic, ns);
887
888                 self->sw_head = aq_ring_next_dx(self, self->sw_head);
889         }
890 #endif
891 }
892
893 int aq_ring_rx_fill(struct aq_ring_s *self)
894 {
895         struct aq_ring_buff_s *buff = NULL;
896         int err = 0;
897         int i = 0;
898
899         if (aq_ring_avail_dx(self) < min_t(unsigned int, AQ_CFG_RX_REFILL_THRES,
900                                            self->size / 2))
901                 return err;
902
903         for (i = aq_ring_avail_dx(self); i--;
904                 self->sw_tail = aq_ring_next_dx(self, self->sw_tail)) {
905                 buff = &self->buff_ring[self->sw_tail];
906
907                 buff->flags = 0U;
908                 buff->len = self->frame_max;
909
910                 err = aq_get_rxpages(self, buff);
911                 if (err)
912                         goto err_exit;
913
914                 buff->pa = aq_buf_daddr(&buff->rxdata);
915                 buff = NULL;
916         }
917
918 err_exit:
919         return err;
920 }
921
922 void aq_ring_rx_deinit(struct aq_ring_s *self)
923 {
924         if (!self)
925                 return;
926
927         for (; self->sw_head != self->sw_tail;
928                 self->sw_head = aq_ring_next_dx(self, self->sw_head)) {
929                 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];
930
931                 aq_free_rxpage(&buff->rxdata, aq_nic_get_dev(self->aq_nic));
932         }
933 }
934
935 void aq_ring_free(struct aq_ring_s *self)
936 {
937         if (!self)
938                 return;
939
940         kfree(self->buff_ring);
941
942         if (self->dx_ring)
943                 dma_free_coherent(aq_nic_get_dev(self->aq_nic),
944                                   self->size * self->dx_size, self->dx_ring,
945                                   self->dx_ring_pa);
946 }
947
948 unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data)
949 {
950         unsigned int count;
951         unsigned int start;
952
953         if (self->ring_type == ATL_RING_RX) {
954                 /* This data should mimic aq_ethtool_queue_rx_stat_names structure */
955                 do {
956                         count = 0;
957                         start = u64_stats_fetch_begin(&self->stats.rx.syncp);
958                         data[count] = self->stats.rx.packets;
959                         data[++count] = self->stats.rx.jumbo_packets;
960                         data[++count] = self->stats.rx.lro_packets;
961                         data[++count] = self->stats.rx.errors;
962                         data[++count] = self->stats.rx.alloc_fails;
963                         data[++count] = self->stats.rx.skb_alloc_fails;
964                         data[++count] = self->stats.rx.polls;
965                         data[++count] = self->stats.rx.pg_flips;
966                         data[++count] = self->stats.rx.pg_reuses;
967                         data[++count] = self->stats.rx.pg_losts;
968                         data[++count] = self->stats.rx.xdp_aborted;
969                         data[++count] = self->stats.rx.xdp_drop;
970                         data[++count] = self->stats.rx.xdp_pass;
971                         data[++count] = self->stats.rx.xdp_tx;
972                         data[++count] = self->stats.rx.xdp_invalid;
973                         data[++count] = self->stats.rx.xdp_redirect;
974                 } while (u64_stats_fetch_retry(&self->stats.rx.syncp, start));
975         } else {
976                 /* This data should mimic aq_ethtool_queue_tx_stat_names structure */
977                 do {
978                         count = 0;
979                         start = u64_stats_fetch_begin(&self->stats.tx.syncp);
980                         data[count] = self->stats.tx.packets;
981                         data[++count] = self->stats.tx.queue_restarts;
982                 } while (u64_stats_fetch_retry(&self->stats.tx.syncp, start));
983         }
984
985         return ++count;
986 }