drivers/net/ethernet/aquantia/atlantic/aq_ring.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Atlantic Network Driver
   3  *
   4  * Copyright (C) 2014-2019 aQuantia Corporation
   5  * Copyright (C) 2019-2020 Marvell International Ltd.
   6  */
   7
   8 /* File aq_ring.c: Definition of functions for Rx/Tx rings. */
   9
  10 #include "aq_ring.h"
  11 #include "aq_nic.h"
  12 #include "aq_hw.h"
  13 #include "aq_hw_utils.h"
  14 #include "aq_ptp.h"
  15
  16 #include <linux/netdevice.h>
  17 #include <linux/etherdevice.h>
  18
  19 static inline void aq_free_rxpage(struct aq_rxpage *rxpage, struct device *dev)
  20 {
  21         unsigned int len = PAGE_SIZE << rxpage->order;
  22
  23         dma_unmap_page(dev, rxpage->daddr, len, DMA_FROM_DEVICE);
  24
  25         /* Drop the ref for being in the ring. */
  26         __free_pages(rxpage->page, rxpage->order);
  27         rxpage->page = NULL;
  28 }
  29
  30 static int aq_get_rxpage(struct aq_rxpage *rxpage, unsigned int order,
  31                          struct device *dev)
  32 {
  33         struct page *page;
  34         int ret = -ENOMEM;
  35         dma_addr_t daddr;
  36
  37         page = dev_alloc_pages(order);
  38         if (unlikely(!page))
  39                 goto err_exit;
  40
  41         daddr = dma_map_page(dev, page, 0, PAGE_SIZE << order,
  42                              DMA_FROM_DEVICE);
  43
  44         if (unlikely(dma_mapping_error(dev, daddr)))
  45                 goto free_page;
  46
  47         rxpage->page = page;
  48         rxpage->daddr = daddr;
  49         rxpage->order = order;
  50         rxpage->pg_off = 0;
  51
  52         return 0;
  53
  54 free_page:
  55         __free_pages(page, order);
  56
  57 err_exit:
  58         return ret;
  59 }
  60
  61 static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf,
  62                           int order)
  63 {
  64         int ret;
  65
  66         if (rxbuf->rxdata.page) {
  67                 /* One means ring is the only user and can reuse */
  68                 if (page_ref_count(rxbuf->rxdata.page) > 1) {
  69                         /* Try reuse buffer */
  70                         rxbuf->rxdata.pg_off += AQ_CFG_RX_FRAME_MAX;
  71                         if (rxbuf->rxdata.pg_off + AQ_CFG_RX_FRAME_MAX <=
  72                                 (PAGE_SIZE << order)) {
  73                                 self->stats.rx.pg_flips++;
  74                         } else {
  75                                 /* Buffer exhausted. We have other users and
  76                                  * should release this page and realloc
  77                                  */
  78                                 aq_free_rxpage(&rxbuf->rxdata,
  79                                                aq_nic_get_dev(self->aq_nic));
  80                                 self->stats.rx.pg_losts++;
  81                         }
  82                 } else {
  83                         rxbuf->rxdata.pg_off = 0;
  84                         self->stats.rx.pg_reuses++;
  85                 }
  86         }
  87
  88         if (!rxbuf->rxdata.page) {
  89                 ret = aq_get_rxpage(&rxbuf->rxdata, order,
  90                                     aq_nic_get_dev(self->aq_nic));
  91                 return ret;
  92         }
  93
  94         return 0;
  95 }
  96
  97 static struct aq_ring_s *aq_ring_alloc(struct aq_ring_s *self,
  98                                        struct aq_nic_s *aq_nic)
  99 {
 100         int err = 0;
 101
 102         self->buff_ring =
 103                 kcalloc(self->size, sizeof(struct aq_ring_buff_s), GFP_KERNEL);
 104
 105         if (!self->buff_ring) {
 106                 err = -ENOMEM;
 107                 goto err_exit;
 108         }
 109         self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic),
 110                                            self->size * self->dx_size,
 111                                            &self->dx_ring_pa, GFP_KERNEL);
 112         if (!self->dx_ring) {
 113                 err = -ENOMEM;
 114                 goto err_exit;
 115         }
 116
 117 err_exit:
 118         if (err < 0) {
 119                 aq_ring_free(self);
 120                 self = NULL;
 121         }
 122
 123         return self;
 124 }
 125
 126 struct aq_ring_s *aq_ring_tx_alloc(struct aq_ring_s *self,
 127                                    struct aq_nic_s *aq_nic,
 128                                    unsigned int idx,
 129                                    struct aq_nic_cfg_s *aq_nic_cfg)
 130 {
 131         int err = 0;
 132
 133         self->aq_nic = aq_nic;
 134         self->idx = idx;
 135         self->size = aq_nic_cfg->txds;
 136         self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size;
 137
 138         self = aq_ring_alloc(self, aq_nic);
 139         if (!self) {
 140                 err = -ENOMEM;
 141                 goto err_exit;
 142         }
 143
 144 err_exit:
 145         if (err < 0) {
 146                 aq_ring_free(self);
 147                 self = NULL;
 148         }
 149
 150         return self;
 151 }
 152
 153 struct aq_ring_s *aq_ring_rx_alloc(struct aq_ring_s *self,
 154                                    struct aq_nic_s *aq_nic,
 155                                    unsigned int idx,
 156                                    struct aq_nic_cfg_s *aq_nic_cfg)
 157 {
 158         int err = 0;
 159
 160         self->aq_nic = aq_nic;
 161         self->idx = idx;
 162         self->size = aq_nic_cfg->rxds;
 163         self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size;
 164         self->page_order = fls(AQ_CFG_RX_FRAME_MAX / PAGE_SIZE +
 165                                (AQ_CFG_RX_FRAME_MAX % PAGE_SIZE ? 1 : 0)) - 1;
 166
 167         if (aq_nic_cfg->rxpageorder > self->page_order)
 168                 self->page_order = aq_nic_cfg->rxpageorder;
 169
 170         self = aq_ring_alloc(self, aq_nic);
 171         if (!self) {
 172                 err = -ENOMEM;
 173                 goto err_exit;
 174         }
 175
 176 err_exit:
 177         if (err < 0) {
 178                 aq_ring_free(self);
 179                 self = NULL;
 180         }
 181
 182         return self;
 183 }
 184
 185 struct aq_ring_s *
 186 aq_ring_hwts_rx_alloc(struct aq_ring_s *self, struct aq_nic_s *aq_nic,
 187                       unsigned int idx, unsigned int size, unsigned int dx_size)
 188 {
 189         struct device *dev = aq_nic_get_dev(aq_nic);
 190         size_t sz = size * dx_size + AQ_CFG_RXDS_DEF;
 191
 192         memset(self, 0, sizeof(*self));
 193
 194         self->aq_nic = aq_nic;
 195         self->idx = idx;
 196         self->size = size;
 197         self->dx_size = dx_size;
 198
 199         self->dx_ring = dma_alloc_coherent(dev, sz, &self->dx_ring_pa,
 200                                            GFP_KERNEL);
 201         if (!self->dx_ring) {
 202                 aq_ring_free(self);
 203                 return NULL;
 204         }
 205
 206         return self;
 207 }
 208
 209 int aq_ring_init(struct aq_ring_s *self, const enum atl_ring_type ring_type)
 210 {
 211         self->hw_head = 0;
 212         self->sw_head = 0;
 213         self->sw_tail = 0;
 214         self->ring_type = ring_type;
 215
 216         return 0;
 217 }
 218
 219 static inline bool aq_ring_dx_in_range(unsigned int h, unsigned int i,
 220                                        unsigned int t)
 221 {
 222         return (h < t) ? ((h < i) && (i < t)) : ((h < i) || (i < t));
 223 }
 224
 225 void aq_ring_update_queue_state(struct aq_ring_s *ring)
 226 {
 227         if (aq_ring_avail_dx(ring) <= AQ_CFG_SKB_FRAGS_MAX)
 228                 aq_ring_queue_stop(ring);
 229         else if (aq_ring_avail_dx(ring) > AQ_CFG_RESTART_DESC_THRES)
 230                 aq_ring_queue_wake(ring);
 231 }
 232
 233 void aq_ring_queue_wake(struct aq_ring_s *ring)
 234 {
 235         struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic);
 236
 237         if (__netif_subqueue_stopped(ndev,
 238                                      AQ_NIC_RING2QMAP(ring->aq_nic,
 239                                                       ring->idx))) {
 240                 netif_wake_subqueue(ndev,
 241                                     AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx));
 242                 ring->stats.tx.queue_restarts++;
 243         }
 244 }
 245
 246 void aq_ring_queue_stop(struct aq_ring_s *ring)
 247 {
 248         struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic);
 249
 250         if (!__netif_subqueue_stopped(ndev,
 251                                       AQ_NIC_RING2QMAP(ring->aq_nic,
 252                                                        ring->idx)))
 253                 netif_stop_subqueue(ndev,
 254                                     AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx));
 255 }
 256
 257 bool aq_ring_tx_clean(struct aq_ring_s *self)
 258 {
 259         struct device *dev = aq_nic_get_dev(self->aq_nic);
 260         unsigned int budget;
 261
 262         for (budget = AQ_CFG_TX_CLEAN_BUDGET;
 263              budget && self->sw_head != self->hw_head; budget--) {
 264                 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];
 265
 266                 if (likely(buff->is_mapped)) {
 267                         if (unlikely(buff->is_sop)) {
 268                                 if (!buff->is_eop &&
 269                                     buff->eop_index != 0xffffU &&
 270                                     (!aq_ring_dx_in_range(self->sw_head,
 271                                                 buff->eop_index,
 272                                                 self->hw_head)))
 273                                         break;
 274
 275                                 dma_unmap_single(dev, buff->pa, buff->len,
 276                                                  DMA_TO_DEVICE);
 277                         } else {
 278                                 dma_unmap_page(dev, buff->pa, buff->len,
 279                                                DMA_TO_DEVICE);
 280                         }
 281                 }
 282
 283                 if (unlikely(buff->is_eop)) {
 284                         ++self->stats.tx.packets;
 285                         self->stats.tx.bytes += buff->skb->len;
 286
 287                         dev_kfree_skb_any(buff->skb);
 288                 }
 289                 buff->pa = 0U;
 290                 buff->eop_index = 0xffffU;
 291                 self->sw_head = aq_ring_next_dx(self, self->sw_head);
 292         }
 293
 294         return !!budget;
 295 }
 296
 297 static void aq_rx_checksum(struct aq_ring_s *self,
 298                            struct aq_ring_buff_s *buff,
 299                            struct sk_buff *skb)
 300 {
 301         if (!(self->aq_nic->ndev->features & NETIF_F_RXCSUM))
 302                 return;
 303
 304         if (unlikely(buff->is_cso_err)) {
 305                 ++self->stats.rx.errors;
 306                 skb->ip_summed = CHECKSUM_NONE;
 307                 return;
 308         }
 309         if (buff->is_ip_cso) {
 310                 __skb_incr_checksum_unnecessary(skb);
 311         } else {
 312                 skb->ip_summed = CHECKSUM_NONE;
 313         }
 314
 315         if (buff->is_udp_cso || buff->is_tcp_cso)
 316                 __skb_incr_checksum_unnecessary(skb);
 317 }
 318
 319 #define AQ_SKB_ALIGN SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
 320 int aq_ring_rx_clean(struct aq_ring_s *self,
 321                      struct napi_struct *napi,
 322                      int *work_done,
 323                      int budget)
 324 {
 325         struct net_device *ndev = aq_nic_get_ndev(self->aq_nic);
 326         bool is_rsc_completed = true;
 327         int err = 0;
 328
 329         for (; (self->sw_head != self->hw_head) && budget;
 330                 self->sw_head = aq_ring_next_dx(self, self->sw_head),
 331                 --budget, ++(*work_done)) {
 332                 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];
 333                 bool is_ptp_ring = aq_ptp_ring(self->aq_nic, self);
 334                 struct aq_ring_buff_s *buff_ = NULL;
 335                 struct sk_buff *skb = NULL;
 336                 unsigned int next_ = 0U;
 337                 unsigned int i = 0U;
 338                 u16 hdr_len;
 339
 340                 if (buff->is_cleaned)
 341                         continue;
 342
 343                 if (!buff->is_eop) {
 344                         buff_ = buff;
 345                         do {
 346                                 next_ = buff_->next,
 347                                 buff_ = &self->buff_ring[next_];
 348                                 is_rsc_completed =
 349                                         aq_ring_dx_in_range(self->sw_head,
 350                                                             next_,
 351                                                             self->hw_head);
 352
 353                                 if (unlikely(!is_rsc_completed))
 354                                         break;
 355
 356                                 buff->is_error |= buff_->is_error;
 357                                 buff->is_cso_err |= buff_->is_cso_err;
 358
 359                         } while (!buff_->is_eop);
 360
 361                         if (!is_rsc_completed) {
 362                                 err = 0;
 363                                 goto err_exit;
 364                         }
 365                         if (buff->is_error ||
 366                             (buff->is_lro && buff->is_cso_err)) {
 367                                 buff_ = buff;
 368                                 do {
 369                                         next_ = buff_->next,
 370                                         buff_ = &self->buff_ring[next_];
 371
 372                                         buff_->is_cleaned = true;
 373                                 } while (!buff_->is_eop);
 374
 375                                 ++self->stats.rx.errors;
 376                                 continue;
 377                         }
 378                 }
 379
 380                 if (buff->is_error) {
 381                         ++self->stats.rx.errors;
 382                         continue;
 383                 }
 384
 385                 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic),
 386                                               buff->rxdata.daddr,
 387                                               buff->rxdata.pg_off,
 388                                               buff->len, DMA_FROM_DEVICE);
 389
 390                 /* for single fragment packets use build_skb() */
 391                 if (buff->is_eop &&
 392                     buff->len <= AQ_CFG_RX_FRAME_MAX - AQ_SKB_ALIGN) {
 393                         skb = build_skb(aq_buf_vaddr(&buff->rxdata),
 394                                         AQ_CFG_RX_FRAME_MAX);
 395                         if (unlikely(!skb)) {
 396                                 err = -ENOMEM;
 397                                 goto err_exit;
 398                         }
 399                         if (is_ptp_ring)
 400                                 buff->len -=
 401                                         aq_ptp_extract_ts(self->aq_nic, skb,
 402                                                 aq_buf_vaddr(&buff->rxdata),
 403                                                 buff->len);
 404                         skb_put(skb, buff->len);
 405                         page_ref_inc(buff->rxdata.page);
 406                 } else {
 407                         skb = napi_alloc_skb(napi, AQ_CFG_RX_HDR_SIZE);
 408                         if (unlikely(!skb)) {
 409                                 err = -ENOMEM;
 410                                 goto err_exit;
 411                         }
 412                         if (is_ptp_ring)
 413                                 buff->len -=
 414                                         aq_ptp_extract_ts(self->aq_nic, skb,
 415                                                 aq_buf_vaddr(&buff->rxdata),
 416                                                 buff->len);
 417
 418                         hdr_len = buff->len;
 419                         if (hdr_len > AQ_CFG_RX_HDR_SIZE)
 420                                 hdr_len = eth_get_headlen(skb->dev,
 421                                                           aq_buf_vaddr(&buff->rxdata),
 422                                                           AQ_CFG_RX_HDR_SIZE);
 423
 424                         memcpy(__skb_put(skb, hdr_len), aq_buf_vaddr(&buff->rxdata),
 425                                ALIGN(hdr_len, sizeof(long)));
 426
 427                         if (buff->len - hdr_len > 0) {
 428                                 skb_add_rx_frag(skb, 0, buff->rxdata.page,
 429                                                 buff->rxdata.pg_off + hdr_len,
 430                                                 buff->len - hdr_len,
 431                                                 AQ_CFG_RX_FRAME_MAX);
 432                                 page_ref_inc(buff->rxdata.page);
 433                         }
 434
 435                         if (!buff->is_eop) {
 436                                 buff_ = buff;
 437                                 i = 1U;
 438                                 do {
 439                                         next_ = buff_->next,
 440                                         buff_ = &self->buff_ring[next_];
 441
 442                                         dma_sync_single_range_for_cpu(
 443                                                         aq_nic_get_dev(self->aq_nic),
 444                                                         buff_->rxdata.daddr,
 445                                                         buff_->rxdata.pg_off,
 446                                                         buff_->len,
 447                                                         DMA_FROM_DEVICE);
 448                                         skb_add_rx_frag(skb, i++,
 449                                                         buff_->rxdata.page,
 450                                                         buff_->rxdata.pg_off,
 451                                                         buff_->len,
 452                                                         AQ_CFG_RX_FRAME_MAX);
 453                                         page_ref_inc(buff_->rxdata.page);
 454                                         buff_->is_cleaned = 1;
 455
 456                                         buff->is_ip_cso &= buff_->is_ip_cso;
 457                                         buff->is_udp_cso &= buff_->is_udp_cso;
 458                                         buff->is_tcp_cso &= buff_->is_tcp_cso;
 459                                         buff->is_cso_err |= buff_->is_cso_err;
 460
 461                                 } while (!buff_->is_eop);
 462                         }
 463                 }
 464
 465                 if (buff->is_vlan)
 466                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 467                                                buff->vlan_rx_tag);
 468
 469                 skb->protocol = eth_type_trans(skb, ndev);
 470
 471                 aq_rx_checksum(self, buff, skb);
 472
 473                 skb_set_hash(skb, buff->rss_hash,
 474                              buff->is_hash_l4 ? PKT_HASH_TYPE_L4 :
 475                              PKT_HASH_TYPE_NONE);
 476                 /* Send all PTP traffic to 0 queue */
 477                 skb_record_rx_queue(skb,
 478                                     is_ptp_ring ? 0
 479                                                 : AQ_NIC_RING2QMAP(self->aq_nic,
 480                                                                    self->idx));
 481
 482                 ++self->stats.rx.packets;
 483                 self->stats.rx.bytes += skb->len;
 484
 485                 napi_gro_receive(napi, skb);
 486         }
 487
 488 err_exit:
 489         return err;
 490 }
 491
 492 void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic)
 493 {
 494 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
 495         while (self->sw_head != self->hw_head) {
 496                 u64 ns;
 497
 498                 aq_nic->aq_hw_ops->extract_hwts(aq_nic->aq_hw,
 499                                                 self->dx_ring +
 500                                                 (self->sw_head * self->dx_size),
 501                                                 self->dx_size, &ns);
 502                 aq_ptp_tx_hwtstamp(aq_nic, ns);
 503
 504                 self->sw_head = aq_ring_next_dx(self, self->sw_head);
 505         }
 506 #endif
 507 }
 508
 509 int aq_ring_rx_fill(struct aq_ring_s *self)
 510 {
 511         unsigned int page_order = self->page_order;
 512         struct aq_ring_buff_s *buff = NULL;
 513         int err = 0;
 514         int i = 0;
 515
 516         if (aq_ring_avail_dx(self) < min_t(unsigned int, AQ_CFG_RX_REFILL_THRES,
 517                                            self->size / 2))
 518                 return err;
 519
 520         for (i = aq_ring_avail_dx(self); i--;
 521                 self->sw_tail = aq_ring_next_dx(self, self->sw_tail)) {
 522                 buff = &self->buff_ring[self->sw_tail];
 523
 524                 buff->flags = 0U;
 525                 buff->len = AQ_CFG_RX_FRAME_MAX;
 526
 527                 err = aq_get_rxpages(self, buff, page_order);
 528                 if (err)
 529                         goto err_exit;
 530
 531                 buff->pa = aq_buf_daddr(&buff->rxdata);
 532                 buff = NULL;
 533         }
 534
 535 err_exit:
 536         return err;
 537 }
 538
 539 void aq_ring_rx_deinit(struct aq_ring_s *self)
 540 {
 541         if (!self)
 542                 return;
 543
 544         for (; self->sw_head != self->sw_tail;
 545                 self->sw_head = aq_ring_next_dx(self, self->sw_head)) {
 546                 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];
 547
 548                 aq_free_rxpage(&buff->rxdata, aq_nic_get_dev(self->aq_nic));
 549         }
 550 }
 551
 552 void aq_ring_free(struct aq_ring_s *self)
 553 {
 554         if (!self)
 555                 return;
 556
 557         kfree(self->buff_ring);
 558
 559         if (self->dx_ring)
 560                 dma_free_coherent(aq_nic_get_dev(self->aq_nic),
 561                                   self->size * self->dx_size, self->dx_ring,
 562                                   self->dx_ring_pa);
 563 }
 564
 565 unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data)
 566 {
 567         unsigned int count = 0U;
 568
 569         if (self->ring_type == ATL_RING_RX) {
 570                 /* This data should mimic aq_ethtool_queue_rx_stat_names structure */
 571                 data[count] = self->stats.rx.packets;
 572                 data[++count] = self->stats.rx.jumbo_packets;
 573                 data[++count] = self->stats.rx.lro_packets;
 574                 data[++count] = self->stats.rx.errors;
 575         } else {
 576                 /* This data should mimic aq_ethtool_queue_tx_stat_names structure */
 577                 data[count] = self->stats.tx.packets;
 578                 data[++count] = self->stats.tx.queue_restarts;
 579         }
 580
 581         return ++count;
 582 }