drivers/net/wireless/ath/wil6210/txrx.c

   1 /*
   2  * Copyright (c) 2012-2017 Qualcomm Atheros, Inc.
   3  * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
   4  *
   5  * Permission to use, copy, modify, and/or distribute this software for any
   6  * purpose with or without fee is hereby granted, provided that the above
   7  * copyright notice and this permission notice appear in all copies.
   8  *
   9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  16  */
  17
  18 #include <linux/etherdevice.h>
  19 #include <net/ieee80211_radiotap.h>
  20 #include <linux/if_arp.h>
  21 #include <linux/moduleparam.h>
  22 #include <linux/ip.h>
  23 #include <linux/ipv6.h>
  24 #include <net/ipv6.h>
  25 #include <linux/prefetch.h>
  26
  27 #include "wil6210.h"
  28 #include "wmi.h"
  29 #include "txrx.h"
  30 #include "trace.h"
  31 #include "txrx_edma.h"
  32
  33 bool rx_align_2;
  34 module_param(rx_align_2, bool, 0444);
  35 MODULE_PARM_DESC(rx_align_2, " align Rx buffers on 4*n+2, default - no");
  36
  37 bool rx_large_buf;
  38 module_param(rx_large_buf, bool, 0444);
  39 MODULE_PARM_DESC(rx_large_buf, " allocate 8KB RX buffers, default - no");
  40
  41 /* Drop Tx packets in case Tx ring is full */
  42 bool drop_if_ring_full;
  43
  44 static inline uint wil_rx_snaplen(void)
  45 {
  46         return rx_align_2 ? 6 : 0;
  47 }
  48
  49 /* wil_ring_wmark_low - low watermark for available descriptor space */
  50 static inline int wil_ring_wmark_low(struct wil_ring *ring)
  51 {
  52         return ring->size / 8;
  53 }
  54
  55 /* wil_ring_wmark_high - high watermark for available descriptor space */
  56 static inline int wil_ring_wmark_high(struct wil_ring *ring)
  57 {
  58         return ring->size / 4;
  59 }
  60
  61 /* returns true if num avail descriptors is lower than wmark_low */
  62 static inline int wil_ring_avail_low(struct wil_ring *ring)
  63 {
  64         return wil_ring_avail_tx(ring) < wil_ring_wmark_low(ring);
  65 }
  66
  67 /* returns true if num avail descriptors is higher than wmark_high */
  68 static inline int wil_ring_avail_high(struct wil_ring *ring)
  69 {
  70         return wil_ring_avail_tx(ring) > wil_ring_wmark_high(ring);
  71 }
  72
  73 /* returns true when all tx vrings are empty */
  74 bool wil_is_tx_idle(struct wil6210_priv *wil)
  75 {
  76         int i;
  77         unsigned long data_comp_to;
  78         int min_ring_id = wil_get_min_tx_ring_id(wil);
  79
  80         for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
  81                 struct wil_ring *vring = &wil->ring_tx[i];
  82                 int vring_index = vring - wil->ring_tx;
  83                 struct wil_ring_tx_data *txdata =
  84                         &wil->ring_tx_data[vring_index];
  85
  86                 spin_lock(&txdata->lock);
  87
  88                 if (!vring->va || !txdata->enabled) {
  89                         spin_unlock(&txdata->lock);
  90                         continue;
  91                 }
  92
  93                 data_comp_to = jiffies + msecs_to_jiffies(
  94                                         WIL_DATA_COMPLETION_TO_MS);
  95                 if (test_bit(wil_status_napi_en, wil->status)) {
  96                         while (!wil_ring_is_empty(vring)) {
  97                                 if (time_after(jiffies, data_comp_to)) {
  98                                         wil_dbg_pm(wil,
  99                                                    "TO waiting for idle tx\n");
 100                                         spin_unlock(&txdata->lock);
 101                                         return false;
 102                                 }
 103                                 wil_dbg_ratelimited(wil,
 104                                                     "tx vring is not empty -> NAPI\n");
 105                                 spin_unlock(&txdata->lock);
 106                                 napi_synchronize(&wil->napi_tx);
 107                                 msleep(20);
 108                                 spin_lock(&txdata->lock);
 109                                 if (!vring->va || !txdata->enabled)
 110                                         break;
 111                         }
 112                 }
 113
 114                 spin_unlock(&txdata->lock);
 115         }
 116
 117         return true;
 118 }
 119
 120 static int wil_vring_alloc(struct wil6210_priv *wil, struct wil_ring *vring)
 121 {
 122         struct device *dev = wil_to_dev(wil);
 123         size_t sz = vring->size * sizeof(vring->va[0]);
 124         uint i;
 125
 126         wil_dbg_misc(wil, "vring_alloc:\n");
 127
 128         BUILD_BUG_ON(sizeof(vring->va[0]) != 32);
 129
 130         vring->swhead = 0;
 131         vring->swtail = 0;
 132         vring->ctx = kcalloc(vring->size, sizeof(vring->ctx[0]), GFP_KERNEL);
 133         if (!vring->ctx) {
 134                 vring->va = NULL;
 135                 return -ENOMEM;
 136         }
 137
 138         /* vring->va should be aligned on its size rounded up to power of 2
 139          * This is granted by the dma_alloc_coherent.
 140          *
 141          * HW has limitation that all vrings addresses must share the same
 142          * upper 16 msb bits part of 48 bits address. To workaround that,
 143          * if we are using more than 32 bit addresses switch to 32 bit
 144          * allocation before allocating vring memory.
 145          *
 146          * There's no check for the return value of dma_set_mask_and_coherent,
 147          * since we assume if we were able to set the mask during
 148          * initialization in this system it will not fail if we set it again
 149          */
 150         if (wil->dma_addr_size > 32)
 151                 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
 152
 153         vring->va = dma_alloc_coherent(dev, sz, &vring->pa, GFP_KERNEL);
 154         if (!vring->va) {
 155                 kfree(vring->ctx);
 156                 vring->ctx = NULL;
 157                 return -ENOMEM;
 158         }
 159
 160         if (wil->dma_addr_size > 32)
 161                 dma_set_mask_and_coherent(dev,
 162                                           DMA_BIT_MASK(wil->dma_addr_size));
 163
 164         /* initially, all descriptors are SW owned
 165          * For Tx and Rx, ownership bit is at the same location, thus
 166          * we can use any
 167          */
 168         for (i = 0; i < vring->size; i++) {
 169                 volatile struct vring_tx_desc *_d =
 170                         &vring->va[i].tx.legacy;
 171
 172                 _d->dma.status = TX_DMA_STATUS_DU;
 173         }
 174
 175         wil_dbg_misc(wil, "vring[%d] 0x%p:%pad 0x%p\n", vring->size,
 176                      vring->va, &vring->pa, vring->ctx);
 177
 178         return 0;
 179 }
 180
 181 static void wil_txdesc_unmap(struct device *dev, union wil_tx_desc *desc,
 182                              struct wil_ctx *ctx)
 183 {
 184         struct vring_tx_desc *d = &desc->legacy;
 185         dma_addr_t pa = wil_desc_addr(&d->dma.addr);
 186         u16 dmalen = le16_to_cpu(d->dma.length);
 187
 188         switch (ctx->mapped_as) {
 189         case wil_mapped_as_single:
 190                 dma_unmap_single(dev, pa, dmalen, DMA_TO_DEVICE);
 191                 break;
 192         case wil_mapped_as_page:
 193                 dma_unmap_page(dev, pa, dmalen, DMA_TO_DEVICE);
 194                 break;
 195         default:
 196                 break;
 197         }
 198 }
 199
 200 static void wil_vring_free(struct wil6210_priv *wil, struct wil_ring *vring)
 201 {
 202         struct device *dev = wil_to_dev(wil);
 203         size_t sz = vring->size * sizeof(vring->va[0]);
 204
 205         lockdep_assert_held(&wil->mutex);
 206         if (!vring->is_rx) {
 207                 int vring_index = vring - wil->ring_tx;
 208
 209                 wil_dbg_misc(wil, "free Tx vring %d [%d] 0x%p:%pad 0x%p\n",
 210                              vring_index, vring->size, vring->va,
 211                              &vring->pa, vring->ctx);
 212         } else {
 213                 wil_dbg_misc(wil, "free Rx vring [%d] 0x%p:%pad 0x%p\n",
 214                              vring->size, vring->va,
 215                              &vring->pa, vring->ctx);
 216         }
 217
 218         while (!wil_ring_is_empty(vring)) {
 219                 dma_addr_t pa;
 220                 u16 dmalen;
 221                 struct wil_ctx *ctx;
 222
 223                 if (!vring->is_rx) {
 224                         struct vring_tx_desc dd, *d = &dd;
 225                         volatile struct vring_tx_desc *_d =
 226                                         &vring->va[vring->swtail].tx.legacy;
 227
 228                         ctx = &vring->ctx[vring->swtail];
 229                         if (!ctx) {
 230                                 wil_dbg_txrx(wil,
 231                                              "ctx(%d) was already completed\n",
 232                                              vring->swtail);
 233                                 vring->swtail = wil_ring_next_tail(vring);
 234                                 continue;
 235                         }
 236                         *d = *_d;
 237                         wil_txdesc_unmap(dev, (union wil_tx_desc *)d, ctx);
 238                         if (ctx->skb)
 239                                 dev_kfree_skb_any(ctx->skb);
 240                         vring->swtail = wil_ring_next_tail(vring);
 241                 } else { /* rx */
 242                         struct vring_rx_desc dd, *d = &dd;
 243                         volatile struct vring_rx_desc *_d =
 244                                 &vring->va[vring->swhead].rx.legacy;
 245
 246                         ctx = &vring->ctx[vring->swhead];
 247                         *d = *_d;
 248                         pa = wil_desc_addr(&d->dma.addr);
 249                         dmalen = le16_to_cpu(d->dma.length);
 250                         dma_unmap_single(dev, pa, dmalen, DMA_FROM_DEVICE);
 251                         kfree_skb(ctx->skb);
 252                         wil_ring_advance_head(vring, 1);
 253                 }
 254         }
 255         dma_free_coherent(dev, sz, (void *)vring->va, vring->pa);
 256         kfree(vring->ctx);
 257         vring->pa = 0;
 258         vring->va = NULL;
 259         vring->ctx = NULL;
 260 }
 261
 262 /**
 263  * Allocate one skb for Rx VRING
 264  *
 265  * Safe to call from IRQ
 266  */
 267 static int wil_vring_alloc_skb(struct wil6210_priv *wil, struct wil_ring *vring,
 268                                u32 i, int headroom)
 269 {
 270         struct device *dev = wil_to_dev(wil);
 271         unsigned int sz = wil->rx_buf_len + ETH_HLEN + wil_rx_snaplen();
 272         struct vring_rx_desc dd, *d = &dd;
 273         volatile struct vring_rx_desc *_d = &vring->va[i].rx.legacy;
 274         dma_addr_t pa;
 275         struct sk_buff *skb = dev_alloc_skb(sz + headroom);
 276
 277         if (unlikely(!skb))
 278                 return -ENOMEM;
 279
 280         skb_reserve(skb, headroom);
 281         skb_put(skb, sz);
 282
 283         /**
 284          * Make sure that the network stack calculates checksum for packets
 285          * which failed the HW checksum calculation
 286          */
 287         skb->ip_summed = CHECKSUM_NONE;
 288
 289         pa = dma_map_single(dev, skb->data, skb->len, DMA_FROM_DEVICE);
 290         if (unlikely(dma_mapping_error(dev, pa))) {
 291                 kfree_skb(skb);
 292                 return -ENOMEM;
 293         }
 294
 295         d->dma.d0 = RX_DMA_D0_CMD_DMA_RT | RX_DMA_D0_CMD_DMA_IT;
 296         wil_desc_addr_set(&d->dma.addr, pa);
 297         /* ip_length don't care */
 298         /* b11 don't care */
 299         /* error don't care */
 300         d->dma.status = 0; /* BIT(0) should be 0 for HW_OWNED */
 301         d->dma.length = cpu_to_le16(sz);
 302         *_d = *d;
 303         vring->ctx[i].skb = skb;
 304
 305         return 0;
 306 }
 307
 308 /**
 309  * Adds radiotap header
 310  *
 311  * Any error indicated as "Bad FCS"
 312  *
 313  * Vendor data for 04:ce:14-1 (Wilocity-1) consists of:
 314  *  - Rx descriptor: 32 bytes
 315  *  - Phy info
 316  */
 317 static void wil_rx_add_radiotap_header(struct wil6210_priv *wil,
 318                                        struct sk_buff *skb)
 319 {
 320         struct wil6210_rtap {
 321                 struct ieee80211_radiotap_header rthdr;
 322                 /* fields should be in the order of bits in rthdr.it_present */
 323                 /* flags */
 324                 u8 flags;
 325                 /* channel */
 326                 __le16 chnl_freq __aligned(2);
 327                 __le16 chnl_flags;
 328                 /* MCS */
 329                 u8 mcs_present;
 330                 u8 mcs_flags;
 331                 u8 mcs_index;
 332         } __packed;
 333         struct vring_rx_desc *d = wil_skb_rxdesc(skb);
 334         struct wil6210_rtap *rtap;
 335         int rtap_len = sizeof(struct wil6210_rtap);
 336         struct ieee80211_channel *ch = wil->monitor_chandef.chan;
 337
 338         if (skb_headroom(skb) < rtap_len &&
 339             pskb_expand_head(skb, rtap_len, 0, GFP_ATOMIC)) {
 340                 wil_err(wil, "Unable to expand headroom to %d\n", rtap_len);
 341                 return;
 342         }
 343
 344         rtap = skb_push(skb, rtap_len);
 345         memset(rtap, 0, rtap_len);
 346
 347         rtap->rthdr.it_version = PKTHDR_RADIOTAP_VERSION;
 348         rtap->rthdr.it_len = cpu_to_le16(rtap_len);
 349         rtap->rthdr.it_present = cpu_to_le32((1 << IEEE80211_RADIOTAP_FLAGS) |
 350                         (1 << IEEE80211_RADIOTAP_CHANNEL) |
 351                         (1 << IEEE80211_RADIOTAP_MCS));
 352         if (d->dma.status & RX_DMA_STATUS_ERROR)
 353                 rtap->flags |= IEEE80211_RADIOTAP_F_BADFCS;
 354
 355         rtap->chnl_freq = cpu_to_le16(ch ? ch->center_freq : 58320);
 356         rtap->chnl_flags = cpu_to_le16(0);
 357
 358         rtap->mcs_present = IEEE80211_RADIOTAP_MCS_HAVE_MCS;
 359         rtap->mcs_flags = 0;
 360         rtap->mcs_index = wil_rxdesc_mcs(d);
 361 }
 362
 363 static bool wil_is_rx_idle(struct wil6210_priv *wil)
 364 {
 365         struct vring_rx_desc *_d;
 366         struct wil_ring *ring = &wil->ring_rx;
 367
 368         _d = (struct vring_rx_desc *)&ring->va[ring->swhead].rx.legacy;
 369         if (_d->dma.status & RX_DMA_STATUS_DU)
 370                 return false;
 371
 372         return true;
 373 }
 374
 375 static int wil_rx_get_cid_by_skb(struct wil6210_priv *wil, struct sk_buff *skb)
 376 {
 377         struct vring_rx_desc *d = wil_skb_rxdesc(skb);
 378         int mid = wil_rxdesc_mid(d);
 379         struct wil6210_vif *vif = wil->vifs[mid];
 380         /* cid from DMA descriptor is limited to 3 bits.
 381          * In case of cid>=8, the value would be cid modulo 8 and we need to
 382          * find real cid by locating the transmitter (ta) inside sta array
 383          */
 384         int cid = wil_rxdesc_cid(d);
 385         unsigned int snaplen = wil_rx_snaplen();
 386         struct ieee80211_hdr_3addr *hdr;
 387         int i;
 388         unsigned char *ta;
 389         u8 ftype;
 390
 391         /* in monitor mode there are no connections */
 392         if (vif->wdev.iftype == NL80211_IFTYPE_MONITOR)
 393                 return cid;
 394
 395         ftype = wil_rxdesc_ftype(d) << 2;
 396         if (likely(ftype == IEEE80211_FTYPE_DATA)) {
 397                 if (unlikely(skb->len < ETH_HLEN + snaplen)) {
 398                         wil_err_ratelimited(wil,
 399                                             "Short data frame, len = %d\n",
 400                                             skb->len);
 401                         return -ENOENT;
 402                 }
 403                 ta = wil_skb_get_sa(skb);
 404         } else {
 405                 if (unlikely(skb->len < sizeof(struct ieee80211_hdr_3addr))) {
 406                         wil_err_ratelimited(wil, "Short frame, len = %d\n",
 407                                             skb->len);
 408                         return -ENOENT;
 409                 }
 410                 hdr = (void *)skb->data;
 411                 ta = hdr->addr2;
 412         }
 413
 414         if (max_assoc_sta <= WIL6210_RX_DESC_MAX_CID)
 415                 return cid;
 416
 417         /* assuming no concurrency between AP interfaces and STA interfaces.
 418          * multista is used only in P2P_GO or AP mode. In other modes return
 419          * cid from the rx descriptor
 420          */
 421         if (vif->wdev.iftype != NL80211_IFTYPE_P2P_GO &&
 422             vif->wdev.iftype != NL80211_IFTYPE_AP)
 423                 return cid;
 424
 425         /* For Rx packets cid from rx descriptor is limited to 3 bits (0..7),
 426          * to find the real cid, compare transmitter address with the stored
 427          * stations mac address in the driver sta array
 428          */
 429         for (i = cid; i < max_assoc_sta; i += WIL6210_RX_DESC_MAX_CID) {
 430                 if (wil->sta[i].status != wil_sta_unused &&
 431                     ether_addr_equal(wil->sta[i].addr, ta)) {
 432                         cid = i;
 433                         break;
 434                 }
 435         }
 436         if (i >= max_assoc_sta) {
 437                 wil_err_ratelimited(wil, "Could not find cid for frame with transmit addr = %pM, iftype = %d, frametype = %d, len = %d\n",
 438                                     ta, vif->wdev.iftype, ftype, skb->len);
 439                 cid = -ENOENT;
 440         }
 441
 442         return cid;
 443 }
 444
 445 /**
 446  * reap 1 frame from @swhead
 447  *
 448  * Rx descriptor copied to skb->cb
 449  *
 450  * Safe to call from IRQ
 451  */
 452 static struct sk_buff *wil_vring_reap_rx(struct wil6210_priv *wil,
 453                                          struct wil_ring *vring)
 454 {
 455         struct device *dev = wil_to_dev(wil);
 456         struct wil6210_vif *vif;
 457         struct net_device *ndev;
 458         volatile struct vring_rx_desc *_d;
 459         struct vring_rx_desc *d;
 460         struct sk_buff *skb;
 461         dma_addr_t pa;
 462         unsigned int snaplen = wil_rx_snaplen();
 463         unsigned int sz = wil->rx_buf_len + ETH_HLEN + snaplen;
 464         u16 dmalen;
 465         u8 ftype;
 466         int cid, mid;
 467         int i;
 468         struct wil_net_stats *stats;
 469
 470         BUILD_BUG_ON(sizeof(struct skb_rx_info) > sizeof(skb->cb));
 471
 472 again:
 473         if (unlikely(wil_ring_is_empty(vring)))
 474                 return NULL;
 475
 476         i = (int)vring->swhead;
 477         _d = &vring->va[i].rx.legacy;
 478         if (unlikely(!(_d->dma.status & RX_DMA_STATUS_DU))) {
 479                 /* it is not error, we just reached end of Rx done area */
 480                 return NULL;
 481         }
 482
 483         skb = vring->ctx[i].skb;
 484         vring->ctx[i].skb = NULL;
 485         wil_ring_advance_head(vring, 1);
 486         if (!skb) {
 487                 wil_err(wil, "No Rx skb at [%d]\n", i);
 488                 goto again;
 489         }
 490         d = wil_skb_rxdesc(skb);
 491         *d = *_d;
 492         pa = wil_desc_addr(&d->dma.addr);
 493
 494         dma_unmap_single(dev, pa, sz, DMA_FROM_DEVICE);
 495         dmalen = le16_to_cpu(d->dma.length);
 496
 497         trace_wil6210_rx(i, d);
 498         wil_dbg_txrx(wil, "Rx[%3d] : %d bytes\n", i, dmalen);
 499         wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE, 32, 4,
 500                           (const void *)d, sizeof(*d), false);
 501
 502         mid = wil_rxdesc_mid(d);
 503         vif = wil->vifs[mid];
 504
 505         if (unlikely(!vif)) {
 506                 wil_dbg_txrx(wil, "skipped RX descriptor with invalid mid %d",
 507                              mid);
 508                 kfree_skb(skb);
 509                 goto again;
 510         }
 511         ndev = vif_to_ndev(vif);
 512         if (unlikely(dmalen > sz)) {
 513                 wil_err_ratelimited(wil, "Rx size too large: %d bytes!\n",
 514                                     dmalen);
 515                 kfree_skb(skb);
 516                 goto again;
 517         }
 518         skb_trim(skb, dmalen);
 519
 520         prefetch(skb->data);
 521
 522         wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1,
 523                           skb->data, skb_headlen(skb), false);
 524
 525         cid = wil_rx_get_cid_by_skb(wil, skb);
 526         if (cid == -ENOENT) {
 527                 kfree_skb(skb);
 528                 goto again;
 529         }
 530         wil_skb_set_cid(skb, (u8)cid);
 531         stats = &wil->sta[cid].stats;
 532
 533         stats->last_mcs_rx = wil_rxdesc_mcs(d);
 534         if (stats->last_mcs_rx < ARRAY_SIZE(stats->rx_per_mcs))
 535                 stats->rx_per_mcs[stats->last_mcs_rx]++;
 536
 537         /* use radiotap header only if required */
 538         if (ndev->type == ARPHRD_IEEE80211_RADIOTAP)
 539                 wil_rx_add_radiotap_header(wil, skb);
 540
 541         /* no extra checks if in sniffer mode */
 542         if (ndev->type != ARPHRD_ETHER)
 543                 return skb;
 544         /* Non-data frames may be delivered through Rx DMA channel (ex: BAR)
 545          * Driver should recognize it by frame type, that is found
 546          * in Rx descriptor. If type is not data, it is 802.11 frame as is
 547          */
 548         ftype = wil_rxdesc_ftype(d) << 2;
 549         if (unlikely(ftype != IEEE80211_FTYPE_DATA)) {
 550                 u8 fc1 = wil_rxdesc_fc1(d);
 551                 int tid = wil_rxdesc_tid(d);
 552                 u16 seq = wil_rxdesc_seq(d);
 553
 554                 wil_dbg_txrx(wil,
 555                              "Non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n",
 556                              fc1, mid, cid, tid, seq);
 557                 stats->rx_non_data_frame++;
 558                 if (wil_is_back_req(fc1)) {
 559                         wil_dbg_txrx(wil,
 560                                      "BAR: MID %d CID %d TID %d Seq 0x%03x\n",
 561                                      mid, cid, tid, seq);
 562                         wil_rx_bar(wil, vif, cid, tid, seq);
 563                 } else {
 564                         /* print again all info. One can enable only this
 565                          * without overhead for printing every Rx frame
 566                          */
 567                         wil_dbg_txrx(wil,
 568                                      "Unhandled non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n",
 569                                      fc1, mid, cid, tid, seq);
 570                         wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE, 32, 4,
 571                                           (const void *)d, sizeof(*d), false);
 572                         wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1,
 573                                           skb->data, skb_headlen(skb), false);
 574                 }
 575                 kfree_skb(skb);
 576                 goto again;
 577         }
 578
 579         /* L4 IDENT is on when HW calculated checksum, check status
 580          * and in case of error drop the packet
 581          * higher stack layers will handle retransmission (if required)
 582          */
 583         if (likely(d->dma.status & RX_DMA_STATUS_L4I)) {
 584                 /* L4 protocol identified, csum calculated */
 585                 if (likely((d->dma.error & RX_DMA_ERROR_L4_ERR) == 0))
 586                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 587                 /* If HW reports bad checksum, let IP stack re-check it
 588                  * For example, HW don't understand Microsoft IP stack that
 589                  * mis-calculates TCP checksum - if it should be 0x0,
 590                  * it writes 0xffff in violation of RFC 1624
 591                  */
 592                 else
 593                         stats->rx_csum_err++;
 594         }
 595
 596         if (snaplen) {
 597                 /* Packet layout
 598                  * +-------+-------+---------+------------+------+
 599                  * | SA(6) | DA(6) | SNAP(6) | ETHTYPE(2) | DATA |
 600                  * +-------+-------+---------+------------+------+
 601                  * Need to remove SNAP, shifting SA and DA forward
 602                  */
 603                 memmove(skb->data + snaplen, skb->data, 2 * ETH_ALEN);
 604                 skb_pull(skb, snaplen);
 605         }
 606
 607         return skb;
 608 }
 609
 610 /**
 611  * allocate and fill up to @count buffers in rx ring
 612  * buffers posted at @swtail
 613  * Note: we have a single RX queue for servicing all VIFs, but we
 614  * allocate skbs with headroom according to main interface only. This
 615  * means it will not work with monitor interface together with other VIFs.
 616  * Currently we only support monitor interface on its own without other VIFs,
 617  * and we will need to fix this code once we add support.
 618  */
 619 static int wil_rx_refill(struct wil6210_priv *wil, int count)
 620 {
 621         struct net_device *ndev = wil->main_ndev;
 622         struct wil_ring *v = &wil->ring_rx;
 623         u32 next_tail;
 624         int rc = 0;
 625         int headroom = ndev->type == ARPHRD_IEEE80211_RADIOTAP ?
 626                         WIL6210_RTAP_SIZE : 0;
 627
 628         for (; next_tail = wil_ring_next_tail(v),
 629              (next_tail != v->swhead) && (count-- > 0);
 630              v->swtail = next_tail) {
 631                 rc = wil_vring_alloc_skb(wil, v, v->swtail, headroom);
 632                 if (unlikely(rc)) {
 633                         wil_err_ratelimited(wil, "Error %d in rx refill[%d]\n",
 634                                             rc, v->swtail);
 635                         break;
 636                 }
 637         }
 638
 639         /* make sure all writes to descriptors (shared memory) are done before
 640          * committing them to HW
 641          */
 642         wmb();
 643
 644         wil_w(wil, v->hwtail, v->swtail);
 645
 646         return rc;
 647 }
 648
 649 /**
 650  * reverse_memcmp - Compare two areas of memory, in reverse order
 651  * @cs: One area of memory
 652  * @ct: Another area of memory
 653  * @count: The size of the area.
 654  *
 655  * Cut'n'paste from original memcmp (see lib/string.c)
 656  * with minimal modifications
 657  */
 658 int reverse_memcmp(const void *cs, const void *ct, size_t count)
 659 {
 660         const unsigned char *su1, *su2;
 661         int res = 0;
 662
 663         for (su1 = cs + count - 1, su2 = ct + count - 1; count > 0;
 664              --su1, --su2, count--) {
 665                 res = *su1 - *su2;
 666                 if (res)
 667                         break;
 668         }
 669         return res;
 670 }
 671
 672 static int wil_rx_crypto_check(struct wil6210_priv *wil, struct sk_buff *skb)
 673 {
 674         struct vring_rx_desc *d = wil_skb_rxdesc(skb);
 675         int cid = wil_skb_get_cid(skb);
 676         int tid = wil_rxdesc_tid(d);
 677         int key_id = wil_rxdesc_key_id(d);
 678         int mc = wil_rxdesc_mcast(d);
 679         struct wil_sta_info *s = &wil->sta[cid];
 680         struct wil_tid_crypto_rx *c = mc ? &s->group_crypto_rx :
 681                                       &s->tid_crypto_rx[tid];
 682         struct wil_tid_crypto_rx_single *cc = &c->key_id[key_id];
 683         const u8 *pn = (u8 *)&d->mac.pn_15_0;
 684
 685         if (!cc->key_set) {
 686                 wil_err_ratelimited(wil,
 687                                     "Key missing. CID %d TID %d MCast %d KEY_ID %d\n",
 688                                     cid, tid, mc, key_id);
 689                 return -EINVAL;
 690         }
 691
 692         if (reverse_memcmp(pn, cc->pn, IEEE80211_GCMP_PN_LEN) <= 0) {
 693                 wil_err_ratelimited(wil,
 694                                     "Replay attack. CID %d TID %d MCast %d KEY_ID %d PN %6phN last %6phN\n",
 695                                     cid, tid, mc, key_id, pn, cc->pn);
 696                 return -EINVAL;
 697         }
 698         memcpy(cc->pn, pn, IEEE80211_GCMP_PN_LEN);
 699
 700         return 0;
 701 }
 702
 703 static int wil_rx_error_check(struct wil6210_priv *wil, struct sk_buff *skb,
 704                               struct wil_net_stats *stats)
 705 {
 706         struct vring_rx_desc *d = wil_skb_rxdesc(skb);
 707
 708         if ((d->dma.status & RX_DMA_STATUS_ERROR) &&
 709             (d->dma.error & RX_DMA_ERROR_MIC)) {
 710                 stats->rx_mic_error++;
 711                 wil_dbg_txrx(wil, "MIC error, dropping packet\n");
 712                 return -EFAULT;
 713         }
 714
 715         return 0;
 716 }
 717
 718 static void wil_get_netif_rx_params(struct sk_buff *skb, int *cid,
 719                                     int *security)
 720 {
 721         struct vring_rx_desc *d = wil_skb_rxdesc(skb);
 722
 723         *cid = wil_skb_get_cid(skb);
 724         *security = wil_rxdesc_security(d);
 725 }
 726
 727 /*
 728  * Pass Rx packet to the netif. Update statistics.
 729  * Called in softirq context (NAPI poll).
 730  */
 731 void wil_netif_rx_any(struct sk_buff *skb, struct net_device *ndev)
 732 {
 733         gro_result_t rc = GRO_NORMAL;
 734         struct wil6210_vif *vif = ndev_to_vif(ndev);
 735         struct wil6210_priv *wil = ndev_to_wil(ndev);
 736         struct wireless_dev *wdev = vif_to_wdev(vif);
 737         unsigned int len = skb->len;
 738         int cid;
 739         int security;
 740         u8 *sa, *da = wil_skb_get_da(skb);
 741         /* here looking for DA, not A1, thus Rxdesc's 'mcast' indication
 742          * is not suitable, need to look at data
 743          */
 744         int mcast = is_multicast_ether_addr(da);
 745         struct wil_net_stats *stats;
 746         struct sk_buff *xmit_skb = NULL;
 747         static const char * const gro_res_str[] = {
 748                 [GRO_MERGED]            = "GRO_MERGED",
 749                 [GRO_MERGED_FREE]       = "GRO_MERGED_FREE",
 750                 [GRO_HELD]              = "GRO_HELD",
 751                 [GRO_NORMAL]            = "GRO_NORMAL",
 752                 [GRO_DROP]              = "GRO_DROP",
 753                 [GRO_CONSUMED]          = "GRO_CONSUMED",
 754         };
 755
 756         wil->txrx_ops.get_netif_rx_params(skb, &cid, &security);
 757
 758         stats = &wil->sta[cid].stats;
 759
 760         skb_orphan(skb);
 761
 762         if (security && (wil->txrx_ops.rx_crypto_check(wil, skb) != 0)) {
 763                 rc = GRO_DROP;
 764                 dev_kfree_skb(skb);
 765                 stats->rx_replay++;
 766                 goto stats;
 767         }
 768
 769         /* check errors reported by HW and update statistics */
 770         if (unlikely(wil->txrx_ops.rx_error_check(wil, skb, stats))) {
 771                 dev_kfree_skb(skb);
 772                 return;
 773         }
 774
 775         if (wdev->iftype == NL80211_IFTYPE_STATION) {
 776                 sa = wil_skb_get_sa(skb);
 777                 if (mcast && ether_addr_equal(sa, ndev->dev_addr)) {
 778                         /* mcast packet looped back to us */
 779                         rc = GRO_DROP;
 780                         dev_kfree_skb(skb);
 781                         goto stats;
 782                 }
 783         } else if (wdev->iftype == NL80211_IFTYPE_AP && !vif->ap_isolate) {
 784                 if (mcast) {
 785                         /* send multicast frames both to higher layers in
 786                          * local net stack and back to the wireless medium
 787                          */
 788                         xmit_skb = skb_copy(skb, GFP_ATOMIC);
 789                 } else {
 790                         int xmit_cid = wil_find_cid(wil, vif->mid, da);
 791
 792                         if (xmit_cid >= 0) {
 793                                 /* The destination station is associated to
 794                                  * this AP (in this VLAN), so send the frame
 795                                  * directly to it and do not pass it to local
 796                                  * net stack.
 797                                  */
 798                                 xmit_skb = skb;
 799                                 skb = NULL;
 800                         }
 801                 }
 802         }
 803         if (xmit_skb) {
 804                 /* Send to wireless media and increase priority by 256 to
 805                  * keep the received priority instead of reclassifying
 806                  * the frame (see cfg80211_classify8021d).
 807                  */
 808                 xmit_skb->dev = ndev;
 809                 xmit_skb->priority += 256;
 810                 xmit_skb->protocol = htons(ETH_P_802_3);
 811                 skb_reset_network_header(xmit_skb);
 812                 skb_reset_mac_header(xmit_skb);
 813                 wil_dbg_txrx(wil, "Rx -> Tx %d bytes\n", len);
 814                 dev_queue_xmit(xmit_skb);
 815         }
 816
 817         if (skb) { /* deliver to local stack */
 818                 skb->protocol = eth_type_trans(skb, ndev);
 819                 skb->dev = ndev;
 820                 rc = napi_gro_receive(&wil->napi_rx, skb);
 821                 wil_dbg_txrx(wil, "Rx complete %d bytes => %s\n",
 822                              len, gro_res_str[rc]);
 823         }
 824 stats:
 825         /* statistics. rc set to GRO_NORMAL for AP bridging */
 826         if (unlikely(rc == GRO_DROP)) {
 827                 ndev->stats.rx_dropped++;
 828                 stats->rx_dropped++;
 829                 wil_dbg_txrx(wil, "Rx drop %d bytes\n", len);
 830         } else {
 831                 ndev->stats.rx_packets++;
 832                 stats->rx_packets++;
 833                 ndev->stats.rx_bytes += len;
 834                 stats->rx_bytes += len;
 835                 if (mcast)
 836                         ndev->stats.multicast++;
 837         }
 838 }
 839
 840 /**
 841  * Proceed all completed skb's from Rx VRING
 842  *
 843  * Safe to call from NAPI poll, i.e. softirq with interrupts enabled
 844  */
 845 void wil_rx_handle(struct wil6210_priv *wil, int *quota)
 846 {
 847         struct net_device *ndev = wil->main_ndev;
 848         struct wireless_dev *wdev = ndev->ieee80211_ptr;
 849         struct wil_ring *v = &wil->ring_rx;
 850         struct sk_buff *skb;
 851
 852         if (unlikely(!v->va)) {
 853                 wil_err(wil, "Rx IRQ while Rx not yet initialized\n");
 854                 return;
 855         }
 856         wil_dbg_txrx(wil, "rx_handle\n");
 857         while ((*quota > 0) && (NULL != (skb = wil_vring_reap_rx(wil, v)))) {
 858                 (*quota)--;
 859
 860                 /* monitor is currently supported on main interface only */
 861                 if (wdev->iftype == NL80211_IFTYPE_MONITOR) {
 862                         skb->dev = ndev;
 863                         skb_reset_mac_header(skb);
 864                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 865                         skb->pkt_type = PACKET_OTHERHOST;
 866                         skb->protocol = htons(ETH_P_802_2);
 867                         wil_netif_rx_any(skb, ndev);
 868                 } else {
 869                         wil_rx_reorder(wil, skb);
 870                 }
 871         }
 872         wil_rx_refill(wil, v->size);
 873 }
 874
 875 static void wil_rx_buf_len_init(struct wil6210_priv *wil)
 876 {
 877         wil->rx_buf_len = rx_large_buf ?
 878                 WIL_MAX_ETH_MTU : TXRX_BUF_LEN_DEFAULT - WIL_MAX_MPDU_OVERHEAD;
 879         if (mtu_max > wil->rx_buf_len) {
 880                 /* do not allow RX buffers to be smaller than mtu_max, for
 881                  * backward compatibility (mtu_max parameter was also used
 882                  * to support receiving large packets)
 883                  */
 884                 wil_info(wil, "Override RX buffer to mtu_max(%d)\n", mtu_max);
 885                 wil->rx_buf_len = mtu_max;
 886         }
 887 }
 888
 889 static int wil_rx_init(struct wil6210_priv *wil, uint order)
 890 {
 891         struct wil_ring *vring = &wil->ring_rx;
 892         int rc;
 893
 894         wil_dbg_misc(wil, "rx_init\n");
 895
 896         if (vring->va) {
 897                 wil_err(wil, "Rx ring already allocated\n");
 898                 return -EINVAL;
 899         }
 900
 901         wil_rx_buf_len_init(wil);
 902
 903         vring->size = 1 << order;
 904         vring->is_rx = true;
 905         rc = wil_vring_alloc(wil, vring);
 906         if (rc)
 907                 return rc;
 908
 909         rc = wmi_rx_chain_add(wil, vring);
 910         if (rc)
 911                 goto err_free;
 912
 913         rc = wil_rx_refill(wil, vring->size);
 914         if (rc)
 915                 goto err_free;
 916
 917         return 0;
 918  err_free:
 919         wil_vring_free(wil, vring);
 920
 921         return rc;
 922 }
 923
 924 static void wil_rx_fini(struct wil6210_priv *wil)
 925 {
 926         struct wil_ring *vring = &wil->ring_rx;
 927
 928         wil_dbg_misc(wil, "rx_fini\n");
 929
 930         if (vring->va)
 931                 wil_vring_free(wil, vring);
 932 }
 933
 934 static int wil_tx_desc_map(union wil_tx_desc *desc, dma_addr_t pa,
 935                            u32 len, int vring_index)
 936 {
 937         struct vring_tx_desc *d = &desc->legacy;
 938
 939         wil_desc_addr_set(&d->dma.addr, pa);
 940         d->dma.ip_length = 0;
 941         /* 0..6: mac_length; 7:ip_version 0-IP6 1-IP4*/
 942         d->dma.b11 = 0/*14 | BIT(7)*/;
 943         d->dma.error = 0;
 944         d->dma.status = 0; /* BIT(0) should be 0 for HW_OWNED */
 945         d->dma.length = cpu_to_le16((u16)len);
 946         d->dma.d0 = (vring_index << DMA_CFG_DESC_TX_0_QID_POS);
 947         d->mac.d[0] = 0;
 948         d->mac.d[1] = 0;
 949         d->mac.d[2] = 0;
 950         d->mac.ucode_cmd = 0;
 951         /* translation type:  0 - bypass; 1 - 802.3; 2 - native wifi */
 952         d->mac.d[2] = BIT(MAC_CFG_DESC_TX_2_SNAP_HDR_INSERTION_EN_POS) |
 953                       (1 << MAC_CFG_DESC_TX_2_L2_TRANSLATION_TYPE_POS);
 954
 955         return 0;
 956 }
 957
 958 void wil_tx_data_init(struct wil_ring_tx_data *txdata)
 959 {
 960         spin_lock_bh(&txdata->lock);
 961         txdata->dot1x_open = 0;
 962         txdata->enabled = 0;
 963         txdata->idle = 0;
 964         txdata->last_idle = 0;
 965         txdata->begin = 0;
 966         txdata->agg_wsize = 0;
 967         txdata->agg_timeout = 0;
 968         txdata->agg_amsdu = 0;
 969         txdata->addba_in_progress = false;
 970         txdata->mid = U8_MAX;
 971         spin_unlock_bh(&txdata->lock);
 972 }
 973
 974 static int wil_vring_init_tx(struct wil6210_vif *vif, int id, int size,
 975                              int cid, int tid)
 976 {
 977         struct wil6210_priv *wil = vif_to_wil(vif);
 978         int rc;
 979         struct wmi_vring_cfg_cmd cmd = {
 980                 .action = cpu_to_le32(WMI_VRING_CMD_ADD),
 981                 .vring_cfg = {
 982                         .tx_sw_ring = {
 983                                 .max_mpdu_size =
 984                                         cpu_to_le16(wil_mtu2macbuf(mtu_max)),
 985                                 .ring_size = cpu_to_le16(size),
 986                         },
 987                         .ringid = id,
 988                         .encap_trans_type = WMI_VRING_ENC_TYPE_802_3,
 989                         .mac_ctrl = 0,
 990                         .to_resolution = 0,
 991                         .agg_max_wsize = 0,
 992                         .schd_params = {
 993                                 .priority = cpu_to_le16(0),
 994                                 .timeslot_us = cpu_to_le16(0xfff),
 995                         },
 996                 },
 997         };
 998         struct {
 999                 struct wmi_cmd_hdr wmi;
1000                 struct wmi_vring_cfg_done_event cmd;
1001         } __packed reply = {
1002                 .cmd = {.status = WMI_FW_STATUS_FAILURE},
1003         };
1004         struct wil_ring *vring = &wil->ring_tx[id];
1005         struct wil_ring_tx_data *txdata = &wil->ring_tx_data[id];
1006
1007         if (cid >= WIL6210_RX_DESC_MAX_CID) {
1008                 cmd.vring_cfg.cidxtid = CIDXTID_EXTENDED_CID_TID;
1009                 cmd.vring_cfg.cid = cid;
1010                 cmd.vring_cfg.tid = tid;
1011         } else {
1012                 cmd.vring_cfg.cidxtid = mk_cidxtid(cid, tid);
1013         }
1014
1015         wil_dbg_misc(wil, "vring_init_tx: max_mpdu_size %d\n",
1016                      cmd.vring_cfg.tx_sw_ring.max_mpdu_size);
1017         lockdep_assert_held(&wil->mutex);
1018
1019         if (vring->va) {
1020                 wil_err(wil, "Tx ring [%d] already allocated\n", id);
1021                 rc = -EINVAL;
1022                 goto out;
1023         }
1024
1025         wil_tx_data_init(txdata);
1026         vring->is_rx = false;
1027         vring->size = size;
1028         rc = wil_vring_alloc(wil, vring);
1029         if (rc)
1030                 goto out;
1031
1032         wil->ring2cid_tid[id][0] = cid;
1033         wil->ring2cid_tid[id][1] = tid;
1034
1035         cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa);
1036
1037         if (!vif->privacy)
1038                 txdata->dot1x_open = true;
1039         rc = wmi_call(wil, WMI_VRING_CFG_CMDID, vif->mid, &cmd, sizeof(cmd),
1040                       WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply), 100);
1041         if (rc)
1042                 goto out_free;
1043
1044         if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) {
1045                 wil_err(wil, "Tx config failed, status 0x%02x\n",
1046                         reply.cmd.status);
1047                 rc = -EINVAL;
1048                 goto out_free;
1049         }
1050
1051         spin_lock_bh(&txdata->lock);
1052         vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr);
1053         txdata->mid = vif->mid;
1054         txdata->enabled = 1;
1055         spin_unlock_bh(&txdata->lock);
1056
1057         if (txdata->dot1x_open && (agg_wsize >= 0))
1058                 wil_addba_tx_request(wil, id, agg_wsize);
1059
1060         return 0;
1061  out_free:
1062         spin_lock_bh(&txdata->lock);
1063         txdata->dot1x_open = false;
1064         txdata->enabled = 0;
1065         spin_unlock_bh(&txdata->lock);
1066         wil_vring_free(wil, vring);
1067         wil->ring2cid_tid[id][0] = max_assoc_sta;
1068         wil->ring2cid_tid[id][1] = 0;
1069
1070  out:
1071
1072         return rc;
1073 }
1074
1075 static int wil_tx_vring_modify(struct wil6210_vif *vif, int ring_id, int cid,
1076                                int tid)
1077 {
1078         struct wil6210_priv *wil = vif_to_wil(vif);
1079         int rc;
1080         struct wmi_vring_cfg_cmd cmd = {
1081                 .action = cpu_to_le32(WMI_VRING_CMD_MODIFY),
1082                 .vring_cfg = {
1083                         .tx_sw_ring = {
1084                                 .max_mpdu_size =
1085                                         cpu_to_le16(wil_mtu2macbuf(mtu_max)),
1086                                 .ring_size = 0,
1087                         },
1088                         .ringid = ring_id,
1089                         .cidxtid = mk_cidxtid(cid, tid),
1090                         .encap_trans_type = WMI_VRING_ENC_TYPE_802_3,
1091                         .mac_ctrl = 0,
1092                         .to_resolution = 0,
1093                         .agg_max_wsize = 0,
1094                         .schd_params = {
1095                                 .priority = cpu_to_le16(0),
1096                                 .timeslot_us = cpu_to_le16(0xfff),
1097                         },
1098                 },
1099         };
1100         struct {
1101                 struct wmi_cmd_hdr wmi;
1102                 struct wmi_vring_cfg_done_event cmd;
1103         } __packed reply = {
1104                 .cmd = {.status = WMI_FW_STATUS_FAILURE},
1105         };
1106         struct wil_ring *vring = &wil->ring_tx[ring_id];
1107         struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_id];
1108
1109         wil_dbg_misc(wil, "vring_modify: ring %d cid %d tid %d\n", ring_id,
1110                      cid, tid);
1111         lockdep_assert_held(&wil->mutex);
1112
1113         if (!vring->va) {
1114                 wil_err(wil, "Tx ring [%d] not allocated\n", ring_id);
1115                 return -EINVAL;
1116         }
1117
1118         if (wil->ring2cid_tid[ring_id][0] != cid ||
1119             wil->ring2cid_tid[ring_id][1] != tid) {
1120                 wil_err(wil, "ring info does not match cid=%u tid=%u\n",
1121                         wil->ring2cid_tid[ring_id][0],
1122                         wil->ring2cid_tid[ring_id][1]);
1123         }
1124
1125         cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa);
1126
1127         rc = wmi_call(wil, WMI_VRING_CFG_CMDID, vif->mid, &cmd, sizeof(cmd),
1128                       WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply), 100);
1129         if (rc)
1130                 goto fail;
1131
1132         if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) {
1133                 wil_err(wil, "Tx modify failed, status 0x%02x\n",
1134                         reply.cmd.status);
1135                 rc = -EINVAL;
1136                 goto fail;
1137         }
1138
1139         /* set BA aggregation window size to 0 to force a new BA with the
1140          * new AP
1141          */
1142         txdata->agg_wsize = 0;
1143         if (txdata->dot1x_open && agg_wsize >= 0)
1144                 wil_addba_tx_request(wil, ring_id, agg_wsize);
1145
1146         return 0;
1147 fail:
1148         spin_lock_bh(&txdata->lock);
1149         txdata->dot1x_open = false;
1150         txdata->enabled = 0;
1151         spin_unlock_bh(&txdata->lock);
1152         wil->ring2cid_tid[ring_id][0] = max_assoc_sta;
1153         wil->ring2cid_tid[ring_id][1] = 0;
1154         return rc;
1155 }
1156
1157 int wil_vring_init_bcast(struct wil6210_vif *vif, int id, int size)
1158 {
1159         struct wil6210_priv *wil = vif_to_wil(vif);
1160         int rc;
1161         struct wmi_bcast_vring_cfg_cmd cmd = {
1162                 .action = cpu_to_le32(WMI_VRING_CMD_ADD),
1163                 .vring_cfg = {
1164                         .tx_sw_ring = {
1165                                 .max_mpdu_size =
1166                                         cpu_to_le16(wil_mtu2macbuf(mtu_max)),
1167                                 .ring_size = cpu_to_le16(size),
1168                         },
1169                         .ringid = id,
1170                         .encap_trans_type = WMI_VRING_ENC_TYPE_802_3,
1171                 },
1172         };
1173         struct {
1174                 struct wmi_cmd_hdr wmi;
1175                 struct wmi_vring_cfg_done_event cmd;
1176         } __packed reply = {
1177                 .cmd = {.status = WMI_FW_STATUS_FAILURE},
1178         };
1179         struct wil_ring *vring = &wil->ring_tx[id];
1180         struct wil_ring_tx_data *txdata = &wil->ring_tx_data[id];
1181
1182         wil_dbg_misc(wil, "vring_init_bcast: max_mpdu_size %d\n",
1183                      cmd.vring_cfg.tx_sw_ring.max_mpdu_size);
1184         lockdep_assert_held(&wil->mutex);
1185
1186         if (vring->va) {
1187                 wil_err(wil, "Tx ring [%d] already allocated\n", id);
1188                 rc = -EINVAL;
1189                 goto out;
1190         }
1191
1192         wil_tx_data_init(txdata);
1193         vring->is_rx = false;
1194         vring->size = size;
1195         rc = wil_vring_alloc(wil, vring);
1196         if (rc)
1197                 goto out;
1198
1199         wil->ring2cid_tid[id][0] = max_assoc_sta; /* CID */
1200         wil->ring2cid_tid[id][1] = 0; /* TID */
1201
1202         cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa);
1203
1204         if (!vif->privacy)
1205                 txdata->dot1x_open = true;
1206         rc = wmi_call(wil, WMI_BCAST_VRING_CFG_CMDID, vif->mid,
1207                       &cmd, sizeof(cmd),
1208                       WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply), 100);
1209         if (rc)
1210                 goto out_free;
1211
1212         if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) {
1213                 wil_err(wil, "Tx config failed, status 0x%02x\n",
1214                         reply.cmd.status);
1215                 rc = -EINVAL;
1216                 goto out_free;
1217         }
1218
1219         spin_lock_bh(&txdata->lock);
1220         vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr);
1221         txdata->mid = vif->mid;
1222         txdata->enabled = 1;
1223         spin_unlock_bh(&txdata->lock);
1224
1225         return 0;
1226  out_free:
1227         spin_lock_bh(&txdata->lock);
1228         txdata->enabled = 0;
1229         txdata->dot1x_open = false;
1230         spin_unlock_bh(&txdata->lock);
1231         wil_vring_free(wil, vring);
1232  out:
1233
1234         return rc;
1235 }
1236
1237 static struct wil_ring *wil_find_tx_ucast(struct wil6210_priv *wil,
1238                                           struct wil6210_vif *vif,
1239                                           struct sk_buff *skb)
1240 {
1241         int i, cid;
1242         const u8 *da = wil_skb_get_da(skb);
1243         int min_ring_id = wil_get_min_tx_ring_id(wil);
1244
1245         cid = wil_find_cid(wil, vif->mid, da);
1246
1247         if (cid < 0 || cid >= max_assoc_sta)
1248                 return NULL;
1249
1250         /* TODO: fix for multiple TID */
1251         for (i = min_ring_id; i < ARRAY_SIZE(wil->ring2cid_tid); i++) {
1252                 if (!wil->ring_tx_data[i].dot1x_open &&
1253                     skb->protocol != cpu_to_be16(ETH_P_PAE))
1254                         continue;
1255                 if (wil->ring2cid_tid[i][0] == cid) {
1256                         struct wil_ring *v = &wil->ring_tx[i];
1257                         struct wil_ring_tx_data *txdata = &wil->ring_tx_data[i];
1258
1259                         wil_dbg_txrx(wil, "find_tx_ucast: (%pM) -> [%d]\n",
1260                                      da, i);
1261                         if (v->va && txdata->enabled) {
1262                                 return v;
1263                         } else {
1264                                 wil_dbg_txrx(wil,
1265                                              "find_tx_ucast: vring[%d] not valid\n",
1266                                              i);
1267                                 return NULL;
1268                         }
1269                 }
1270         }
1271
1272         return NULL;
1273 }
1274
1275 static int wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,
1276                        struct wil_ring *ring, struct sk_buff *skb);
1277
1278 static struct wil_ring *wil_find_tx_ring_sta(struct wil6210_priv *wil,
1279                                              struct wil6210_vif *vif,
1280                                              struct sk_buff *skb)
1281 {
1282         struct wil_ring *ring;
1283         int i;
1284         u8 cid;
1285         struct wil_ring_tx_data  *txdata;
1286         int min_ring_id = wil_get_min_tx_ring_id(wil);
1287
1288         /* In the STA mode, it is expected to have only 1 VRING
1289          * for the AP we connected to.
1290          * find 1-st vring eligible for this skb and use it.
1291          */
1292         for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
1293                 ring = &wil->ring_tx[i];
1294                 txdata = &wil->ring_tx_data[i];
1295                 if (!ring->va || !txdata->enabled || txdata->mid != vif->mid)
1296                         continue;
1297
1298                 cid = wil->ring2cid_tid[i][0];
1299                 if (cid >= max_assoc_sta) /* skip BCAST */
1300                         continue;
1301
1302                 if (!wil->ring_tx_data[i].dot1x_open &&
1303                     skb->protocol != cpu_to_be16(ETH_P_PAE))
1304                         continue;
1305
1306                 wil_dbg_txrx(wil, "Tx -> ring %d\n", i);
1307
1308                 return ring;
1309         }
1310
1311         wil_dbg_txrx(wil, "Tx while no rings active?\n");
1312
1313         return NULL;
1314 }
1315
1316 /* Use one of 2 strategies:
1317  *
1318  * 1. New (real broadcast):
1319  *    use dedicated broadcast vring
1320  * 2. Old (pseudo-DMS):
1321  *    Find 1-st vring and return it;
1322  *    duplicate skb and send it to other active vrings;
1323  *    in all cases override dest address to unicast peer's address
1324  * Use old strategy when new is not supported yet:
1325  *  - for PBSS
1326  */
1327 static struct wil_ring *wil_find_tx_bcast_1(struct wil6210_priv *wil,
1328                                             struct wil6210_vif *vif,
1329                                             struct sk_buff *skb)
1330 {
1331         struct wil_ring *v;
1332         struct wil_ring_tx_data *txdata;
1333         int i = vif->bcast_ring;
1334
1335         if (i < 0)
1336                 return NULL;
1337         v = &wil->ring_tx[i];
1338         txdata = &wil->ring_tx_data[i];
1339         if (!v->va || !txdata->enabled)
1340                 return NULL;
1341         if (!wil->ring_tx_data[i].dot1x_open &&
1342             skb->protocol != cpu_to_be16(ETH_P_PAE))
1343                 return NULL;
1344
1345         return v;
1346 }
1347
1348 static void wil_set_da_for_vring(struct wil6210_priv *wil,
1349                                  struct sk_buff *skb, int vring_index)
1350 {
1351         u8 *da = wil_skb_get_da(skb);
1352         int cid = wil->ring2cid_tid[vring_index][0];
1353
1354         ether_addr_copy(da, wil->sta[cid].addr);
1355 }
1356
1357 static struct wil_ring *wil_find_tx_bcast_2(struct wil6210_priv *wil,
1358                                             struct wil6210_vif *vif,
1359                                             struct sk_buff *skb)
1360 {
1361         struct wil_ring *v, *v2;
1362         struct sk_buff *skb2;
1363         int i;
1364         u8 cid;
1365         const u8 *src = wil_skb_get_sa(skb);
1366         struct wil_ring_tx_data *txdata, *txdata2;
1367         int min_ring_id = wil_get_min_tx_ring_id(wil);
1368
1369         /* find 1-st vring eligible for data */
1370         for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
1371                 v = &wil->ring_tx[i];
1372                 txdata = &wil->ring_tx_data[i];
1373                 if (!v->va || !txdata->enabled || txdata->mid != vif->mid)
1374                         continue;
1375
1376                 cid = wil->ring2cid_tid[i][0];
1377                 if (cid >= max_assoc_sta) /* skip BCAST */
1378                         continue;
1379                 if (!wil->ring_tx_data[i].dot1x_open &&
1380                     skb->protocol != cpu_to_be16(ETH_P_PAE))
1381                         continue;
1382
1383                 /* don't Tx back to source when re-routing Rx->Tx at the AP */
1384                 if (0 == memcmp(wil->sta[cid].addr, src, ETH_ALEN))
1385                         continue;
1386
1387                 goto found;
1388         }
1389
1390         wil_dbg_txrx(wil, "Tx while no vrings active?\n");
1391
1392         return NULL;
1393
1394 found:
1395         wil_dbg_txrx(wil, "BCAST -> ring %d\n", i);
1396         wil_set_da_for_vring(wil, skb, i);
1397
1398         /* find other active vrings and duplicate skb for each */
1399         for (i++; i < WIL6210_MAX_TX_RINGS; i++) {
1400                 v2 = &wil->ring_tx[i];
1401                 txdata2 = &wil->ring_tx_data[i];
1402                 if (!v2->va || txdata2->mid != vif->mid)
1403                         continue;
1404                 cid = wil->ring2cid_tid[i][0];
1405                 if (cid >= max_assoc_sta) /* skip BCAST */
1406                         continue;
1407                 if (!wil->ring_tx_data[i].dot1x_open &&
1408                     skb->protocol != cpu_to_be16(ETH_P_PAE))
1409                         continue;
1410
1411                 if (0 == memcmp(wil->sta[cid].addr, src, ETH_ALEN))
1412                         continue;
1413
1414                 skb2 = skb_copy(skb, GFP_ATOMIC);
1415                 if (skb2) {
1416                         wil_dbg_txrx(wil, "BCAST DUP -> ring %d\n", i);
1417                         wil_set_da_for_vring(wil, skb2, i);
1418                         wil_tx_ring(wil, vif, v2, skb2);
1419                         /* successful call to wil_tx_ring takes skb2 ref */
1420                         dev_kfree_skb_any(skb2);
1421                 } else {
1422                         wil_err(wil, "skb_copy failed\n");
1423                 }
1424         }
1425
1426         return v;
1427 }
1428
1429 static inline
1430 void wil_tx_desc_set_nr_frags(struct vring_tx_desc *d, int nr_frags)
1431 {
1432         d->mac.d[2] |= (nr_frags << MAC_CFG_DESC_TX_2_NUM_OF_DESCRIPTORS_POS);
1433 }
1434
1435 /**
1436  * Sets the descriptor @d up for csum and/or TSO offloading. The corresponding
1437  * @skb is used to obtain the protocol and headers length.
1438  * @tso_desc_type is a descriptor type for TSO: 0 - a header, 1 - first data,
1439  * 2 - middle, 3 - last descriptor.
1440  */
1441
1442 static void wil_tx_desc_offload_setup_tso(struct vring_tx_desc *d,
1443                                           struct sk_buff *skb,
1444                                           int tso_desc_type, bool is_ipv4,
1445                                           int tcp_hdr_len, int skb_net_hdr_len)
1446 {
1447         d->dma.b11 = ETH_HLEN; /* MAC header length */
1448         d->dma.b11 |= is_ipv4 << DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS;
1449
1450         d->dma.d0 |= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS);
1451         /* L4 header len: TCP header length */
1452         d->dma.d0 |= (tcp_hdr_len & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK);
1453
1454         /* Setup TSO: bit and desc type */
1455         d->dma.d0 |= (BIT(DMA_CFG_DESC_TX_0_TCP_SEG_EN_POS)) |
1456                 (tso_desc_type << DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS);
1457         d->dma.d0 |= (is_ipv4 << DMA_CFG_DESC_TX_0_IPV4_CHECKSUM_EN_POS);
1458
1459         d->dma.ip_length = skb_net_hdr_len;
1460         /* Enable TCP/UDP checksum */
1461         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS);
1462         /* Calculate pseudo-header */
1463         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS);
1464 }
1465
1466 /**
1467  * Sets the descriptor @d up for csum. The corresponding
1468  * @skb is used to obtain the protocol and headers length.
1469  * Returns the protocol: 0 - not TCP, 1 - TCPv4, 2 - TCPv6.
1470  * Note, if d==NULL, the function only returns the protocol result.
1471  *
1472  * It is very similar to previous wil_tx_desc_offload_setup_tso. This
1473  * is "if unrolling" to optimize the critical path.
1474  */
1475
1476 static int wil_tx_desc_offload_setup(struct vring_tx_desc *d,
1477                                      struct sk_buff *skb){
1478         int protocol;
1479
1480         if (skb->ip_summed != CHECKSUM_PARTIAL)
1481                 return 0;
1482
1483         d->dma.b11 = ETH_HLEN; /* MAC header length */
1484
1485         switch (skb->protocol) {
1486         case cpu_to_be16(ETH_P_IP):
1487                 protocol = ip_hdr(skb)->protocol;
1488                 d->dma.b11 |= BIT(DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS);
1489                 break;
1490         case cpu_to_be16(ETH_P_IPV6):
1491                 protocol = ipv6_hdr(skb)->nexthdr;
1492                 break;
1493         default:
1494                 return -EINVAL;
1495         }
1496
1497         switch (protocol) {
1498         case IPPROTO_TCP:
1499                 d->dma.d0 |= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS);
1500                 /* L4 header len: TCP header length */
1501                 d->dma.d0 |=
1502                 (tcp_hdrlen(skb) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK);
1503                 break;
1504         case IPPROTO_UDP:
1505                 /* L4 header len: UDP header length */
1506                 d->dma.d0 |=
1507                 (sizeof(struct udphdr) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK);
1508                 break;
1509         default:
1510                 return -EINVAL;
1511         }
1512
1513         d->dma.ip_length = skb_network_header_len(skb);
1514         /* Enable TCP/UDP checksum */
1515         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS);
1516         /* Calculate pseudo-header */
1517         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS);
1518
1519         return 0;
1520 }
1521
1522 static inline void wil_tx_last_desc(struct vring_tx_desc *d)
1523 {
1524         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS) |
1525               BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS) |
1526               BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS);
1527 }
1528
1529 static inline void wil_set_tx_desc_last_tso(volatile struct vring_tx_desc *d)
1530 {
1531         d->dma.d0 |= wil_tso_type_lst <<
1532                   DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS;
1533 }
1534
1535 static int __wil_tx_vring_tso(struct wil6210_priv *wil, struct wil6210_vif *vif,
1536                               struct wil_ring *vring, struct sk_buff *skb)
1537 {
1538         struct device *dev = wil_to_dev(wil);
1539
1540         /* point to descriptors in shared memory */
1541         volatile struct vring_tx_desc *_desc = NULL, *_hdr_desc,
1542                                       *_first_desc = NULL;
1543
1544         /* pointers to shadow descriptors */
1545         struct vring_tx_desc desc_mem, hdr_desc_mem, first_desc_mem,
1546                              *d = &hdr_desc_mem, *hdr_desc = &hdr_desc_mem,
1547                              *first_desc = &first_desc_mem;
1548
1549         /* pointer to shadow descriptors' context */
1550         struct wil_ctx *hdr_ctx, *first_ctx = NULL;
1551
1552         int descs_used = 0; /* total number of used descriptors */
1553         int sg_desc_cnt = 0; /* number of descriptors for current mss*/
1554
1555         u32 swhead = vring->swhead;
1556         int used, avail = wil_ring_avail_tx(vring);
1557         int nr_frags = skb_shinfo(skb)->nr_frags;
1558         int min_desc_required = nr_frags + 1;
1559         int mss = skb_shinfo(skb)->gso_size;    /* payload size w/o headers */
1560         int f, len, hdrlen, headlen;
1561         int vring_index = vring - wil->ring_tx;
1562         struct wil_ring_tx_data *txdata = &wil->ring_tx_data[vring_index];
1563         uint i = swhead;
1564         dma_addr_t pa;
1565         const skb_frag_t *frag = NULL;
1566         int rem_data = mss;
1567         int lenmss;
1568         int hdr_compensation_need = true;
1569         int desc_tso_type = wil_tso_type_first;
1570         bool is_ipv4;
1571         int tcp_hdr_len;
1572         int skb_net_hdr_len;
1573         int gso_type;
1574         int rc = -EINVAL;
1575
1576         wil_dbg_txrx(wil, "tx_vring_tso: %d bytes to vring %d\n", skb->len,
1577                      vring_index);
1578
1579         if (unlikely(!txdata->enabled))
1580                 return -EINVAL;
1581
1582         /* A typical page 4K is 3-4 payloads, we assume each fragment
1583          * is a full payload, that's how min_desc_required has been
1584          * calculated. In real we might need more or less descriptors,
1585          * this is the initial check only.
1586          */
1587         if (unlikely(avail < min_desc_required)) {
1588                 wil_err_ratelimited(wil,
1589                                     "TSO: Tx ring[%2d] full. No space for %d fragments\n",
1590                                     vring_index, min_desc_required);
1591                 return -ENOMEM;
1592         }
1593
1594         /* Header Length = MAC header len + IP header len + TCP header len*/
1595         hdrlen = ETH_HLEN +
1596                 (int)skb_network_header_len(skb) +
1597                 tcp_hdrlen(skb);
1598
1599         gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV6 | SKB_GSO_TCPV4);
1600         switch (gso_type) {
1601         case SKB_GSO_TCPV4:
1602                 /* TCP v4, zero out the IP length and IPv4 checksum fields
1603                  * as required by the offloading doc
1604                  */
1605                 ip_hdr(skb)->tot_len = 0;
1606                 ip_hdr(skb)->check = 0;
1607                 is_ipv4 = true;
1608                 break;
1609         case SKB_GSO_TCPV6:
1610                 /* TCP v6, zero out the payload length */
1611                 ipv6_hdr(skb)->payload_len = 0;
1612                 is_ipv4 = false;
1613                 break;
1614         default:
1615                 /* other than TCPv4 or TCPv6 types are not supported for TSO.
1616                  * It is also illegal for both to be set simultaneously
1617                  */
1618                 return -EINVAL;
1619         }
1620
1621         if (skb->ip_summed != CHECKSUM_PARTIAL)
1622                 return -EINVAL;
1623
1624         /* tcp header length and skb network header length are fixed for all
1625          * packet's descriptors - read then once here
1626          */
1627         tcp_hdr_len = tcp_hdrlen(skb);
1628         skb_net_hdr_len = skb_network_header_len(skb);
1629
1630         _hdr_desc = &vring->va[i].tx.legacy;
1631
1632         pa = dma_map_single(dev, skb->data, hdrlen, DMA_TO_DEVICE);
1633         if (unlikely(dma_mapping_error(dev, pa))) {
1634                 wil_err(wil, "TSO: Skb head DMA map error\n");
1635                 goto err_exit;
1636         }
1637
1638         wil->txrx_ops.tx_desc_map((union wil_tx_desc *)hdr_desc, pa,
1639                                   hdrlen, vring_index);
1640         wil_tx_desc_offload_setup_tso(hdr_desc, skb, wil_tso_type_hdr, is_ipv4,
1641                                       tcp_hdr_len, skb_net_hdr_len);
1642         wil_tx_last_desc(hdr_desc);
1643
1644         vring->ctx[i].mapped_as = wil_mapped_as_single;
1645         hdr_ctx = &vring->ctx[i];
1646
1647         descs_used++;
1648         headlen = skb_headlen(skb) - hdrlen;
1649
1650         for (f = headlen ? -1 : 0; f < nr_frags; f++)  {
1651                 if (headlen) {
1652                         len = headlen;
1653                         wil_dbg_txrx(wil, "TSO: process skb head, len %u\n",
1654                                      len);
1655                 } else {
1656                         frag = &skb_shinfo(skb)->frags[f];
1657                         len = frag->size;
1658                         wil_dbg_txrx(wil, "TSO: frag[%d]: len %u\n", f, len);
1659                 }
1660
1661                 while (len) {
1662                         wil_dbg_txrx(wil,
1663                                      "TSO: len %d, rem_data %d, descs_used %d\n",
1664                                      len, rem_data, descs_used);
1665
1666                         if (descs_used == avail)  {
1667                                 wil_err_ratelimited(wil, "TSO: ring overflow\n");
1668                                 rc = -ENOMEM;
1669                                 goto mem_error;
1670                         }
1671
1672                         lenmss = min_t(int, rem_data, len);
1673                         i = (swhead + descs_used) % vring->size;
1674                         wil_dbg_txrx(wil, "TSO: lenmss %d, i %d\n", lenmss, i);
1675
1676                         if (!headlen) {
1677                                 pa = skb_frag_dma_map(dev, frag,
1678                                                       frag->size - len, lenmss,
1679                                                       DMA_TO_DEVICE);
1680                                 vring->ctx[i].mapped_as = wil_mapped_as_page;
1681                         } else {
1682                                 pa = dma_map_single(dev,
1683                                                     skb->data +
1684                                                     skb_headlen(skb) - headlen,
1685                                                     lenmss,
1686                                                     DMA_TO_DEVICE);
1687                                 vring->ctx[i].mapped_as = wil_mapped_as_single;
1688                                 headlen -= lenmss;
1689                         }
1690
1691                         if (unlikely(dma_mapping_error(dev, pa))) {
1692                                 wil_err(wil, "TSO: DMA map page error\n");
1693                                 goto mem_error;
1694                         }
1695
1696                         _desc = &vring->va[i].tx.legacy;
1697
1698                         if (!_first_desc) {
1699                                 _first_desc = _desc;
1700                                 first_ctx = &vring->ctx[i];
1701                                 d = first_desc;
1702                         } else {
1703                                 d = &desc_mem;
1704                         }
1705
1706                         wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d,
1707                                                   pa, lenmss, vring_index);
1708                         wil_tx_desc_offload_setup_tso(d, skb, desc_tso_type,
1709                                                       is_ipv4, tcp_hdr_len,
1710                                                       skb_net_hdr_len);
1711
1712                         /* use tso_type_first only once */
1713                         desc_tso_type = wil_tso_type_mid;
1714
1715                         descs_used++;  /* desc used so far */
1716                         sg_desc_cnt++; /* desc used for this segment */
1717                         len -= lenmss;
1718                         rem_data -= lenmss;
1719
1720                         wil_dbg_txrx(wil,
1721                                      "TSO: len %d, rem_data %d, descs_used %d, sg_desc_cnt %d,\n",
1722                                      len, rem_data, descs_used, sg_desc_cnt);
1723
1724                         /* Close the segment if reached mss size or last frag*/
1725                         if (rem_data == 0 || (f == nr_frags - 1 && len == 0)) {
1726                                 if (hdr_compensation_need) {
1727                                         /* first segment include hdr desc for
1728                                          * release
1729                                          */
1730                                         hdr_ctx->nr_frags = sg_desc_cnt;
1731                                         wil_tx_desc_set_nr_frags(first_desc,
1732                                                                  sg_desc_cnt +
1733                                                                  1);
1734                                         hdr_compensation_need = false;
1735                                 } else {
1736                                         wil_tx_desc_set_nr_frags(first_desc,
1737                                                                  sg_desc_cnt);
1738                                 }
1739                                 first_ctx->nr_frags = sg_desc_cnt - 1;
1740
1741                                 wil_tx_last_desc(d);
1742
1743                                 /* first descriptor may also be the last
1744                                  * for this mss - make sure not to copy
1745                                  * it twice
1746                                  */
1747                                 if (first_desc != d)
1748                                         *_first_desc = *first_desc;
1749
1750                                 /*last descriptor will be copied at the end
1751                                  * of this TS processing
1752                                  */
1753                                 if (f < nr_frags - 1 || len > 0)
1754                                         *_desc = *d;
1755
1756                                 rem_data = mss;
1757                                 _first_desc = NULL;
1758                                 sg_desc_cnt = 0;
1759                         } else if (first_desc != d) /* update mid descriptor */
1760                                         *_desc = *d;
1761                 }
1762         }
1763
1764         /* first descriptor may also be the last.
1765          * in this case d pointer is invalid
1766          */
1767         if (_first_desc == _desc)
1768                 d = first_desc;
1769
1770         /* Last data descriptor */
1771         wil_set_tx_desc_last_tso(d);
1772         *_desc = *d;
1773
1774         /* Fill the total number of descriptors in first desc (hdr)*/
1775         wil_tx_desc_set_nr_frags(hdr_desc, descs_used);
1776         *_hdr_desc = *hdr_desc;
1777
1778         /* hold reference to skb
1779          * to prevent skb release before accounting
1780          * in case of immediate "tx done"
1781          */
1782         vring->ctx[i].skb = skb_get(skb);
1783
1784         /* performance monitoring */
1785         used = wil_ring_used_tx(vring);
1786         if (wil_val_in_range(wil->ring_idle_trsh,
1787                              used, used + descs_used)) {
1788                 txdata->idle += get_cycles() - txdata->last_idle;
1789                 wil_dbg_txrx(wil,  "Ring[%2d] not idle %d -> %d\n",
1790                              vring_index, used, used + descs_used);
1791         }
1792
1793         /* Make sure to advance the head only after descriptor update is done.
1794          * This will prevent a race condition where the completion thread
1795          * will see the DU bit set from previous run and will handle the
1796          * skb before it was completed.
1797          */
1798         wmb();
1799
1800         /* advance swhead */
1801         wil_ring_advance_head(vring, descs_used);
1802         wil_dbg_txrx(wil, "TSO: Tx swhead %d -> %d\n", swhead, vring->swhead);
1803
1804         /* make sure all writes to descriptors (shared memory) are done before
1805          * committing them to HW
1806          */
1807         wmb();
1808
1809         if (wil->tx_latency)
1810                 *(ktime_t *)&skb->cb = ktime_get();
1811         else
1812                 memset(skb->cb, 0, sizeof(ktime_t));
1813
1814         wil_w(wil, vring->hwtail, vring->swhead);
1815         return 0;
1816
1817 mem_error:
1818         while (descs_used > 0) {
1819                 struct wil_ctx *ctx;
1820
1821                 i = (swhead + descs_used - 1) % vring->size;
1822                 d = (struct vring_tx_desc *)&vring->va[i].tx.legacy;
1823                 _desc = &vring->va[i].tx.legacy;
1824                 *d = *_desc;
1825                 _desc->dma.status = TX_DMA_STATUS_DU;
1826                 ctx = &vring->ctx[i];
1827                 wil_txdesc_unmap(dev, (union wil_tx_desc *)d, ctx);
1828                 memset(ctx, 0, sizeof(*ctx));
1829                 descs_used--;
1830         }
1831 err_exit:
1832         return rc;
1833 }
1834
1835 static int __wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,
1836                          struct wil_ring *ring, struct sk_buff *skb)
1837 {
1838         struct device *dev = wil_to_dev(wil);
1839         struct vring_tx_desc dd, *d = &dd;
1840         volatile struct vring_tx_desc *_d;
1841         u32 swhead = ring->swhead;
1842         int avail = wil_ring_avail_tx(ring);
1843         int nr_frags = skb_shinfo(skb)->nr_frags;
1844         uint f = 0;
1845         int ring_index = ring - wil->ring_tx;
1846         struct wil_ring_tx_data  *txdata = &wil->ring_tx_data[ring_index];
1847         uint i = swhead;
1848         dma_addr_t pa;
1849         int used;
1850         bool mcast = (ring_index == vif->bcast_ring);
1851         uint len = skb_headlen(skb);
1852
1853         wil_dbg_txrx(wil, "tx_ring: %d bytes to ring %d, nr_frags %d\n",
1854                      skb->len, ring_index, nr_frags);
1855
1856         if (unlikely(!txdata->enabled))
1857                 return -EINVAL;
1858
1859         if (unlikely(avail < 1 + nr_frags)) {
1860                 wil_err_ratelimited(wil,
1861                                     "Tx ring[%2d] full. No space for %d fragments\n",
1862                                     ring_index, 1 + nr_frags);
1863                 return -ENOMEM;
1864         }
1865         _d = &ring->va[i].tx.legacy;
1866
1867         pa = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE);
1868
1869         wil_dbg_txrx(wil, "Tx[%2d] skb %d bytes 0x%p -> %pad\n", ring_index,
1870                      skb_headlen(skb), skb->data, &pa);
1871         wil_hex_dump_txrx("Tx ", DUMP_PREFIX_OFFSET, 16, 1,
1872                           skb->data, skb_headlen(skb), false);
1873
1874         if (unlikely(dma_mapping_error(dev, pa)))
1875                 return -EINVAL;
1876         ring->ctx[i].mapped_as = wil_mapped_as_single;
1877         /* 1-st segment */
1878         wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d, pa, len,
1879                                    ring_index);
1880         if (unlikely(mcast)) {
1881                 d->mac.d[0] |= BIT(MAC_CFG_DESC_TX_0_MCS_EN_POS); /* MCS 0 */
1882                 if (unlikely(len > WIL_BCAST_MCS0_LIMIT)) /* set MCS 1 */
1883                         d->mac.d[0] |= (1 << MAC_CFG_DESC_TX_0_MCS_INDEX_POS);
1884         }
1885         /* Process TCP/UDP checksum offloading */
1886         if (unlikely(wil_tx_desc_offload_setup(d, skb))) {
1887                 wil_err(wil, "Tx[%2d] Failed to set cksum, drop packet\n",
1888                         ring_index);
1889                 goto dma_error;
1890         }
1891
1892         ring->ctx[i].nr_frags = nr_frags;
1893         wil_tx_desc_set_nr_frags(d, nr_frags + 1);
1894
1895         /* middle segments */
1896         for (; f < nr_frags; f++) {
1897                 const struct skb_frag_struct *frag =
1898                                 &skb_shinfo(skb)->frags[f];
1899                 int len = skb_frag_size(frag);
1900
1901                 *_d = *d;
1902                 wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", ring_index, i);
1903                 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4,
1904                                   (const void *)d, sizeof(*d), false);
1905                 i = (swhead + f + 1) % ring->size;
1906                 _d = &ring->va[i].tx.legacy;
1907                 pa = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag),
1908                                       DMA_TO_DEVICE);
1909                 if (unlikely(dma_mapping_error(dev, pa))) {
1910                         wil_err(wil, "Tx[%2d] failed to map fragment\n",
1911                                 ring_index);
1912                         goto dma_error;
1913                 }
1914                 ring->ctx[i].mapped_as = wil_mapped_as_page;
1915                 wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d,
1916                                            pa, len, ring_index);
1917                 /* no need to check return code -
1918                  * if it succeeded for 1-st descriptor,
1919                  * it will succeed here too
1920                  */
1921                 wil_tx_desc_offload_setup(d, skb);
1922         }
1923         /* for the last seg only */
1924         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS);
1925         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS);
1926         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS);
1927         *_d = *d;
1928         wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", ring_index, i);
1929         wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4,
1930                           (const void *)d, sizeof(*d), false);
1931
1932         /* hold reference to skb
1933          * to prevent skb release before accounting
1934          * in case of immediate "tx done"
1935          */
1936         ring->ctx[i].skb = skb_get(skb);
1937
1938         /* performance monitoring */
1939         used = wil_ring_used_tx(ring);
1940         if (wil_val_in_range(wil->ring_idle_trsh,
1941                              used, used + nr_frags + 1)) {
1942                 txdata->idle += get_cycles() - txdata->last_idle;
1943                 wil_dbg_txrx(wil,  "Ring[%2d] not idle %d -> %d\n",
1944                              ring_index, used, used + nr_frags + 1);
1945         }
1946
1947         /* Make sure to advance the head only after descriptor update is done.
1948          * This will prevent a race condition where the completion thread
1949          * will see the DU bit set from previous run and will handle the
1950          * skb before it was completed.
1951          */
1952         wmb();
1953
1954         /* advance swhead */
1955         wil_ring_advance_head(ring, nr_frags + 1);
1956         wil_dbg_txrx(wil, "Tx[%2d] swhead %d -> %d\n", ring_index, swhead,
1957                      ring->swhead);
1958         trace_wil6210_tx(ring_index, swhead, skb->len, nr_frags);
1959
1960         /* make sure all writes to descriptors (shared memory) are done before
1961          * committing them to HW
1962          */
1963         wmb();
1964
1965         if (wil->tx_latency)
1966                 *(ktime_t *)&skb->cb = ktime_get();
1967         else
1968                 memset(skb->cb, 0, sizeof(ktime_t));
1969
1970         wil_w(wil, ring->hwtail, ring->swhead);
1971
1972         return 0;
1973  dma_error:
1974         /* unmap what we have mapped */
1975         nr_frags = f + 1; /* frags mapped + one for skb head */
1976         for (f = 0; f < nr_frags; f++) {
1977                 struct wil_ctx *ctx;
1978
1979                 i = (swhead + f) % ring->size;
1980                 ctx = &ring->ctx[i];
1981                 _d = &ring->va[i].tx.legacy;
1982                 *d = *_d;
1983                 _d->dma.status = TX_DMA_STATUS_DU;
1984                 wil->txrx_ops.tx_desc_unmap(dev,
1985                                             (union wil_tx_desc *)d,
1986                                             ctx);
1987
1988                 memset(ctx, 0, sizeof(*ctx));
1989         }
1990
1991         return -EINVAL;
1992 }
1993
1994 static int wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,
1995                        struct wil_ring *ring, struct sk_buff *skb)
1996 {
1997         int ring_index = ring - wil->ring_tx;
1998         struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_index];
1999         int rc;
2000
2001         spin_lock(&txdata->lock);
2002
2003         if (test_bit(wil_status_suspending, wil->status) ||
2004             test_bit(wil_status_suspended, wil->status) ||
2005             test_bit(wil_status_resuming, wil->status)) {
2006                 wil_dbg_txrx(wil,
2007                              "suspend/resume in progress. drop packet\n");
2008                 spin_unlock(&txdata->lock);
2009                 return -EINVAL;
2010         }
2011
2012         rc = (skb_is_gso(skb) ? wil->txrx_ops.tx_ring_tso : __wil_tx_ring)
2013              (wil, vif, ring, skb);
2014
2015         spin_unlock(&txdata->lock);
2016
2017         return rc;
2018 }
2019
2020 /**
2021  * Check status of tx vrings and stop/wake net queues if needed
2022  * It will start/stop net queues of a specific VIF net_device.
2023  *
2024  * This function does one of two checks:
2025  * In case check_stop is true, will check if net queues need to be stopped. If
2026  * the conditions for stopping are met, netif_tx_stop_all_queues() is called.
2027  * In case check_stop is false, will check if net queues need to be waked. If
2028  * the conditions for waking are met, netif_tx_wake_all_queues() is called.
2029  * vring is the vring which is currently being modified by either adding
2030  * descriptors (tx) into it or removing descriptors (tx complete) from it. Can
2031  * be null when irrelevant (e.g. connect/disconnect events).
2032  *
2033  * The implementation is to stop net queues if modified vring has low
2034  * descriptor availability. Wake if all vrings are not in low descriptor
2035  * availability and modified vring has high descriptor availability.
2036  */
2037 static inline void __wil_update_net_queues(struct wil6210_priv *wil,
2038                                            struct wil6210_vif *vif,
2039                                            struct wil_ring *ring,
2040                                            bool check_stop)
2041 {
2042         int i;
2043         int min_ring_id = wil_get_min_tx_ring_id(wil);
2044
2045         if (unlikely(!vif))
2046                 return;
2047
2048         if (ring)
2049                 wil_dbg_txrx(wil, "vring %d, mid %d, check_stop=%d, stopped=%d",
2050                              (int)(ring - wil->ring_tx), vif->mid, check_stop,
2051                              vif->net_queue_stopped);
2052         else
2053                 wil_dbg_txrx(wil, "check_stop=%d, mid=%d, stopped=%d",
2054                              check_stop, vif->mid, vif->net_queue_stopped);
2055
2056         if (ring && drop_if_ring_full)
2057                 /* no need to stop/wake net queues */
2058                 return;
2059
2060         if (check_stop == vif->net_queue_stopped)
2061                 /* net queues already in desired state */
2062                 return;
2063
2064         if (check_stop) {
2065                 if (!ring || unlikely(wil_ring_avail_low(ring))) {
2066                         /* not enough room in the vring */
2067                         netif_tx_stop_all_queues(vif_to_ndev(vif));
2068                         vif->net_queue_stopped = true;
2069                         wil_dbg_txrx(wil, "netif_tx_stop called\n");
2070                 }
2071                 return;
2072         }
2073
2074         /* Do not wake the queues in suspend flow */
2075         if (test_bit(wil_status_suspending, wil->status) ||
2076             test_bit(wil_status_suspended, wil->status))
2077                 return;
2078
2079         /* check wake */
2080         for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
2081                 struct wil_ring *cur_ring = &wil->ring_tx[i];
2082                 struct wil_ring_tx_data  *txdata = &wil->ring_tx_data[i];
2083
2084                 if (txdata->mid != vif->mid || !cur_ring->va ||
2085                     !txdata->enabled || cur_ring == ring)
2086                         continue;
2087
2088                 if (wil_ring_avail_low(cur_ring)) {
2089                         wil_dbg_txrx(wil, "ring %d full, can't wake\n",
2090                                      (int)(cur_ring - wil->ring_tx));
2091                         return;
2092                 }
2093         }
2094
2095         if (!ring || wil_ring_avail_high(ring)) {
2096                 /* enough room in the ring */
2097                 wil_dbg_txrx(wil, "calling netif_tx_wake\n");
2098                 netif_tx_wake_all_queues(vif_to_ndev(vif));
2099                 vif->net_queue_stopped = false;
2100         }
2101 }
2102
2103 void wil_update_net_queues(struct wil6210_priv *wil, struct wil6210_vif *vif,
2104                            struct wil_ring *ring, bool check_stop)
2105 {
2106         spin_lock(&wil->net_queue_lock);
2107         __wil_update_net_queues(wil, vif, ring, check_stop);
2108         spin_unlock(&wil->net_queue_lock);
2109 }
2110
2111 void wil_update_net_queues_bh(struct wil6210_priv *wil, struct wil6210_vif *vif,
2112                               struct wil_ring *ring, bool check_stop)
2113 {
2114         spin_lock_bh(&wil->net_queue_lock);
2115         __wil_update_net_queues(wil, vif, ring, check_stop);
2116         spin_unlock_bh(&wil->net_queue_lock);
2117 }
2118
2119 netdev_tx_t wil_start_xmit(struct sk_buff *skb, struct net_device *ndev)
2120 {
2121         struct wil6210_vif *vif = ndev_to_vif(ndev);
2122         struct wil6210_priv *wil = vif_to_wil(vif);
2123         const u8 *da = wil_skb_get_da(skb);
2124         bool bcast = is_multicast_ether_addr(da);
2125         struct wil_ring *ring;
2126         static bool pr_once_fw;
2127         int rc;
2128
2129         wil_dbg_txrx(wil, "start_xmit\n");
2130         if (unlikely(!test_bit(wil_status_fwready, wil->status))) {
2131                 if (!pr_once_fw) {
2132                         wil_err(wil, "FW not ready\n");
2133                         pr_once_fw = true;
2134                 }
2135                 goto drop;
2136         }
2137         if (unlikely(!test_bit(wil_vif_fwconnected, vif->status))) {
2138                 wil_dbg_ratelimited(wil,
2139                                     "VIF not connected, packet dropped\n");
2140                 goto drop;
2141         }
2142         if (unlikely(vif->wdev.iftype == NL80211_IFTYPE_MONITOR)) {
2143                 wil_err(wil, "Xmit in monitor mode not supported\n");
2144                 goto drop;
2145         }
2146         pr_once_fw = false;
2147
2148         /* find vring */
2149         if (vif->wdev.iftype == NL80211_IFTYPE_STATION && !vif->pbss) {
2150                 /* in STA mode (ESS), all to same VRING (to AP) */
2151                 ring = wil_find_tx_ring_sta(wil, vif, skb);
2152         } else if (bcast) {
2153                 if (vif->pbss)
2154                         /* in pbss, no bcast VRING - duplicate skb in
2155                          * all stations VRINGs
2156                          */
2157                         ring = wil_find_tx_bcast_2(wil, vif, skb);
2158                 else if (vif->wdev.iftype == NL80211_IFTYPE_AP)
2159                         /* AP has a dedicated bcast VRING */
2160                         ring = wil_find_tx_bcast_1(wil, vif, skb);
2161                 else
2162                         /* unexpected combination, fallback to duplicating
2163                          * the skb in all stations VRINGs
2164                          */
2165                         ring = wil_find_tx_bcast_2(wil, vif, skb);
2166         } else {
2167                 /* unicast, find specific VRING by dest. address */
2168                 ring = wil_find_tx_ucast(wil, vif, skb);
2169         }
2170         if (unlikely(!ring)) {
2171                 wil_dbg_txrx(wil, "No Tx RING found for %pM\n", da);
2172                 goto drop;
2173         }
2174         /* set up vring entry */
2175         rc = wil_tx_ring(wil, vif, ring, skb);
2176
2177         switch (rc) {
2178         case 0:
2179                 /* shall we stop net queues? */
2180                 wil_update_net_queues_bh(wil, vif, ring, true);
2181                 /* statistics will be updated on the tx_complete */
2182                 dev_kfree_skb_any(skb);
2183                 return NETDEV_TX_OK;
2184         case -ENOMEM:
2185                 if (drop_if_ring_full)
2186                         goto drop;
2187                 return NETDEV_TX_BUSY;
2188         default:
2189                 break; /* goto drop; */
2190         }
2191  drop:
2192         ndev->stats.tx_dropped++;
2193         dev_kfree_skb_any(skb);
2194
2195         return NET_XMIT_DROP;
2196 }
2197
2198 void wil_tx_latency_calc(struct wil6210_priv *wil, struct sk_buff *skb,
2199                          struct wil_sta_info *sta)
2200 {
2201         int skb_time_us;
2202         int bin;
2203
2204         if (!wil->tx_latency)
2205                 return;
2206
2207         if (ktime_to_ms(*(ktime_t *)&skb->cb) == 0)
2208                 return;
2209
2210         skb_time_us = ktime_us_delta(ktime_get(), *(ktime_t *)&skb->cb);
2211         bin = skb_time_us / wil->tx_latency_res;
2212         bin = min_t(int, bin, WIL_NUM_LATENCY_BINS - 1);
2213
2214         wil_dbg_txrx(wil, "skb time %dus => bin %d\n", skb_time_us, bin);
2215         sta->tx_latency_bins[bin]++;
2216         sta->stats.tx_latency_total_us += skb_time_us;
2217         if (skb_time_us < sta->stats.tx_latency_min_us)
2218                 sta->stats.tx_latency_min_us = skb_time_us;
2219         if (skb_time_us > sta->stats.tx_latency_max_us)
2220                 sta->stats.tx_latency_max_us = skb_time_us;
2221 }
2222
2223 /**
2224  * Clean up transmitted skb's from the Tx VRING
2225  *
2226  * Return number of descriptors cleared
2227  *
2228  * Safe to call from IRQ
2229  */
2230 int wil_tx_complete(struct wil6210_vif *vif, int ringid)
2231 {
2232         struct wil6210_priv *wil = vif_to_wil(vif);
2233         struct net_device *ndev = vif_to_ndev(vif);
2234         struct device *dev = wil_to_dev(wil);
2235         struct wil_ring *vring = &wil->ring_tx[ringid];
2236         struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ringid];
2237         int done = 0;
2238         int cid = wil->ring2cid_tid[ringid][0];
2239         struct wil_net_stats *stats = NULL;
2240         volatile struct vring_tx_desc *_d;
2241         int used_before_complete;
2242         int used_new;
2243
2244         if (unlikely(!vring->va)) {
2245                 wil_err(wil, "Tx irq[%d]: vring not initialized\n", ringid);
2246                 return 0;
2247         }
2248
2249         if (unlikely(!txdata->enabled)) {
2250                 wil_info(wil, "Tx irq[%d]: vring disabled\n", ringid);
2251                 return 0;
2252         }
2253
2254         wil_dbg_txrx(wil, "tx_complete: (%d)\n", ringid);
2255
2256         used_before_complete = wil_ring_used_tx(vring);
2257
2258         if (cid < max_assoc_sta)
2259                 stats = &wil->sta[cid].stats;
2260
2261         while (!wil_ring_is_empty(vring)) {
2262                 int new_swtail;
2263                 struct wil_ctx *ctx = &vring->ctx[vring->swtail];
2264                 /**
2265                  * For the fragmented skb, HW will set DU bit only for the
2266                  * last fragment. look for it.
2267                  * In TSO the first DU will include hdr desc
2268                  */
2269                 int lf = (vring->swtail + ctx->nr_frags) % vring->size;
2270                 /* TODO: check we are not past head */
2271
2272                 _d = &vring->va[lf].tx.legacy;
2273                 if (unlikely(!(_d->dma.status & TX_DMA_STATUS_DU)))
2274                         break;
2275
2276                 new_swtail = (lf + 1) % vring->size;
2277                 while (vring->swtail != new_swtail) {
2278                         struct vring_tx_desc dd, *d = &dd;
2279                         u16 dmalen;
2280                         struct sk_buff *skb;
2281
2282                         ctx = &vring->ctx[vring->swtail];
2283                         skb = ctx->skb;
2284                         _d = &vring->va[vring->swtail].tx.legacy;
2285
2286                         *d = *_d;
2287
2288                         dmalen = le16_to_cpu(d->dma.length);
2289                         trace_wil6210_tx_done(ringid, vring->swtail, dmalen,
2290                                               d->dma.error);
2291                         wil_dbg_txrx(wil,
2292                                      "TxC[%2d][%3d] : %d bytes, status 0x%02x err 0x%02x\n",
2293                                      ringid, vring->swtail, dmalen,
2294                                      d->dma.status, d->dma.error);
2295                         wil_hex_dump_txrx("TxCD ", DUMP_PREFIX_NONE, 32, 4,
2296                                           (const void *)d, sizeof(*d), false);
2297
2298                         wil->txrx_ops.tx_desc_unmap(dev,
2299                                                     (union wil_tx_desc *)d,
2300                                                     ctx);
2301
2302                         if (skb) {
2303                                 if (likely(d->dma.error == 0)) {
2304                                         ndev->stats.tx_packets++;
2305                                         ndev->stats.tx_bytes += skb->len;
2306                                         if (stats) {
2307                                                 stats->tx_packets++;
2308                                                 stats->tx_bytes += skb->len;
2309
2310                                                 wil_tx_latency_calc(wil, skb,
2311                                                         &wil->sta[cid]);
2312                                         }
2313                                 } else {
2314                                         ndev->stats.tx_errors++;
2315                                         if (stats)
2316                                                 stats->tx_errors++;
2317                                 }
2318                                 wil_consume_skb(skb, d->dma.error == 0);
2319                         }
2320                         memset(ctx, 0, sizeof(*ctx));
2321                         /* Make sure the ctx is zeroed before updating the tail
2322                          * to prevent a case where wil_tx_ring will see
2323                          * this descriptor as used and handle it before ctx zero
2324                          * is completed.
2325                          */
2326                         wmb();
2327                         /* There is no need to touch HW descriptor:
2328                          * - ststus bit TX_DMA_STATUS_DU is set by design,
2329                          *   so hardware will not try to process this desc.,
2330                          * - rest of descriptor will be initialized on Tx.
2331                          */
2332                         vring->swtail = wil_ring_next_tail(vring);
2333                         done++;
2334                 }
2335         }
2336
2337         /* performance monitoring */
2338         used_new = wil_ring_used_tx(vring);
2339         if (wil_val_in_range(wil->ring_idle_trsh,
2340                              used_new, used_before_complete)) {
2341                 wil_dbg_txrx(wil, "Ring[%2d] idle %d -> %d\n",
2342                              ringid, used_before_complete, used_new);
2343                 txdata->last_idle = get_cycles();
2344         }
2345
2346         /* shall we wake net queues? */
2347         if (done)
2348                 wil_update_net_queues(wil, vif, vring, false);
2349
2350         return done;
2351 }
2352
2353 static inline int wil_tx_init(struct wil6210_priv *wil)
2354 {
2355         return 0;
2356 }
2357
2358 static inline void wil_tx_fini(struct wil6210_priv *wil) {}
2359
2360 static void wil_get_reorder_params(struct wil6210_priv *wil,
2361                                    struct sk_buff *skb, int *tid, int *cid,
2362                                    int *mid, u16 *seq, int *mcast, int *retry)
2363 {
2364         struct vring_rx_desc *d = wil_skb_rxdesc(skb);
2365
2366         *tid = wil_rxdesc_tid(d);
2367         *cid = wil_skb_get_cid(skb);
2368         *mid = wil_rxdesc_mid(d);
2369         *seq = wil_rxdesc_seq(d);
2370         *mcast = wil_rxdesc_mcast(d);
2371         *retry = wil_rxdesc_retry(d);
2372 }
2373
2374 void wil_init_txrx_ops_legacy_dma(struct wil6210_priv *wil)
2375 {
2376         wil->txrx_ops.configure_interrupt_moderation =
2377                 wil_configure_interrupt_moderation;
2378         /* TX ops */
2379         wil->txrx_ops.tx_desc_map = wil_tx_desc_map;
2380         wil->txrx_ops.tx_desc_unmap = wil_txdesc_unmap;
2381         wil->txrx_ops.tx_ring_tso =  __wil_tx_vring_tso;
2382         wil->txrx_ops.ring_init_tx = wil_vring_init_tx;
2383         wil->txrx_ops.ring_fini_tx = wil_vring_free;
2384         wil->txrx_ops.ring_init_bcast = wil_vring_init_bcast;
2385         wil->txrx_ops.tx_init = wil_tx_init;
2386         wil->txrx_ops.tx_fini = wil_tx_fini;
2387         wil->txrx_ops.tx_ring_modify = wil_tx_vring_modify;
2388         /* RX ops */
2389         wil->txrx_ops.rx_init = wil_rx_init;
2390         wil->txrx_ops.wmi_addba_rx_resp = wmi_addba_rx_resp;
2391         wil->txrx_ops.get_reorder_params = wil_get_reorder_params;
2392         wil->txrx_ops.get_netif_rx_params =
2393                 wil_get_netif_rx_params;
2394         wil->txrx_ops.rx_crypto_check = wil_rx_crypto_check;
2395         wil->txrx_ops.rx_error_check = wil_rx_error_check;
2396         wil->txrx_ops.is_rx_idle = wil_is_rx_idle;
2397         wil->txrx_ops.rx_fini = wil_rx_fini;
2398 }