drivers/infiniband/hw/hfi1/ipoib_tx.c

   1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
   2 /*
   3  * Copyright(c) 2020 Intel Corporation.
   4  *
   5  */
   6
   7 /*
   8  * This file contains HFI1 support for IPOIB SDMA functionality
   9  */
  10
  11 #include <linux/log2.h>
  12 #include <linux/circ_buf.h>
  13
  14 #include "sdma.h"
  15 #include "verbs.h"
  16 #include "trace_ibhdrs.h"
  17 #include "ipoib.h"
  18 #include "trace_tx.h"
  19
  20 /* Add a convenience helper */
  21 #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1))
  22 #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size)
  23 #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size)
  24
  25 /**
  26  * struct ipoib_txreq - IPOIB transmit descriptor
  27  * @txreq: sdma transmit request
  28  * @sdma_hdr: 9b ib headers
  29  * @sdma_status: status returned by sdma engine
  30  * @priv: ipoib netdev private data
  31  * @txq: txq on which skb was output
  32  * @skb: skb to send
  33  */
  34 struct ipoib_txreq {
  35         struct sdma_txreq           txreq;
  36         struct hfi1_sdma_header     sdma_hdr;
  37         int                         sdma_status;
  38         struct hfi1_ipoib_dev_priv *priv;
  39         struct hfi1_ipoib_txq      *txq;
  40         struct sk_buff             *skb;
  41 };
  42
  43 struct ipoib_txparms {
  44         struct hfi1_devdata        *dd;
  45         struct rdma_ah_attr        *ah_attr;
  46         struct hfi1_ibport         *ibp;
  47         struct hfi1_ipoib_txq      *txq;
  48         union hfi1_ipoib_flow       flow;
  49         u32                         dqpn;
  50         u8                          hdr_dwords;
  51         u8                          entropy;
  52 };
  53
  54 static u64 hfi1_ipoib_txreqs(const u64 sent, const u64 completed)
  55 {
  56         return sent - completed;
  57 }
  58
  59 static u64 hfi1_ipoib_used(struct hfi1_ipoib_txq *txq)
  60 {
  61         return hfi1_ipoib_txreqs(txq->sent_txreqs,
  62                                  atomic64_read(&txq->complete_txreqs));
  63 }
  64
  65 static void hfi1_ipoib_stop_txq(struct hfi1_ipoib_txq *txq)
  66 {
  67         trace_hfi1_txq_stop(txq);
  68         if (atomic_inc_return(&txq->stops) == 1)
  69                 netif_stop_subqueue(txq->priv->netdev, txq->q_idx);
  70 }
  71
  72 static void hfi1_ipoib_wake_txq(struct hfi1_ipoib_txq *txq)
  73 {
  74         trace_hfi1_txq_wake(txq);
  75         if (atomic_dec_and_test(&txq->stops))
  76                 netif_wake_subqueue(txq->priv->netdev, txq->q_idx);
  77 }
  78
  79 static uint hfi1_ipoib_ring_hwat(struct hfi1_ipoib_txq *txq)
  80 {
  81         return min_t(uint, txq->priv->netdev->tx_queue_len,
  82                      txq->tx_ring.max_items - 1);
  83 }
  84
  85 static uint hfi1_ipoib_ring_lwat(struct hfi1_ipoib_txq *txq)
  86 {
  87         return min_t(uint, txq->priv->netdev->tx_queue_len,
  88                      txq->tx_ring.max_items) >> 1;
  89 }
  90
  91 static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq)
  92 {
  93         ++txq->sent_txreqs;
  94         if (hfi1_ipoib_used(txq) >= hfi1_ipoib_ring_hwat(txq) &&
  95             !atomic_xchg(&txq->ring_full, 1)) {
  96                 trace_hfi1_txq_full(txq);
  97                 hfi1_ipoib_stop_txq(txq);
  98         }
  99 }
 100
 101 static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq)
 102 {
 103         struct net_device *dev = txq->priv->netdev;
 104
 105         /* If shutting down just return as queue state is irrelevant */
 106         if (unlikely(dev->reg_state != NETREG_REGISTERED))
 107                 return;
 108
 109         /*
 110          * When the queue has been drained to less than half full it will be
 111          * restarted.
 112          * The size of the txreq ring is fixed at initialization.
 113          * The tx queue len can be adjusted upward while the interface is
 114          * running.
 115          * The tx queue len can be large enough to overflow the txreq_ring.
 116          * Use the minimum of the current tx_queue_len or the rings max txreqs
 117          * to protect against ring overflow.
 118          */
 119         if (hfi1_ipoib_used(txq) < hfi1_ipoib_ring_lwat(txq) &&
 120             atomic_xchg(&txq->ring_full, 0)) {
 121                 trace_hfi1_txq_xmit_unstopped(txq);
 122                 hfi1_ipoib_wake_txq(txq);
 123         }
 124 }
 125
 126 static void hfi1_ipoib_free_tx(struct ipoib_txreq *tx, int budget)
 127 {
 128         struct hfi1_ipoib_dev_priv *priv = tx->priv;
 129
 130         if (likely(!tx->sdma_status)) {
 131                 dev_sw_netstats_tx_add(priv->netdev, 1, tx->skb->len);
 132         } else {
 133                 ++priv->netdev->stats.tx_errors;
 134                 dd_dev_warn(priv->dd,
 135                             "%s: Status = 0x%x pbc 0x%llx txq = %d sde = %d\n",
 136                             __func__, tx->sdma_status,
 137                             le64_to_cpu(tx->sdma_hdr.pbc), tx->txq->q_idx,
 138                             tx->txq->sde->this_idx);
 139         }
 140
 141         napi_consume_skb(tx->skb, budget);
 142         sdma_txclean(priv->dd, &tx->txreq);
 143         kmem_cache_free(priv->txreq_cache, tx);
 144 }
 145
 146 static int hfi1_ipoib_drain_tx_ring(struct hfi1_ipoib_txq *txq, int budget)
 147 {
 148         struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;
 149         unsigned long head;
 150         unsigned long tail;
 151         unsigned int max_tx;
 152         int work_done;
 153         int tx_count;
 154
 155         spin_lock_bh(&tx_ring->consumer_lock);
 156
 157         /* Read index before reading contents at that index. */
 158         head = smp_load_acquire(&tx_ring->head);
 159         tail = tx_ring->tail;
 160         max_tx = tx_ring->max_items;
 161
 162         work_done = min_t(int, CIRC_CNT(head, tail, max_tx), budget);
 163
 164         for (tx_count = work_done; tx_count; tx_count--) {
 165                 hfi1_ipoib_free_tx(tx_ring->items[tail], budget);
 166                 tail = CIRC_NEXT(tail, max_tx);
 167         }
 168
 169         atomic64_add(work_done, &txq->complete_txreqs);
 170
 171         /* Finished freeing tx items so store the tail value. */
 172         smp_store_release(&tx_ring->tail, tail);
 173
 174         spin_unlock_bh(&tx_ring->consumer_lock);
 175
 176         hfi1_ipoib_check_queue_stopped(txq);
 177
 178         return work_done;
 179 }
 180
 181 static int hfi1_ipoib_process_tx_ring(struct napi_struct *napi, int budget)
 182 {
 183         struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(napi->dev);
 184         struct hfi1_ipoib_txq *txq = &priv->txqs[napi - priv->tx_napis];
 185
 186         int work_done = hfi1_ipoib_drain_tx_ring(txq, budget);
 187
 188         if (work_done < budget)
 189                 napi_complete_done(napi, work_done);
 190
 191         return work_done;
 192 }
 193
 194 static void hfi1_ipoib_add_tx(struct ipoib_txreq *tx)
 195 {
 196         struct hfi1_ipoib_circ_buf *tx_ring = &tx->txq->tx_ring;
 197         unsigned long head;
 198         unsigned long tail;
 199         size_t max_tx;
 200
 201         spin_lock(&tx_ring->producer_lock);
 202
 203         head = tx_ring->head;
 204         tail = READ_ONCE(tx_ring->tail);
 205         max_tx = tx_ring->max_items;
 206
 207         if (likely(CIRC_SPACE(head, tail, max_tx))) {
 208                 tx_ring->items[head] = tx;
 209
 210                 /* Finish storing txreq before incrementing head. */
 211                 smp_store_release(&tx_ring->head, CIRC_ADD(head, 1, max_tx));
 212                 napi_schedule_irqoff(tx->txq->napi);
 213         } else {
 214                 struct hfi1_ipoib_txq *txq = tx->txq;
 215                 struct hfi1_ipoib_dev_priv *priv = tx->priv;
 216
 217                 /* Ring was full */
 218                 hfi1_ipoib_free_tx(tx, 0);
 219                 atomic64_inc(&txq->complete_txreqs);
 220                 dd_dev_dbg(priv->dd, "txq %d full.\n", txq->q_idx);
 221         }
 222
 223         spin_unlock(&tx_ring->producer_lock);
 224 }
 225
 226 static void hfi1_ipoib_sdma_complete(struct sdma_txreq *txreq, int status)
 227 {
 228         struct ipoib_txreq *tx = container_of(txreq, struct ipoib_txreq, txreq);
 229
 230         tx->sdma_status = status;
 231
 232         hfi1_ipoib_add_tx(tx);
 233 }
 234
 235 static int hfi1_ipoib_build_ulp_payload(struct ipoib_txreq *tx,
 236                                         struct ipoib_txparms *txp)
 237 {
 238         struct hfi1_devdata *dd = txp->dd;
 239         struct sdma_txreq *txreq = &tx->txreq;
 240         struct sk_buff *skb = tx->skb;
 241         int ret = 0;
 242         int i;
 243
 244         if (skb_headlen(skb)) {
 245                 ret = sdma_txadd_kvaddr(dd, txreq, skb->data, skb_headlen(skb));
 246                 if (unlikely(ret))
 247                         return ret;
 248         }
 249
 250         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 251                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 252
 253                 ret = sdma_txadd_page(dd,
 254                                       txreq,
 255                                       skb_frag_page(frag),
 256                                       frag->bv_offset,
 257                                       skb_frag_size(frag));
 258                 if (unlikely(ret))
 259                         break;
 260         }
 261
 262         return ret;
 263 }
 264
 265 static int hfi1_ipoib_build_tx_desc(struct ipoib_txreq *tx,
 266                                     struct ipoib_txparms *txp)
 267 {
 268         struct hfi1_devdata *dd = txp->dd;
 269         struct sdma_txreq *txreq = &tx->txreq;
 270         struct hfi1_sdma_header *sdma_hdr = &tx->sdma_hdr;
 271         u16 pkt_bytes =
 272                 sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2) + tx->skb->len;
 273         int ret;
 274
 275         ret = sdma_txinit(txreq, 0, pkt_bytes, hfi1_ipoib_sdma_complete);
 276         if (unlikely(ret))
 277                 return ret;
 278
 279         /* add pbc + headers */
 280         ret = sdma_txadd_kvaddr(dd,
 281                                 txreq,
 282                                 sdma_hdr,
 283                                 sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2));
 284         if (unlikely(ret))
 285                 return ret;
 286
 287         /* add the ulp payload */
 288         return hfi1_ipoib_build_ulp_payload(tx, txp);
 289 }
 290
 291 static void hfi1_ipoib_build_ib_tx_headers(struct ipoib_txreq *tx,
 292                                            struct ipoib_txparms *txp)
 293 {
 294         struct hfi1_ipoib_dev_priv *priv = tx->priv;
 295         struct hfi1_sdma_header *sdma_hdr = &tx->sdma_hdr;
 296         struct sk_buff *skb = tx->skb;
 297         struct hfi1_pportdata *ppd = ppd_from_ibp(txp->ibp);
 298         struct rdma_ah_attr *ah_attr = txp->ah_attr;
 299         struct ib_other_headers *ohdr;
 300         struct ib_grh *grh;
 301         u16 dwords;
 302         u16 slid;
 303         u16 dlid;
 304         u16 lrh0;
 305         u32 bth0;
 306         u32 sqpn = (u32)(priv->netdev->dev_addr[1] << 16 |
 307                          priv->netdev->dev_addr[2] << 8 |
 308                          priv->netdev->dev_addr[3]);
 309         u16 payload_dwords;
 310         u8 pad_cnt;
 311
 312         pad_cnt = -skb->len & 3;
 313
 314         /* Includes ICRC */
 315         payload_dwords = ((skb->len + pad_cnt) >> 2) + SIZE_OF_CRC;
 316
 317         /* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */
 318         txp->hdr_dwords = 7;
 319
 320         if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
 321                 grh = &sdma_hdr->hdr.ibh.u.l.grh;
 322                 txp->hdr_dwords +=
 323                         hfi1_make_grh(txp->ibp,
 324                                       grh,
 325                                       rdma_ah_read_grh(ah_attr),
 326                                       txp->hdr_dwords - LRH_9B_DWORDS,
 327                                       payload_dwords);
 328                 lrh0 = HFI1_LRH_GRH;
 329                 ohdr = &sdma_hdr->hdr.ibh.u.l.oth;
 330         } else {
 331                 lrh0 = HFI1_LRH_BTH;
 332                 ohdr = &sdma_hdr->hdr.ibh.u.oth;
 333         }
 334
 335         lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4;
 336         lrh0 |= (txp->flow.sc5 & 0xf) << 12;
 337
 338         dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 9B);
 339         if (dlid == be16_to_cpu(IB_LID_PERMISSIVE)) {
 340                 slid = be16_to_cpu(IB_LID_PERMISSIVE);
 341         } else {
 342                 u16 lid = (u16)ppd->lid;
 343
 344                 if (lid) {
 345                         lid |= rdma_ah_get_path_bits(ah_attr) &
 346                                 ((1 << ppd->lmc) - 1);
 347                         slid = lid;
 348                 } else {
 349                         slid = be16_to_cpu(IB_LID_PERMISSIVE);
 350                 }
 351         }
 352
 353         /* Includes ICRC */
 354         dwords = txp->hdr_dwords + payload_dwords;
 355
 356         /* Build the lrh */
 357         sdma_hdr->hdr.hdr_type = HFI1_PKT_TYPE_9B;
 358         hfi1_make_ib_hdr(&sdma_hdr->hdr.ibh, lrh0, dwords, dlid, slid);
 359
 360         /* Build the bth */
 361         bth0 = (IB_OPCODE_UD_SEND_ONLY << 24) | (pad_cnt << 20) | priv->pkey;
 362
 363         ohdr->bth[0] = cpu_to_be32(bth0);
 364         ohdr->bth[1] = cpu_to_be32(txp->dqpn);
 365         ohdr->bth[2] = cpu_to_be32(mask_psn((u32)txp->txq->sent_txreqs));
 366
 367         /* Build the deth */
 368         ohdr->u.ud.deth[0] = cpu_to_be32(priv->qkey);
 369         ohdr->u.ud.deth[1] = cpu_to_be32((txp->entropy <<
 370                                           HFI1_IPOIB_ENTROPY_SHIFT) | sqpn);
 371
 372         /* Construct the pbc. */
 373         sdma_hdr->pbc =
 374                 cpu_to_le64(create_pbc(ppd,
 375                                        ib_is_sc5(txp->flow.sc5) <<
 376                                                               PBC_DC_INFO_SHIFT,
 377                                        0,
 378                                        sc_to_vlt(priv->dd, txp->flow.sc5),
 379                                        dwords - SIZE_OF_CRC +
 380                                                 (sizeof(sdma_hdr->pbc) >> 2)));
 381 }
 382
 383 static struct ipoib_txreq *hfi1_ipoib_send_dma_common(struct net_device *dev,
 384                                                       struct sk_buff *skb,
 385                                                       struct ipoib_txparms *txp)
 386 {
 387         struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
 388         struct ipoib_txreq *tx;
 389         int ret;
 390
 391         tx = kmem_cache_alloc_node(priv->txreq_cache,
 392                                    GFP_ATOMIC,
 393                                    priv->dd->node);
 394         if (unlikely(!tx))
 395                 return ERR_PTR(-ENOMEM);
 396
 397         /* so that we can test if the sdma descriptors are there */
 398         tx->txreq.num_desc = 0;
 399         tx->priv = priv;
 400         tx->txq = txp->txq;
 401         tx->skb = skb;
 402         INIT_LIST_HEAD(&tx->txreq.list);
 403
 404         hfi1_ipoib_build_ib_tx_headers(tx, txp);
 405
 406         ret = hfi1_ipoib_build_tx_desc(tx, txp);
 407         if (likely(!ret)) {
 408                 if (txp->txq->flow.as_int != txp->flow.as_int) {
 409                         txp->txq->flow.tx_queue = txp->flow.tx_queue;
 410                         txp->txq->flow.sc5 = txp->flow.sc5;
 411                         txp->txq->sde =
 412                                 sdma_select_engine_sc(priv->dd,
 413                                                       txp->flow.tx_queue,
 414                                                       txp->flow.sc5);
 415                         trace_hfi1_flow_switch(txp->txq);
 416                 }
 417
 418                 return tx;
 419         }
 420
 421         sdma_txclean(priv->dd, &tx->txreq);
 422         kmem_cache_free(priv->txreq_cache, tx);
 423
 424         return ERR_PTR(ret);
 425 }
 426
 427 static int hfi1_ipoib_submit_tx_list(struct net_device *dev,
 428                                      struct hfi1_ipoib_txq *txq)
 429 {
 430         int ret;
 431         u16 count_out;
 432
 433         ret = sdma_send_txlist(txq->sde,
 434                                iowait_get_ib_work(&txq->wait),
 435                                &txq->tx_list,
 436                                &count_out);
 437         if (likely(!ret) || ret == -EBUSY || ret == -ECOMM)
 438                 return ret;
 439
 440         dd_dev_warn(txq->priv->dd, "cannot send skb tx list, err %d.\n", ret);
 441
 442         return ret;
 443 }
 444
 445 static int hfi1_ipoib_flush_tx_list(struct net_device *dev,
 446                                     struct hfi1_ipoib_txq *txq)
 447 {
 448         int ret = 0;
 449
 450         if (!list_empty(&txq->tx_list)) {
 451                 /* Flush the current list */
 452                 ret = hfi1_ipoib_submit_tx_list(dev, txq);
 453
 454                 if (unlikely(ret))
 455                         if (ret != -EBUSY)
 456                                 ++dev->stats.tx_carrier_errors;
 457         }
 458
 459         return ret;
 460 }
 461
 462 static int hfi1_ipoib_submit_tx(struct hfi1_ipoib_txq *txq,
 463                                 struct ipoib_txreq *tx)
 464 {
 465         int ret;
 466
 467         ret = sdma_send_txreq(txq->sde,
 468                               iowait_get_ib_work(&txq->wait),
 469                               &tx->txreq,
 470                               txq->pkts_sent);
 471         if (likely(!ret)) {
 472                 txq->pkts_sent = true;
 473                 iowait_starve_clear(txq->pkts_sent, &txq->wait);
 474         }
 475
 476         return ret;
 477 }
 478
 479 static int hfi1_ipoib_send_dma_single(struct net_device *dev,
 480                                       struct sk_buff *skb,
 481                                       struct ipoib_txparms *txp)
 482 {
 483         struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
 484         struct hfi1_ipoib_txq *txq = txp->txq;
 485         struct ipoib_txreq *tx;
 486         int ret;
 487
 488         tx = hfi1_ipoib_send_dma_common(dev, skb, txp);
 489         if (IS_ERR(tx)) {
 490                 int ret = PTR_ERR(tx);
 491
 492                 dev_kfree_skb_any(skb);
 493
 494                 if (ret == -ENOMEM)
 495                         ++dev->stats.tx_errors;
 496                 else
 497                         ++dev->stats.tx_carrier_errors;
 498
 499                 return NETDEV_TX_OK;
 500         }
 501
 502         ret = hfi1_ipoib_submit_tx(txq, tx);
 503         if (likely(!ret)) {
 504 tx_ok:
 505                 trace_sdma_output_ibhdr(tx->priv->dd,
 506                                         &tx->sdma_hdr.hdr,
 507                                         ib_is_sc5(txp->flow.sc5));
 508                 hfi1_ipoib_check_queue_depth(txq);
 509                 return NETDEV_TX_OK;
 510         }
 511
 512         txq->pkts_sent = false;
 513
 514         if (ret == -EBUSY || ret == -ECOMM)
 515                 goto tx_ok;
 516
 517         sdma_txclean(priv->dd, &tx->txreq);
 518         dev_kfree_skb_any(skb);
 519         kmem_cache_free(priv->txreq_cache, tx);
 520         ++dev->stats.tx_carrier_errors;
 521
 522         return NETDEV_TX_OK;
 523 }
 524
 525 static int hfi1_ipoib_send_dma_list(struct net_device *dev,
 526                                     struct sk_buff *skb,
 527                                     struct ipoib_txparms *txp)
 528 {
 529         struct hfi1_ipoib_txq *txq = txp->txq;
 530         struct ipoib_txreq *tx;
 531
 532         /* Has the flow change ? */
 533         if (txq->flow.as_int != txp->flow.as_int) {
 534                 int ret;
 535
 536                 trace_hfi1_flow_flush(txq);
 537                 ret = hfi1_ipoib_flush_tx_list(dev, txq);
 538                 if (unlikely(ret)) {
 539                         if (ret == -EBUSY)
 540                                 ++dev->stats.tx_dropped;
 541                         dev_kfree_skb_any(skb);
 542                         return NETDEV_TX_OK;
 543                 }
 544         }
 545         tx = hfi1_ipoib_send_dma_common(dev, skb, txp);
 546         if (IS_ERR(tx)) {
 547                 int ret = PTR_ERR(tx);
 548
 549                 dev_kfree_skb_any(skb);
 550
 551                 if (ret == -ENOMEM)
 552                         ++dev->stats.tx_errors;
 553                 else
 554                         ++dev->stats.tx_carrier_errors;
 555
 556                 return NETDEV_TX_OK;
 557         }
 558
 559         list_add_tail(&tx->txreq.list, &txq->tx_list);
 560
 561         hfi1_ipoib_check_queue_depth(txq);
 562
 563         trace_sdma_output_ibhdr(tx->priv->dd,
 564                                 &tx->sdma_hdr.hdr,
 565                                 ib_is_sc5(txp->flow.sc5));
 566
 567         if (!netdev_xmit_more())
 568                 (void)hfi1_ipoib_flush_tx_list(dev, txq);
 569
 570         return NETDEV_TX_OK;
 571 }
 572
 573 static u8 hfi1_ipoib_calc_entropy(struct sk_buff *skb)
 574 {
 575         if (skb_transport_header_was_set(skb)) {
 576                 u8 *hdr = (u8 *)skb_transport_header(skb);
 577
 578                 return (hdr[0] ^ hdr[1] ^ hdr[2] ^ hdr[3]);
 579         }
 580
 581         return (u8)skb_get_queue_mapping(skb);
 582 }
 583
 584 int hfi1_ipoib_send(struct net_device *dev,
 585                     struct sk_buff *skb,
 586                     struct ib_ah *address,
 587                     u32 dqpn)
 588 {
 589         struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
 590         struct ipoib_txparms txp;
 591         struct rdma_netdev *rn = netdev_priv(dev);
 592
 593         if (unlikely(skb->len > rn->mtu + HFI1_IPOIB_ENCAP_LEN)) {
 594                 dd_dev_warn(priv->dd, "packet len %d (> %d) too long to send, dropping\n",
 595                             skb->len,
 596                             rn->mtu + HFI1_IPOIB_ENCAP_LEN);
 597                 ++dev->stats.tx_dropped;
 598                 ++dev->stats.tx_errors;
 599                 dev_kfree_skb_any(skb);
 600                 return NETDEV_TX_OK;
 601         }
 602
 603         txp.dd = priv->dd;
 604         txp.ah_attr = &ibah_to_rvtah(address)->attr;
 605         txp.ibp = to_iport(priv->device, priv->port_num);
 606         txp.txq = &priv->txqs[skb_get_queue_mapping(skb)];
 607         txp.dqpn = dqpn;
 608         txp.flow.sc5 = txp.ibp->sl_to_sc[rdma_ah_get_sl(txp.ah_attr)];
 609         txp.flow.tx_queue = (u8)skb_get_queue_mapping(skb);
 610         txp.entropy = hfi1_ipoib_calc_entropy(skb);
 611
 612         if (netdev_xmit_more() || !list_empty(&txp.txq->tx_list))
 613                 return hfi1_ipoib_send_dma_list(dev, skb, &txp);
 614
 615         return hfi1_ipoib_send_dma_single(dev, skb,  &txp);
 616 }
 617
 618 /*
 619  * hfi1_ipoib_sdma_sleep - ipoib sdma sleep function
 620  *
 621  * This function gets called from sdma_send_txreq() when there are not enough
 622  * sdma descriptors available to send the packet. It adds Tx queue's wait
 623  * structure to sdma engine's dmawait list to be woken up when descriptors
 624  * become available.
 625  */
 626 static int hfi1_ipoib_sdma_sleep(struct sdma_engine *sde,
 627                                  struct iowait_work *wait,
 628                                  struct sdma_txreq *txreq,
 629                                  uint seq,
 630                                  bool pkts_sent)
 631 {
 632         struct hfi1_ipoib_txq *txq =
 633                 container_of(wait->iow, struct hfi1_ipoib_txq, wait);
 634
 635         write_seqlock(&sde->waitlock);
 636
 637         if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED)) {
 638                 if (sdma_progress(sde, seq, txreq)) {
 639                         write_sequnlock(&sde->waitlock);
 640                         return -EAGAIN;
 641                 }
 642
 643                 if (list_empty(&txreq->list))
 644                         /* came from non-list submit */
 645                         list_add_tail(&txreq->list, &txq->tx_list);
 646                 if (list_empty(&txq->wait.list)) {
 647                         if (!atomic_xchg(&txq->no_desc, 1)) {
 648                                 trace_hfi1_txq_queued(txq);
 649                                 hfi1_ipoib_stop_txq(txq);
 650                         }
 651                         iowait_queue(pkts_sent, wait->iow, &sde->dmawait);
 652                 }
 653
 654                 write_sequnlock(&sde->waitlock);
 655                 return -EBUSY;
 656         }
 657
 658         write_sequnlock(&sde->waitlock);
 659         return -EINVAL;
 660 }
 661
 662 /*
 663  * hfi1_ipoib_sdma_wakeup - ipoib sdma wakeup function
 664  *
 665  * This function gets called when SDMA descriptors becomes available and Tx
 666  * queue's wait structure was previously added to sdma engine's dmawait list.
 667  */
 668 static void hfi1_ipoib_sdma_wakeup(struct iowait *wait, int reason)
 669 {
 670         struct hfi1_ipoib_txq *txq =
 671                 container_of(wait, struct hfi1_ipoib_txq, wait);
 672
 673         trace_hfi1_txq_wakeup(txq);
 674         if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED))
 675                 iowait_schedule(wait, system_highpri_wq, WORK_CPU_UNBOUND);
 676 }
 677
 678 static void hfi1_ipoib_flush_txq(struct work_struct *work)
 679 {
 680         struct iowait_work *ioww =
 681                 container_of(work, struct iowait_work, iowork);
 682         struct iowait *wait = iowait_ioww_to_iow(ioww);
 683         struct hfi1_ipoib_txq *txq =
 684                 container_of(wait, struct hfi1_ipoib_txq, wait);
 685         struct net_device *dev = txq->priv->netdev;
 686
 687         if (likely(dev->reg_state == NETREG_REGISTERED) &&
 688             likely(!hfi1_ipoib_flush_tx_list(dev, txq)))
 689                 if (atomic_xchg(&txq->no_desc, 0))
 690                         hfi1_ipoib_wake_txq(txq);
 691 }
 692
 693 int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv)
 694 {
 695         struct net_device *dev = priv->netdev;
 696         char buf[HFI1_IPOIB_TXREQ_NAME_LEN];
 697         unsigned long tx_ring_size;
 698         int i;
 699
 700         /*
 701          * Ring holds 1 less than tx_ring_size
 702          * Round up to next power of 2 in order to hold at least tx_queue_len
 703          */
 704         tx_ring_size = roundup_pow_of_two((unsigned long)dev->tx_queue_len + 1);
 705
 706         snprintf(buf, sizeof(buf), "hfi1_%u_ipoib_txreq_cache", priv->dd->unit);
 707         priv->txreq_cache = kmem_cache_create(buf,
 708                                               sizeof(struct ipoib_txreq),
 709                                               0,
 710                                               0,
 711                                               NULL);
 712         if (!priv->txreq_cache)
 713                 return -ENOMEM;
 714
 715         priv->tx_napis = kcalloc_node(dev->num_tx_queues,
 716                                       sizeof(struct napi_struct),
 717                                       GFP_KERNEL,
 718                                       priv->dd->node);
 719         if (!priv->tx_napis)
 720                 goto free_txreq_cache;
 721
 722         priv->txqs = kcalloc_node(dev->num_tx_queues,
 723                                   sizeof(struct hfi1_ipoib_txq),
 724                                   GFP_KERNEL,
 725                                   priv->dd->node);
 726         if (!priv->txqs)
 727                 goto free_tx_napis;
 728
 729         for (i = 0; i < dev->num_tx_queues; i++) {
 730                 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
 731
 732                 iowait_init(&txq->wait,
 733                             0,
 734                             hfi1_ipoib_flush_txq,
 735                             NULL,
 736                             hfi1_ipoib_sdma_sleep,
 737                             hfi1_ipoib_sdma_wakeup,
 738                             NULL,
 739                             NULL);
 740                 txq->priv = priv;
 741                 txq->sde = NULL;
 742                 INIT_LIST_HEAD(&txq->tx_list);
 743                 atomic64_set(&txq->complete_txreqs, 0);
 744                 atomic_set(&txq->stops, 0);
 745                 atomic_set(&txq->ring_full, 0);
 746                 atomic_set(&txq->no_desc, 0);
 747                 txq->q_idx = i;
 748                 txq->flow.tx_queue = 0xff;
 749                 txq->flow.sc5 = 0xff;
 750                 txq->pkts_sent = false;
 751
 752                 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
 753                                              priv->dd->node);
 754
 755                 txq->tx_ring.items =
 756                         kcalloc_node(tx_ring_size,
 757                                      sizeof(struct ipoib_txreq *),
 758                                      GFP_KERNEL, priv->dd->node);
 759                 if (!txq->tx_ring.items)
 760                         goto free_txqs;
 761
 762                 spin_lock_init(&txq->tx_ring.producer_lock);
 763                 spin_lock_init(&txq->tx_ring.consumer_lock);
 764                 txq->tx_ring.max_items = tx_ring_size;
 765
 766                 txq->napi = &priv->tx_napis[i];
 767                 netif_tx_napi_add(dev, txq->napi,
 768                                   hfi1_ipoib_process_tx_ring,
 769                                   NAPI_POLL_WEIGHT);
 770         }
 771
 772         return 0;
 773
 774 free_txqs:
 775         for (i--; i >= 0; i--) {
 776                 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
 777
 778                 netif_napi_del(txq->napi);
 779                 kfree(txq->tx_ring.items);
 780         }
 781
 782         kfree(priv->txqs);
 783         priv->txqs = NULL;
 784
 785 free_tx_napis:
 786         kfree(priv->tx_napis);
 787         priv->tx_napis = NULL;
 788
 789 free_txreq_cache:
 790         kmem_cache_destroy(priv->txreq_cache);
 791         priv->txreq_cache = NULL;
 792         return -ENOMEM;
 793 }
 794
 795 static void hfi1_ipoib_drain_tx_list(struct hfi1_ipoib_txq *txq)
 796 {
 797         struct sdma_txreq *txreq;
 798         struct sdma_txreq *txreq_tmp;
 799         atomic64_t *complete_txreqs = &txq->complete_txreqs;
 800
 801         list_for_each_entry_safe(txreq, txreq_tmp, &txq->tx_list, list) {
 802                 struct ipoib_txreq *tx =
 803                         container_of(txreq, struct ipoib_txreq, txreq);
 804
 805                 list_del(&txreq->list);
 806                 sdma_txclean(txq->priv->dd, &tx->txreq);
 807                 dev_kfree_skb_any(tx->skb);
 808                 kmem_cache_free(txq->priv->txreq_cache, tx);
 809                 atomic64_inc(complete_txreqs);
 810         }
 811
 812         if (hfi1_ipoib_used(txq))
 813                 dd_dev_warn(txq->priv->dd,
 814                             "txq %d not empty found %llu requests\n",
 815                             txq->q_idx,
 816                             hfi1_ipoib_txreqs(txq->sent_txreqs,
 817                                               atomic64_read(complete_txreqs)));
 818 }
 819
 820 void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv)
 821 {
 822         int i;
 823
 824         for (i = 0; i < priv->netdev->num_tx_queues; i++) {
 825                 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
 826
 827                 iowait_cancel_work(&txq->wait);
 828                 iowait_sdma_drain(&txq->wait);
 829                 hfi1_ipoib_drain_tx_list(txq);
 830                 netif_napi_del(txq->napi);
 831                 (void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items);
 832                 kfree(txq->tx_ring.items);
 833         }
 834
 835         kfree(priv->txqs);
 836         priv->txqs = NULL;
 837
 838         kfree(priv->tx_napis);
 839         priv->tx_napis = NULL;
 840
 841         kmem_cache_destroy(priv->txreq_cache);
 842         priv->txreq_cache = NULL;
 843 }
 844
 845 void hfi1_ipoib_napi_tx_enable(struct net_device *dev)
 846 {
 847         struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
 848         int i;
 849
 850         for (i = 0; i < dev->num_tx_queues; i++) {
 851                 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
 852
 853                 napi_enable(txq->napi);
 854         }
 855 }
 856
 857 void hfi1_ipoib_napi_tx_disable(struct net_device *dev)
 858 {
 859         struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
 860         int i;
 861
 862         for (i = 0; i < dev->num_tx_queues; i++) {
 863                 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
 864
 865                 napi_disable(txq->napi);
 866                 (void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items);
 867         }
 868 }
 869
 870 void hfi1_ipoib_tx_timeout(struct net_device *dev, unsigned int q)
 871 {
 872         struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
 873         struct hfi1_ipoib_txq *txq = &priv->txqs[q];
 874         u64 completed = atomic64_read(&txq->complete_txreqs);
 875
 876         dd_dev_info(priv->dd, "timeout txq %llx q %u stopped %u stops %d no_desc %d ring_full %d\n",
 877                     (unsigned long long)txq, q,
 878                     __netif_subqueue_stopped(dev, txq->q_idx),
 879                     atomic_read(&txq->stops),
 880                     atomic_read(&txq->no_desc),
 881                     atomic_read(&txq->ring_full));
 882         dd_dev_info(priv->dd, "sde %llx engine %u\n",
 883                     (unsigned long long)txq->sde,
 884                     txq->sde ? txq->sde->this_idx : 0);
 885         dd_dev_info(priv->dd, "flow %x\n", txq->flow.as_int);
 886         dd_dev_info(priv->dd, "sent %llu completed %llu used %llu\n",
 887                     txq->sent_txreqs, completed, hfi1_ipoib_used(txq));
 888         dd_dev_info(priv->dd, "tx_queue_len %u max_items %lu\n",
 889                     dev->tx_queue_len, txq->tx_ring.max_items);
 890         dd_dev_info(priv->dd, "head %lu tail %lu\n",
 891                     txq->tx_ring.head, txq->tx_ring.tail);
 892         dd_dev_info(priv->dd, "wait queued %u\n",
 893                     !list_empty(&txq->wait.list));
 894         dd_dev_info(priv->dd, "tx_list empty %u\n",
 895                     list_empty(&txq->tx_list));
 896 }
 897