drivers/net/ethernet/qlogic/qed/qed_ll2.c

   1 /* QLogic qed NIC Driver
   2  * Copyright (c) 2015-2017  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/types.h>
  34 #include <asm/byteorder.h>
  35 #include <linux/dma-mapping.h>
  36 #include <linux/if_vlan.h>
  37 #include <linux/kernel.h>
  38 #include <linux/pci.h>
  39 #include <linux/slab.h>
  40 #include <linux/stddef.h>
  41 #include <linux/workqueue.h>
  42 #include <net/ipv6.h>
  43 #include <linux/bitops.h>
  44 #include <linux/delay.h>
  45 #include <linux/errno.h>
  46 #include <linux/etherdevice.h>
  47 #include <linux/io.h>
  48 #include <linux/list.h>
  49 #include <linux/mutex.h>
  50 #include <linux/spinlock.h>
  51 #include <linux/string.h>
  52 #include <linux/qed/qed_ll2_if.h>
  53 #include "qed.h"
  54 #include "qed_cxt.h"
  55 #include "qed_dev_api.h"
  56 #include "qed_hsi.h"
  57 #include "qed_hw.h"
  58 #include "qed_int.h"
  59 #include "qed_ll2.h"
  60 #include "qed_mcp.h"
  61 #include "qed_ooo.h"
  62 #include "qed_reg_addr.h"
  63 #include "qed_sp.h"
  64 #include "qed_roce.h"
  65
  66 #define QED_LL2_RX_REGISTERED(ll2)      ((ll2)->rx_queue.b_cb_registred)
  67 #define QED_LL2_TX_REGISTERED(ll2)      ((ll2)->tx_queue.b_cb_registred)
  68
  69 #define QED_LL2_TX_SIZE (256)
  70 #define QED_LL2_RX_SIZE (4096)
  71
  72 struct qed_cb_ll2_info {
  73         int rx_cnt;
  74         u32 rx_size;
  75         u8 handle;
  76         bool frags_mapped;
  77
  78         /* Lock protecting LL2 buffer lists in sleepless context */
  79         spinlock_t lock;
  80         struct list_head list;
  81
  82         const struct qed_ll2_cb_ops *cbs;
  83         void *cb_cookie;
  84 };
  85
  86 struct qed_ll2_buffer {
  87         struct list_head list;
  88         void *data;
  89         dma_addr_t phys_addr;
  90 };
  91
  92 static void qed_ll2b_complete_tx_packet(struct qed_hwfn *p_hwfn,
  93                                         u8 connection_handle,
  94                                         void *cookie,
  95                                         dma_addr_t first_frag_addr,
  96                                         bool b_last_fragment,
  97                                         bool b_last_packet)
  98 {
  99         struct qed_dev *cdev = p_hwfn->cdev;
 100         struct sk_buff *skb = cookie;
 101
 102         /* All we need to do is release the mapping */
 103         dma_unmap_single(&p_hwfn->cdev->pdev->dev, first_frag_addr,
 104                          skb_headlen(skb), DMA_TO_DEVICE);
 105
 106         if (cdev->ll2->cbs && cdev->ll2->cbs->tx_cb)
 107                 cdev->ll2->cbs->tx_cb(cdev->ll2->cb_cookie, skb,
 108                                       b_last_fragment);
 109
 110         if (cdev->ll2->frags_mapped)
 111                 /* Case where mapped frags were received, need to
 112                  * free skb with nr_frags marked as 0
 113                  */
 114                 skb_shinfo(skb)->nr_frags = 0;
 115
 116         dev_kfree_skb_any(skb);
 117 }
 118
 119 static int qed_ll2_alloc_buffer(struct qed_dev *cdev,
 120                                 u8 **data, dma_addr_t *phys_addr)
 121 {
 122         *data = kmalloc(cdev->ll2->rx_size, GFP_ATOMIC);
 123         if (!(*data)) {
 124                 DP_INFO(cdev, "Failed to allocate LL2 buffer data\n");
 125                 return -ENOMEM;
 126         }
 127
 128         *phys_addr = dma_map_single(&cdev->pdev->dev,
 129                                     ((*data) + NET_SKB_PAD),
 130                                     cdev->ll2->rx_size, DMA_FROM_DEVICE);
 131         if (dma_mapping_error(&cdev->pdev->dev, *phys_addr)) {
 132                 DP_INFO(cdev, "Failed to map LL2 buffer data\n");
 133                 kfree((*data));
 134                 return -ENOMEM;
 135         }
 136
 137         return 0;
 138 }
 139
 140 static int qed_ll2_dealloc_buffer(struct qed_dev *cdev,
 141                                  struct qed_ll2_buffer *buffer)
 142 {
 143         spin_lock_bh(&cdev->ll2->lock);
 144
 145         dma_unmap_single(&cdev->pdev->dev, buffer->phys_addr,
 146                          cdev->ll2->rx_size, DMA_FROM_DEVICE);
 147         kfree(buffer->data);
 148         list_del(&buffer->list);
 149
 150         cdev->ll2->rx_cnt--;
 151         if (!cdev->ll2->rx_cnt)
 152                 DP_INFO(cdev, "All LL2 entries were removed\n");
 153
 154         spin_unlock_bh(&cdev->ll2->lock);
 155
 156         return 0;
 157 }
 158
 159 static void qed_ll2_kill_buffers(struct qed_dev *cdev)
 160 {
 161         struct qed_ll2_buffer *buffer, *tmp_buffer;
 162
 163         list_for_each_entry_safe(buffer, tmp_buffer, &cdev->ll2->list, list)
 164                 qed_ll2_dealloc_buffer(cdev, buffer);
 165 }
 166
 167 static void qed_ll2b_complete_rx_packet(struct qed_hwfn *p_hwfn,
 168                                         u8 connection_handle,
 169                                         struct qed_ll2_rx_packet *p_pkt,
 170                                         struct core_rx_fast_path_cqe *p_cqe,
 171                                         bool b_last_packet)
 172 {
 173         u16 packet_length = le16_to_cpu(p_cqe->packet_length);
 174         struct qed_ll2_buffer *buffer = p_pkt->cookie;
 175         struct qed_dev *cdev = p_hwfn->cdev;
 176         u16 vlan = le16_to_cpu(p_cqe->vlan);
 177         u32 opaque_data_0, opaque_data_1;
 178         u8 pad = p_cqe->placement_offset;
 179         dma_addr_t new_phys_addr;
 180         struct sk_buff *skb;
 181         bool reuse = false;
 182         int rc = -EINVAL;
 183         u8 *new_data;
 184
 185         opaque_data_0 = le32_to_cpu(p_cqe->opaque_data.data[0]);
 186         opaque_data_1 = le32_to_cpu(p_cqe->opaque_data.data[1]);
 187
 188         DP_VERBOSE(p_hwfn,
 189                    (NETIF_MSG_RX_STATUS | QED_MSG_STORAGE | NETIF_MSG_PKTDATA),
 190                    "Got an LL2 Rx completion: [Buffer at phys 0x%llx, offset 0x%02x] Length 0x%04x Parse_flags 0x%04x vlan 0x%04x Opaque data [0x%08x:0x%08x]\n",
 191                    (u64)p_pkt->rx_buf_addr, pad, packet_length,
 192                    le16_to_cpu(p_cqe->parse_flags.flags), vlan,
 193                    opaque_data_0, opaque_data_1);
 194
 195         if ((cdev->dp_module & NETIF_MSG_PKTDATA) && buffer->data) {
 196                 print_hex_dump(KERN_INFO, "",
 197                                DUMP_PREFIX_OFFSET, 16, 1,
 198                                buffer->data, packet_length, false);
 199         }
 200
 201         /* Determine if data is valid */
 202         if (packet_length < ETH_HLEN)
 203                 reuse = true;
 204
 205         /* Allocate a replacement for buffer; Reuse upon failure */
 206         if (!reuse)
 207                 rc = qed_ll2_alloc_buffer(p_hwfn->cdev, &new_data,
 208                                           &new_phys_addr);
 209
 210         /* If need to reuse or there's no replacement buffer, repost this */
 211         if (rc)
 212                 goto out_post;
 213         dma_unmap_single(&cdev->pdev->dev, buffer->phys_addr,
 214                          cdev->ll2->rx_size, DMA_FROM_DEVICE);
 215
 216         skb = build_skb(buffer->data, 0);
 217         if (!skb) {
 218                 rc = -ENOMEM;
 219                 goto out_post;
 220         }
 221
 222         pad += NET_SKB_PAD;
 223         skb_reserve(skb, pad);
 224         skb_put(skb, packet_length);
 225         skb_checksum_none_assert(skb);
 226
 227         /* Get parital ethernet information instead of eth_type_trans(),
 228          * Since we don't have an associated net_device.
 229          */
 230         skb_reset_mac_header(skb);
 231         skb->protocol = eth_hdr(skb)->h_proto;
 232
 233         /* Pass SKB onward */
 234         if (cdev->ll2->cbs && cdev->ll2->cbs->rx_cb) {
 235                 if (vlan)
 236                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan);
 237                 cdev->ll2->cbs->rx_cb(cdev->ll2->cb_cookie, skb,
 238                                       opaque_data_0, opaque_data_1);
 239         }
 240
 241         /* Update Buffer information and update FW producer */
 242         buffer->data = new_data;
 243         buffer->phys_addr = new_phys_addr;
 244
 245 out_post:
 246         rc = qed_ll2_post_rx_buffer(QED_LEADING_HWFN(cdev), cdev->ll2->handle,
 247                                     buffer->phys_addr, 0,  buffer, 1);
 248
 249         if (rc)
 250                 qed_ll2_dealloc_buffer(cdev, buffer);
 251 }
 252
 253 static struct qed_ll2_info *__qed_ll2_handle_sanity(struct qed_hwfn *p_hwfn,
 254                                                     u8 connection_handle,
 255                                                     bool b_lock,
 256                                                     bool b_only_active)
 257 {
 258         struct qed_ll2_info *p_ll2_conn, *p_ret = NULL;
 259
 260         if (connection_handle >= QED_MAX_NUM_OF_LL2_CONNECTIONS)
 261                 return NULL;
 262
 263         if (!p_hwfn->p_ll2_info)
 264                 return NULL;
 265
 266         p_ll2_conn = &p_hwfn->p_ll2_info[connection_handle];
 267
 268         if (b_only_active) {
 269                 if (b_lock)
 270                         mutex_lock(&p_ll2_conn->mutex);
 271                 if (p_ll2_conn->b_active)
 272                         p_ret = p_ll2_conn;
 273                 if (b_lock)
 274                         mutex_unlock(&p_ll2_conn->mutex);
 275         } else {
 276                 p_ret = p_ll2_conn;
 277         }
 278
 279         return p_ret;
 280 }
 281
 282 static struct qed_ll2_info *qed_ll2_handle_sanity(struct qed_hwfn *p_hwfn,
 283                                                   u8 connection_handle)
 284 {
 285         return __qed_ll2_handle_sanity(p_hwfn, connection_handle, false, true);
 286 }
 287
 288 static struct qed_ll2_info *qed_ll2_handle_sanity_lock(struct qed_hwfn *p_hwfn,
 289                                                        u8 connection_handle)
 290 {
 291         return __qed_ll2_handle_sanity(p_hwfn, connection_handle, true, true);
 292 }
 293
 294 static struct qed_ll2_info *qed_ll2_handle_sanity_inactive(struct qed_hwfn
 295                                                            *p_hwfn,
 296                                                            u8 connection_handle)
 297 {
 298         return __qed_ll2_handle_sanity(p_hwfn, connection_handle, false, false);
 299 }
 300
 301 static void qed_ll2_txq_flush(struct qed_hwfn *p_hwfn, u8 connection_handle)
 302 {
 303         bool b_last_packet = false, b_last_frag = false;
 304         struct qed_ll2_tx_packet *p_pkt = NULL;
 305         struct qed_ll2_info *p_ll2_conn;
 306         struct qed_ll2_tx_queue *p_tx;
 307         dma_addr_t tx_frag;
 308
 309         p_ll2_conn = qed_ll2_handle_sanity_inactive(p_hwfn, connection_handle);
 310         if (!p_ll2_conn)
 311                 return;
 312
 313         p_tx = &p_ll2_conn->tx_queue;
 314
 315         while (!list_empty(&p_tx->active_descq)) {
 316                 p_pkt = list_first_entry(&p_tx->active_descq,
 317                                          struct qed_ll2_tx_packet, list_entry);
 318                 if (!p_pkt)
 319                         break;
 320
 321                 list_del(&p_pkt->list_entry);
 322                 b_last_packet = list_empty(&p_tx->active_descq);
 323                 list_add_tail(&p_pkt->list_entry, &p_tx->free_descq);
 324                 if (p_ll2_conn->conn.conn_type == QED_LL2_TYPE_ISCSI_OOO) {
 325                         struct qed_ooo_buffer *p_buffer;
 326
 327                         p_buffer = (struct qed_ooo_buffer *)p_pkt->cookie;
 328                         qed_ooo_put_free_buffer(p_hwfn, p_hwfn->p_ooo_info,
 329                                                 p_buffer);
 330                 } else {
 331                         p_tx->cur_completing_packet = *p_pkt;
 332                         p_tx->cur_completing_bd_idx = 1;
 333                         b_last_frag =
 334                                 p_tx->cur_completing_bd_idx == p_pkt->bd_used;
 335                         tx_frag = p_pkt->bds_set[0].tx_frag;
 336                         if (p_ll2_conn->conn.gsi_enable)
 337                                 qed_ll2b_release_tx_gsi_packet(p_hwfn,
 338                                                                p_ll2_conn->
 339                                                                my_id,
 340                                                                p_pkt->cookie,
 341                                                                tx_frag,
 342                                                                b_last_frag,
 343                                                                b_last_packet);
 344                         else
 345                                 qed_ll2b_complete_tx_packet(p_hwfn,
 346                                                             p_ll2_conn->my_id,
 347                                                             p_pkt->cookie,
 348                                                             tx_frag,
 349                                                             b_last_frag,
 350                                                             b_last_packet);
 351                 }
 352         }
 353 }
 354
 355 static int qed_ll2_txq_completion(struct qed_hwfn *p_hwfn, void *p_cookie)
 356 {
 357         struct qed_ll2_info *p_ll2_conn = p_cookie;
 358         struct qed_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue;
 359         u16 new_idx = 0, num_bds = 0, num_bds_in_packet = 0;
 360         struct qed_ll2_tx_packet *p_pkt;
 361         bool b_last_frag = false;
 362         unsigned long flags;
 363         dma_addr_t tx_frag;
 364         int rc = -EINVAL;
 365
 366         spin_lock_irqsave(&p_tx->lock, flags);
 367         if (p_tx->b_completing_packet) {
 368                 rc = -EBUSY;
 369                 goto out;
 370         }
 371
 372         new_idx = le16_to_cpu(*p_tx->p_fw_cons);
 373         num_bds = ((s16)new_idx - (s16)p_tx->bds_idx);
 374         while (num_bds) {
 375                 if (list_empty(&p_tx->active_descq))
 376                         goto out;
 377
 378                 p_pkt = list_first_entry(&p_tx->active_descq,
 379                                          struct qed_ll2_tx_packet, list_entry);
 380                 if (!p_pkt)
 381                         goto out;
 382
 383                 p_tx->b_completing_packet = true;
 384                 p_tx->cur_completing_packet = *p_pkt;
 385                 num_bds_in_packet = p_pkt->bd_used;
 386                 list_del(&p_pkt->list_entry);
 387
 388                 if (num_bds < num_bds_in_packet) {
 389                         DP_NOTICE(p_hwfn,
 390                                   "Rest of BDs does not cover whole packet\n");
 391                         goto out;
 392                 }
 393
 394                 num_bds -= num_bds_in_packet;
 395                 p_tx->bds_idx += num_bds_in_packet;
 396                 while (num_bds_in_packet--)
 397                         qed_chain_consume(&p_tx->txq_chain);
 398
 399                 p_tx->cur_completing_bd_idx = 1;
 400                 b_last_frag = p_tx->cur_completing_bd_idx == p_pkt->bd_used;
 401                 list_add_tail(&p_pkt->list_entry, &p_tx->free_descq);
 402
 403                 spin_unlock_irqrestore(&p_tx->lock, flags);
 404                 tx_frag = p_pkt->bds_set[0].tx_frag;
 405                 if (p_ll2_conn->conn.gsi_enable)
 406                         qed_ll2b_complete_tx_gsi_packet(p_hwfn,
 407                                                         p_ll2_conn->my_id,
 408                                                         p_pkt->cookie,
 409                                                         tx_frag,
 410                                                         b_last_frag, !num_bds);
 411                 else
 412                         qed_ll2b_complete_tx_packet(p_hwfn,
 413                                                     p_ll2_conn->my_id,
 414                                                     p_pkt->cookie,
 415                                                     tx_frag,
 416                                                     b_last_frag, !num_bds);
 417                 spin_lock_irqsave(&p_tx->lock, flags);
 418         }
 419
 420         p_tx->b_completing_packet = false;
 421         rc = 0;
 422 out:
 423         spin_unlock_irqrestore(&p_tx->lock, flags);
 424         return rc;
 425 }
 426
 427 static int
 428 qed_ll2_rxq_completion_gsi(struct qed_hwfn *p_hwfn,
 429                            struct qed_ll2_info *p_ll2_info,
 430                            union core_rx_cqe_union *p_cqe,
 431                            unsigned long lock_flags, bool b_last_cqe)
 432 {
 433         struct qed_ll2_rx_queue *p_rx = &p_ll2_info->rx_queue;
 434         struct qed_ll2_rx_packet *p_pkt = NULL;
 435         u16 packet_length, parse_flags, vlan;
 436         u32 src_mac_addrhi;
 437         u16 src_mac_addrlo;
 438
 439         if (!list_empty(&p_rx->active_descq))
 440                 p_pkt = list_first_entry(&p_rx->active_descq,
 441                                          struct qed_ll2_rx_packet, list_entry);
 442         if (!p_pkt) {
 443                 DP_NOTICE(p_hwfn,
 444                           "GSI Rx completion but active_descq is empty\n");
 445                 return -EIO;
 446         }
 447
 448         list_del(&p_pkt->list_entry);
 449         parse_flags = le16_to_cpu(p_cqe->rx_cqe_gsi.parse_flags.flags);
 450         packet_length = le16_to_cpu(p_cqe->rx_cqe_gsi.data_length);
 451         vlan = le16_to_cpu(p_cqe->rx_cqe_gsi.vlan);
 452         src_mac_addrhi = le32_to_cpu(p_cqe->rx_cqe_gsi.src_mac_addrhi);
 453         src_mac_addrlo = le16_to_cpu(p_cqe->rx_cqe_gsi.src_mac_addrlo);
 454         if (qed_chain_consume(&p_rx->rxq_chain) != p_pkt->rxq_bd)
 455                 DP_NOTICE(p_hwfn,
 456                           "Mismatch between active_descq and the LL2 Rx chain\n");
 457         list_add_tail(&p_pkt->list_entry, &p_rx->free_descq);
 458
 459         spin_unlock_irqrestore(&p_rx->lock, lock_flags);
 460         qed_ll2b_complete_rx_gsi_packet(p_hwfn,
 461                                         p_ll2_info->my_id,
 462                                         p_pkt->cookie,
 463                                         p_pkt->rx_buf_addr,
 464                                         packet_length,
 465                                         p_cqe->rx_cqe_gsi.data_length_error,
 466                                         parse_flags,
 467                                         vlan,
 468                                         src_mac_addrhi,
 469                                         src_mac_addrlo, b_last_cqe);
 470         spin_lock_irqsave(&p_rx->lock, lock_flags);
 471
 472         return 0;
 473 }
 474
 475 static int qed_ll2_rxq_completion_reg(struct qed_hwfn *p_hwfn,
 476                                       struct qed_ll2_info *p_ll2_conn,
 477                                       union core_rx_cqe_union *p_cqe,
 478                                       unsigned long *p_lock_flags,
 479                                       bool b_last_cqe)
 480 {
 481         struct qed_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue;
 482         struct qed_ll2_rx_packet *p_pkt = NULL;
 483
 484         if (!list_empty(&p_rx->active_descq))
 485                 p_pkt = list_first_entry(&p_rx->active_descq,
 486                                          struct qed_ll2_rx_packet, list_entry);
 487         if (!p_pkt) {
 488                 DP_NOTICE(p_hwfn,
 489                           "LL2 Rx completion but active_descq is empty\n");
 490                 return -EIO;
 491         }
 492         list_del(&p_pkt->list_entry);
 493
 494         if (qed_chain_consume(&p_rx->rxq_chain) != p_pkt->rxq_bd)
 495                 DP_NOTICE(p_hwfn,
 496                           "Mismatch between active_descq and the LL2 Rx chain\n");
 497         list_add_tail(&p_pkt->list_entry, &p_rx->free_descq);
 498
 499         spin_unlock_irqrestore(&p_rx->lock, *p_lock_flags);
 500         qed_ll2b_complete_rx_packet(p_hwfn, p_ll2_conn->my_id,
 501                                     p_pkt, &p_cqe->rx_cqe_fp, b_last_cqe);
 502         spin_lock_irqsave(&p_rx->lock, *p_lock_flags);
 503
 504         return 0;
 505 }
 506
 507 static int qed_ll2_rxq_completion(struct qed_hwfn *p_hwfn, void *cookie)
 508 {
 509         struct qed_ll2_info *p_ll2_conn = cookie;
 510         struct qed_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue;
 511         union core_rx_cqe_union *cqe = NULL;
 512         u16 cq_new_idx = 0, cq_old_idx = 0;
 513         unsigned long flags = 0;
 514         int rc = 0;
 515
 516         spin_lock_irqsave(&p_rx->lock, flags);
 517         cq_new_idx = le16_to_cpu(*p_rx->p_fw_cons);
 518         cq_old_idx = qed_chain_get_cons_idx(&p_rx->rcq_chain);
 519
 520         while (cq_new_idx != cq_old_idx) {
 521                 bool b_last_cqe = (cq_new_idx == cq_old_idx);
 522
 523                 cqe = qed_chain_consume(&p_rx->rcq_chain);
 524                 cq_old_idx = qed_chain_get_cons_idx(&p_rx->rcq_chain);
 525
 526                 DP_VERBOSE(p_hwfn,
 527                            QED_MSG_LL2,
 528                            "LL2 [sw. cons %04x, fw. at %04x] - Got Packet of type %02x\n",
 529                            cq_old_idx, cq_new_idx, cqe->rx_cqe_sp.type);
 530
 531                 switch (cqe->rx_cqe_sp.type) {
 532                 case CORE_RX_CQE_TYPE_SLOW_PATH:
 533                         DP_NOTICE(p_hwfn, "LL2 - unexpected Rx CQE slowpath\n");
 534                         rc = -EINVAL;
 535                         break;
 536                 case CORE_RX_CQE_TYPE_GSI_OFFLOAD:
 537                         rc = qed_ll2_rxq_completion_gsi(p_hwfn, p_ll2_conn,
 538                                                         cqe, flags, b_last_cqe);
 539                         break;
 540                 case CORE_RX_CQE_TYPE_REGULAR:
 541                         rc = qed_ll2_rxq_completion_reg(p_hwfn, p_ll2_conn,
 542                                                         cqe, &flags,
 543                                                         b_last_cqe);
 544                         break;
 545                 default:
 546                         rc = -EIO;
 547                 }
 548         }
 549
 550         spin_unlock_irqrestore(&p_rx->lock, flags);
 551         return rc;
 552 }
 553
 554 static void qed_ll2_rxq_flush(struct qed_hwfn *p_hwfn, u8 connection_handle)
 555 {
 556         struct qed_ll2_info *p_ll2_conn = NULL;
 557         struct qed_ll2_rx_packet *p_pkt = NULL;
 558         struct qed_ll2_rx_queue *p_rx;
 559
 560         p_ll2_conn = qed_ll2_handle_sanity_inactive(p_hwfn, connection_handle);
 561         if (!p_ll2_conn)
 562                 return;
 563
 564         p_rx = &p_ll2_conn->rx_queue;
 565
 566         while (!list_empty(&p_rx->active_descq)) {
 567                 dma_addr_t rx_buf_addr;
 568                 void *cookie;
 569                 bool b_last;
 570
 571                 p_pkt = list_first_entry(&p_rx->active_descq,
 572                                          struct qed_ll2_rx_packet, list_entry);
 573                 if (!p_pkt)
 574                         break;
 575
 576                 list_move_tail(&p_pkt->list_entry, &p_rx->free_descq);
 577
 578                 if (p_ll2_conn->conn.conn_type == QED_LL2_TYPE_ISCSI_OOO) {
 579                         struct qed_ooo_buffer *p_buffer;
 580
 581                         p_buffer = (struct qed_ooo_buffer *)p_pkt->cookie;
 582                         qed_ooo_put_free_buffer(p_hwfn, p_hwfn->p_ooo_info,
 583                                                 p_buffer);
 584                 } else {
 585                         rx_buf_addr = p_pkt->rx_buf_addr;
 586                         cookie = p_pkt->cookie;
 587
 588                         b_last = list_empty(&p_rx->active_descq);
 589                 }
 590         }
 591 }
 592
 593 #if IS_ENABLED(CONFIG_QED_ISCSI)
 594 static u8 qed_ll2_convert_rx_parse_to_tx_flags(u16 parse_flags)
 595 {
 596         u8 bd_flags = 0;
 597
 598         if (GET_FIELD(parse_flags, PARSING_AND_ERR_FLAGS_TAG8021QEXIST))
 599                 SET_FIELD(bd_flags, CORE_TX_BD_DATA_VLAN_INSERTION, 1);
 600
 601         return bd_flags;
 602 }
 603
 604 static int qed_ll2_lb_rxq_handler(struct qed_hwfn *p_hwfn,
 605                                   struct qed_ll2_info *p_ll2_conn)
 606 {
 607         struct qed_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue;
 608         u16 packet_length = 0, parse_flags = 0, vlan = 0;
 609         struct qed_ll2_rx_packet *p_pkt = NULL;
 610         u32 num_ooo_add_to_peninsula = 0, cid;
 611         union core_rx_cqe_union *cqe = NULL;
 612         u16 cq_new_idx = 0, cq_old_idx = 0;
 613         struct qed_ooo_buffer *p_buffer;
 614         struct ooo_opaque *iscsi_ooo;
 615         u8 placement_offset = 0;
 616         u8 cqe_type;
 617
 618         cq_new_idx = le16_to_cpu(*p_rx->p_fw_cons);
 619         cq_old_idx = qed_chain_get_cons_idx(&p_rx->rcq_chain);
 620         if (cq_new_idx == cq_old_idx)
 621                 return 0;
 622
 623         while (cq_new_idx != cq_old_idx) {
 624                 struct core_rx_fast_path_cqe *p_cqe_fp;
 625
 626                 cqe = qed_chain_consume(&p_rx->rcq_chain);
 627                 cq_old_idx = qed_chain_get_cons_idx(&p_rx->rcq_chain);
 628                 cqe_type = cqe->rx_cqe_sp.type;
 629
 630                 if (cqe_type != CORE_RX_CQE_TYPE_REGULAR) {
 631                         DP_NOTICE(p_hwfn,
 632                                   "Got a non-regular LB LL2 completion [type 0x%02x]\n",
 633                                   cqe_type);
 634                         return -EINVAL;
 635                 }
 636                 p_cqe_fp = &cqe->rx_cqe_fp;
 637
 638                 placement_offset = p_cqe_fp->placement_offset;
 639                 parse_flags = le16_to_cpu(p_cqe_fp->parse_flags.flags);
 640                 packet_length = le16_to_cpu(p_cqe_fp->packet_length);
 641                 vlan = le16_to_cpu(p_cqe_fp->vlan);
 642                 iscsi_ooo = (struct ooo_opaque *)&p_cqe_fp->opaque_data;
 643                 qed_ooo_save_history_entry(p_hwfn, p_hwfn->p_ooo_info,
 644                                            iscsi_ooo);
 645                 cid = le32_to_cpu(iscsi_ooo->cid);
 646
 647                 /* Process delete isle first */
 648                 if (iscsi_ooo->drop_size)
 649                         qed_ooo_delete_isles(p_hwfn, p_hwfn->p_ooo_info, cid,
 650                                              iscsi_ooo->drop_isle,
 651                                              iscsi_ooo->drop_size);
 652
 653                 if (iscsi_ooo->ooo_opcode == TCP_EVENT_NOP)
 654                         continue;
 655
 656                 /* Now process create/add/join isles */
 657                 if (list_empty(&p_rx->active_descq)) {
 658                         DP_NOTICE(p_hwfn,
 659                                   "LL2 OOO RX chain has no submitted buffers\n"
 660                                   );
 661                         return -EIO;
 662                 }
 663
 664                 p_pkt = list_first_entry(&p_rx->active_descq,
 665                                          struct qed_ll2_rx_packet, list_entry);
 666
 667                 if ((iscsi_ooo->ooo_opcode == TCP_EVENT_ADD_NEW_ISLE) ||
 668                     (iscsi_ooo->ooo_opcode == TCP_EVENT_ADD_ISLE_RIGHT) ||
 669                     (iscsi_ooo->ooo_opcode == TCP_EVENT_ADD_ISLE_LEFT) ||
 670                     (iscsi_ooo->ooo_opcode == TCP_EVENT_ADD_PEN) ||
 671                     (iscsi_ooo->ooo_opcode == TCP_EVENT_JOIN)) {
 672                         if (!p_pkt) {
 673                                 DP_NOTICE(p_hwfn,
 674                                           "LL2 OOO RX packet is not valid\n");
 675                                 return -EIO;
 676                         }
 677                         list_del(&p_pkt->list_entry);
 678                         p_buffer = (struct qed_ooo_buffer *)p_pkt->cookie;
 679                         p_buffer->packet_length = packet_length;
 680                         p_buffer->parse_flags = parse_flags;
 681                         p_buffer->vlan = vlan;
 682                         p_buffer->placement_offset = placement_offset;
 683                         qed_chain_consume(&p_rx->rxq_chain);
 684                         list_add_tail(&p_pkt->list_entry, &p_rx->free_descq);
 685
 686                         switch (iscsi_ooo->ooo_opcode) {
 687                         case TCP_EVENT_ADD_NEW_ISLE:
 688                                 qed_ooo_add_new_isle(p_hwfn,
 689                                                      p_hwfn->p_ooo_info,
 690                                                      cid,
 691                                                      iscsi_ooo->ooo_isle,
 692                                                      p_buffer);
 693                                 break;
 694                         case TCP_EVENT_ADD_ISLE_RIGHT:
 695                                 qed_ooo_add_new_buffer(p_hwfn,
 696                                                        p_hwfn->p_ooo_info,
 697                                                        cid,
 698                                                        iscsi_ooo->ooo_isle,
 699                                                        p_buffer,
 700                                                        QED_OOO_RIGHT_BUF);
 701                                 break;
 702                         case TCP_EVENT_ADD_ISLE_LEFT:
 703                                 qed_ooo_add_new_buffer(p_hwfn,
 704                                                        p_hwfn->p_ooo_info,
 705                                                        cid,
 706                                                        iscsi_ooo->ooo_isle,
 707                                                        p_buffer,
 708                                                        QED_OOO_LEFT_BUF);
 709                                 break;
 710                         case TCP_EVENT_JOIN:
 711                                 qed_ooo_add_new_buffer(p_hwfn,
 712                                                        p_hwfn->p_ooo_info,
 713                                                        cid,
 714                                                        iscsi_ooo->ooo_isle +
 715                                                        1,
 716                                                        p_buffer,
 717                                                        QED_OOO_LEFT_BUF);
 718                                 qed_ooo_join_isles(p_hwfn,
 719                                                    p_hwfn->p_ooo_info,
 720                                                    cid, iscsi_ooo->ooo_isle);
 721                                 break;
 722                         case TCP_EVENT_ADD_PEN:
 723                                 num_ooo_add_to_peninsula++;
 724                                 qed_ooo_put_ready_buffer(p_hwfn,
 725                                                          p_hwfn->p_ooo_info,
 726                                                          p_buffer, true);
 727                                 break;
 728                         }
 729                 } else {
 730                         DP_NOTICE(p_hwfn,
 731                                   "Unexpected event (%d) TX OOO completion\n",
 732                                   iscsi_ooo->ooo_opcode);
 733                 }
 734         }
 735
 736         return 0;
 737 }
 738
 739 static void
 740 qed_ooo_submit_tx_buffers(struct qed_hwfn *p_hwfn,
 741                           struct qed_ll2_info *p_ll2_conn)
 742 {
 743         struct qed_ooo_buffer *p_buffer;
 744         int rc;
 745         u16 l4_hdr_offset_w;
 746         dma_addr_t first_frag;
 747         u16 parse_flags;
 748         u8 bd_flags;
 749
 750         /* Submit Tx buffers here */
 751         while ((p_buffer = qed_ooo_get_ready_buffer(p_hwfn,
 752                                                     p_hwfn->p_ooo_info))) {
 753                 l4_hdr_offset_w = 0;
 754                 bd_flags = 0;
 755
 756                 first_frag = p_buffer->rx_buffer_phys_addr +
 757                              p_buffer->placement_offset;
 758                 parse_flags = p_buffer->parse_flags;
 759                 bd_flags = qed_ll2_convert_rx_parse_to_tx_flags(parse_flags);
 760                 SET_FIELD(bd_flags, CORE_TX_BD_DATA_FORCE_VLAN_MODE, 1);
 761                 SET_FIELD(bd_flags, CORE_TX_BD_DATA_L4_PROTOCOL, 1);
 762
 763                 rc = qed_ll2_prepare_tx_packet(p_hwfn, p_ll2_conn->my_id, 1,
 764                                                p_buffer->vlan, bd_flags,
 765                                                l4_hdr_offset_w,
 766                                                p_ll2_conn->conn.tx_dest, 0,
 767                                                first_frag,
 768                                                p_buffer->packet_length,
 769                                                p_buffer, true);
 770                 if (rc) {
 771                         qed_ooo_put_ready_buffer(p_hwfn, p_hwfn->p_ooo_info,
 772                                                  p_buffer, false);
 773                         break;
 774                 }
 775         }
 776 }
 777
 778 static void
 779 qed_ooo_submit_rx_buffers(struct qed_hwfn *p_hwfn,
 780                           struct qed_ll2_info *p_ll2_conn)
 781 {
 782         struct qed_ooo_buffer *p_buffer;
 783         int rc;
 784
 785         while ((p_buffer = qed_ooo_get_free_buffer(p_hwfn,
 786                                                    p_hwfn->p_ooo_info))) {
 787                 rc = qed_ll2_post_rx_buffer(p_hwfn,
 788                                             p_ll2_conn->my_id,
 789                                             p_buffer->rx_buffer_phys_addr,
 790                                             0, p_buffer, true);
 791                 if (rc) {
 792                         qed_ooo_put_free_buffer(p_hwfn,
 793                                                 p_hwfn->p_ooo_info, p_buffer);
 794                         break;
 795                 }
 796         }
 797 }
 798
 799 static int qed_ll2_lb_rxq_completion(struct qed_hwfn *p_hwfn, void *p_cookie)
 800 {
 801         struct qed_ll2_info *p_ll2_conn = (struct qed_ll2_info *)p_cookie;
 802         int rc;
 803
 804         rc = qed_ll2_lb_rxq_handler(p_hwfn, p_ll2_conn);
 805         if (rc)
 806                 return rc;
 807
 808         qed_ooo_submit_rx_buffers(p_hwfn, p_ll2_conn);
 809         qed_ooo_submit_tx_buffers(p_hwfn, p_ll2_conn);
 810
 811         return 0;
 812 }
 813
 814 static int qed_ll2_lb_txq_completion(struct qed_hwfn *p_hwfn, void *p_cookie)
 815 {
 816         struct qed_ll2_info *p_ll2_conn = (struct qed_ll2_info *)p_cookie;
 817         struct qed_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue;
 818         struct qed_ll2_tx_packet *p_pkt = NULL;
 819         struct qed_ooo_buffer *p_buffer;
 820         bool b_dont_submit_rx = false;
 821         u16 new_idx = 0, num_bds = 0;
 822         int rc;
 823
 824         new_idx = le16_to_cpu(*p_tx->p_fw_cons);
 825         num_bds = ((s16)new_idx - (s16)p_tx->bds_idx);
 826
 827         if (!num_bds)
 828                 return 0;
 829
 830         while (num_bds) {
 831                 if (list_empty(&p_tx->active_descq))
 832                         return -EINVAL;
 833
 834                 p_pkt = list_first_entry(&p_tx->active_descq,
 835                                          struct qed_ll2_tx_packet, list_entry);
 836                 if (!p_pkt)
 837                         return -EINVAL;
 838
 839                 if (p_pkt->bd_used != 1) {
 840                         DP_NOTICE(p_hwfn,
 841                                   "Unexpectedly many BDs(%d) in TX OOO completion\n",
 842                                   p_pkt->bd_used);
 843                         return -EINVAL;
 844                 }
 845
 846                 list_del(&p_pkt->list_entry);
 847
 848                 num_bds--;
 849                 p_tx->bds_idx++;
 850                 qed_chain_consume(&p_tx->txq_chain);
 851
 852                 p_buffer = (struct qed_ooo_buffer *)p_pkt->cookie;
 853                 list_add_tail(&p_pkt->list_entry, &p_tx->free_descq);
 854
 855                 if (b_dont_submit_rx) {
 856                         qed_ooo_put_free_buffer(p_hwfn, p_hwfn->p_ooo_info,
 857                                                 p_buffer);
 858                         continue;
 859                 }
 860
 861                 rc = qed_ll2_post_rx_buffer(p_hwfn, p_ll2_conn->my_id,
 862                                             p_buffer->rx_buffer_phys_addr, 0,
 863                                             p_buffer, true);
 864                 if (rc != 0) {
 865                         qed_ooo_put_free_buffer(p_hwfn,
 866                                                 p_hwfn->p_ooo_info, p_buffer);
 867                         b_dont_submit_rx = true;
 868                 }
 869         }
 870
 871         qed_ooo_submit_tx_buffers(p_hwfn, p_ll2_conn);
 872
 873         return 0;
 874 }
 875
 876 static int
 877 qed_ll2_acquire_connection_ooo(struct qed_hwfn *p_hwfn,
 878                                struct qed_ll2_info *p_ll2_info,
 879                                u16 rx_num_ooo_buffers, u16 mtu)
 880 {
 881         struct qed_ooo_buffer *p_buf = NULL;
 882         void *p_virt;
 883         u16 buf_idx;
 884         int rc = 0;
 885
 886         if (p_ll2_info->conn.conn_type != QED_LL2_TYPE_ISCSI_OOO)
 887                 return rc;
 888
 889         if (!rx_num_ooo_buffers)
 890                 return -EINVAL;
 891
 892         for (buf_idx = 0; buf_idx < rx_num_ooo_buffers; buf_idx++) {
 893                 p_buf = kzalloc(sizeof(*p_buf), GFP_KERNEL);
 894                 if (!p_buf) {
 895                         rc = -ENOMEM;
 896                         goto out;
 897                 }
 898
 899                 p_buf->rx_buffer_size = mtu + 26 + ETH_CACHE_LINE_SIZE;
 900                 p_buf->rx_buffer_size = (p_buf->rx_buffer_size +
 901                                          ETH_CACHE_LINE_SIZE - 1) &
 902                                         ~(ETH_CACHE_LINE_SIZE - 1);
 903                 p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
 904                                             p_buf->rx_buffer_size,
 905                                             &p_buf->rx_buffer_phys_addr,
 906                                             GFP_KERNEL);
 907                 if (!p_virt) {
 908                         kfree(p_buf);
 909                         rc = -ENOMEM;
 910                         goto out;
 911                 }
 912
 913                 p_buf->rx_buffer_virt_addr = p_virt;
 914                 qed_ooo_put_free_buffer(p_hwfn, p_hwfn->p_ooo_info, p_buf);
 915         }
 916
 917         DP_VERBOSE(p_hwfn, QED_MSG_LL2,
 918                    "Allocated [%04x] LL2 OOO buffers [each of size 0x%08x]\n",
 919                    rx_num_ooo_buffers, p_buf->rx_buffer_size);
 920
 921 out:
 922         return rc;
 923 }
 924
 925 static void
 926 qed_ll2_establish_connection_ooo(struct qed_hwfn *p_hwfn,
 927                                  struct qed_ll2_info *p_ll2_conn)
 928 {
 929         if (p_ll2_conn->conn.conn_type != QED_LL2_TYPE_ISCSI_OOO)
 930                 return;
 931
 932         qed_ooo_release_all_isles(p_hwfn, p_hwfn->p_ooo_info);
 933         qed_ooo_submit_rx_buffers(p_hwfn, p_ll2_conn);
 934 }
 935
 936 static void qed_ll2_release_connection_ooo(struct qed_hwfn *p_hwfn,
 937                                            struct qed_ll2_info *p_ll2_conn)
 938 {
 939         struct qed_ooo_buffer *p_buffer;
 940
 941         if (p_ll2_conn->conn.conn_type != QED_LL2_TYPE_ISCSI_OOO)
 942                 return;
 943
 944         qed_ooo_release_all_isles(p_hwfn, p_hwfn->p_ooo_info);
 945         while ((p_buffer = qed_ooo_get_free_buffer(p_hwfn,
 946                                                    p_hwfn->p_ooo_info))) {
 947                 dma_free_coherent(&p_hwfn->cdev->pdev->dev,
 948                                   p_buffer->rx_buffer_size,
 949                                   p_buffer->rx_buffer_virt_addr,
 950                                   p_buffer->rx_buffer_phys_addr);
 951                 kfree(p_buffer);
 952         }
 953 }
 954
 955 static void qed_ll2_stop_ooo(struct qed_dev *cdev)
 956 {
 957         struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
 958         u8 *handle = &hwfn->pf_params.iscsi_pf_params.ll2_ooo_queue_id;
 959
 960         DP_VERBOSE(cdev, QED_MSG_STORAGE, "Stopping LL2 OOO queue [%02x]\n",
 961                    *handle);
 962
 963         qed_ll2_terminate_connection(hwfn, *handle);
 964         qed_ll2_release_connection(hwfn, *handle);
 965         *handle = QED_LL2_UNUSED_HANDLE;
 966 }
 967
 968 static int qed_ll2_start_ooo(struct qed_dev *cdev,
 969                              struct qed_ll2_params *params)
 970 {
 971         struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
 972         u8 *handle = &hwfn->pf_params.iscsi_pf_params.ll2_ooo_queue_id;
 973         struct qed_ll2_conn ll2_info = { 0 };
 974         int rc;
 975
 976         ll2_info.conn_type = QED_LL2_TYPE_ISCSI_OOO;
 977         ll2_info.mtu = params->mtu;
 978         ll2_info.rx_drop_ttl0_flg = params->drop_ttl0_packets;
 979         ll2_info.rx_vlan_removal_en = params->rx_vlan_stripping;
 980         ll2_info.tx_tc = OOO_LB_TC;
 981         ll2_info.tx_dest = CORE_TX_DEST_LB;
 982
 983         rc = qed_ll2_acquire_connection(hwfn, &ll2_info,
 984                                         QED_LL2_RX_SIZE, QED_LL2_TX_SIZE,
 985                                         handle);
 986         if (rc) {
 987                 DP_INFO(cdev, "Failed to acquire LL2 OOO connection\n");
 988                 goto out;
 989         }
 990
 991         rc = qed_ll2_establish_connection(hwfn, *handle);
 992         if (rc) {
 993                 DP_INFO(cdev, "Failed to establist LL2 OOO connection\n");
 994                 goto fail;
 995         }
 996
 997         return 0;
 998
 999 fail:
1000         qed_ll2_release_connection(hwfn, *handle);
1001 out:
1002         *handle = QED_LL2_UNUSED_HANDLE;
1003         return rc;
1004 }
1005 #else /* IS_ENABLED(CONFIG_QED_ISCSI) */
1006 static int qed_ll2_lb_rxq_completion(struct qed_hwfn *p_hwfn,
1007                                      void *p_cookie) { return -EINVAL; }
1008 static int qed_ll2_lb_txq_completion(struct qed_hwfn *p_hwfn,
1009                                      void *p_cookie) { return -EINVAL; }
1010 static inline int
1011 qed_ll2_acquire_connection_ooo(struct qed_hwfn *p_hwfn,
1012                                struct qed_ll2_info *p_ll2_info,
1013                                u16 rx_num_ooo_buffers, u16 mtu) { return 0; }
1014 static inline void
1015 qed_ll2_establish_connection_ooo(struct qed_hwfn *p_hwfn,
1016                                  struct qed_ll2_info *p_ll2_conn) { return; }
1017 static inline void
1018 qed_ll2_release_connection_ooo(struct qed_hwfn *p_hwfn,
1019                                struct qed_ll2_info *p_ll2_conn) { return; }
1020 static inline void qed_ll2_stop_ooo(struct qed_dev *cdev) { return; }
1021 static inline int qed_ll2_start_ooo(struct qed_dev *cdev,
1022                                     struct qed_ll2_params *params)
1023                                     { return -EINVAL; }
1024 #endif /* IS_ENABLED(CONFIG_QED_ISCSI) */
1025
1026 static int qed_sp_ll2_rx_queue_start(struct qed_hwfn *p_hwfn,
1027                                      struct qed_ll2_info *p_ll2_conn,
1028                                      u8 action_on_error)
1029 {
1030         enum qed_ll2_conn_type conn_type = p_ll2_conn->conn.conn_type;
1031         struct qed_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue;
1032         struct core_rx_start_ramrod_data *p_ramrod = NULL;
1033         struct qed_spq_entry *p_ent = NULL;
1034         struct qed_sp_init_data init_data;
1035         u16 cqe_pbl_size;
1036         int rc = 0;
1037
1038         /* Get SPQ entry */
1039         memset(&init_data, 0, sizeof(init_data));
1040         init_data.cid = p_ll2_conn->cid;
1041         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
1042         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
1043
1044         rc = qed_sp_init_request(p_hwfn, &p_ent,
1045                                  CORE_RAMROD_RX_QUEUE_START,
1046                                  PROTOCOLID_CORE, &init_data);
1047         if (rc)
1048                 return rc;
1049
1050         p_ramrod = &p_ent->ramrod.core_rx_queue_start;
1051
1052         p_ramrod->sb_id = cpu_to_le16(qed_int_get_sp_sb_id(p_hwfn));
1053         p_ramrod->sb_index = p_rx->rx_sb_index;
1054         p_ramrod->complete_event_flg = 1;
1055
1056         p_ramrod->mtu = cpu_to_le16(p_ll2_conn->conn.mtu);
1057         DMA_REGPAIR_LE(p_ramrod->bd_base,
1058                        p_rx->rxq_chain.p_phys_addr);
1059         cqe_pbl_size = (u16)qed_chain_get_page_cnt(&p_rx->rcq_chain);
1060         p_ramrod->num_of_pbl_pages = cpu_to_le16(cqe_pbl_size);
1061         DMA_REGPAIR_LE(p_ramrod->cqe_pbl_addr,
1062                        qed_chain_get_pbl_phys(&p_rx->rcq_chain));
1063
1064         p_ramrod->drop_ttl0_flg = p_ll2_conn->conn.rx_drop_ttl0_flg;
1065         p_ramrod->inner_vlan_removal_en = p_ll2_conn->conn.rx_vlan_removal_en;
1066         p_ramrod->queue_id = p_ll2_conn->queue_id;
1067         p_ramrod->main_func_queue = (conn_type == QED_LL2_TYPE_ISCSI_OOO) ? 0
1068                                                                           : 1;
1069
1070         if ((IS_MF_DEFAULT(p_hwfn) || IS_MF_SI(p_hwfn)) &&
1071             p_ramrod->main_func_queue && (conn_type != QED_LL2_TYPE_ROCE)) {
1072                 p_ramrod->mf_si_bcast_accept_all = 1;
1073                 p_ramrod->mf_si_mcast_accept_all = 1;
1074         } else {
1075                 p_ramrod->mf_si_bcast_accept_all = 0;
1076                 p_ramrod->mf_si_mcast_accept_all = 0;
1077         }
1078
1079         p_ramrod->action_on_error.error_type = action_on_error;
1080         p_ramrod->gsi_offload_flag = p_ll2_conn->conn.gsi_enable;
1081         return qed_spq_post(p_hwfn, p_ent, NULL);
1082 }
1083
1084 static int qed_sp_ll2_tx_queue_start(struct qed_hwfn *p_hwfn,
1085                                      struct qed_ll2_info *p_ll2_conn)
1086 {
1087         enum qed_ll2_conn_type conn_type = p_ll2_conn->conn.conn_type;
1088         struct qed_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue;
1089         struct core_tx_start_ramrod_data *p_ramrod = NULL;
1090         struct qed_spq_entry *p_ent = NULL;
1091         struct qed_sp_init_data init_data;
1092         u16 pq_id = 0, pbl_size;
1093         int rc = -EINVAL;
1094
1095         if (!QED_LL2_TX_REGISTERED(p_ll2_conn))
1096                 return 0;
1097
1098         if (p_ll2_conn->conn.conn_type == QED_LL2_TYPE_ISCSI_OOO)
1099                 p_ll2_conn->tx_stats_en = 0;
1100         else
1101                 p_ll2_conn->tx_stats_en = 1;
1102
1103         /* Get SPQ entry */
1104         memset(&init_data, 0, sizeof(init_data));
1105         init_data.cid = p_ll2_conn->cid;
1106         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
1107         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
1108
1109         rc = qed_sp_init_request(p_hwfn, &p_ent,
1110                                  CORE_RAMROD_TX_QUEUE_START,
1111                                  PROTOCOLID_CORE, &init_data);
1112         if (rc)
1113                 return rc;
1114
1115         p_ramrod = &p_ent->ramrod.core_tx_queue_start;
1116
1117         p_ramrod->sb_id = cpu_to_le16(qed_int_get_sp_sb_id(p_hwfn));
1118         p_ramrod->sb_index = p_tx->tx_sb_index;
1119         p_ramrod->mtu = cpu_to_le16(p_ll2_conn->conn.mtu);
1120         p_ramrod->stats_en = p_ll2_conn->tx_stats_en;
1121         p_ramrod->stats_id = p_ll2_conn->tx_stats_id;
1122
1123         DMA_REGPAIR_LE(p_ramrod->pbl_base_addr,
1124                        qed_chain_get_pbl_phys(&p_tx->txq_chain));
1125         pbl_size = qed_chain_get_page_cnt(&p_tx->txq_chain);
1126         p_ramrod->pbl_size = cpu_to_le16(pbl_size);
1127
1128         switch (p_ll2_conn->conn.tx_tc) {
1129         case LB_TC:
1130                 pq_id = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LB);
1131                 break;
1132         case OOO_LB_TC:
1133                 pq_id = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OOO);
1134                 break;
1135         default:
1136                 pq_id = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD);
1137                 break;
1138         }
1139
1140         p_ramrod->qm_pq_id = cpu_to_le16(pq_id);
1141
1142         switch (conn_type) {
1143         case QED_LL2_TYPE_FCOE:
1144                 p_ramrod->conn_type = PROTOCOLID_FCOE;
1145                 break;
1146         case QED_LL2_TYPE_ISCSI:
1147         case QED_LL2_TYPE_ISCSI_OOO:
1148                 p_ramrod->conn_type = PROTOCOLID_ISCSI;
1149                 break;
1150         case QED_LL2_TYPE_ROCE:
1151                 p_ramrod->conn_type = PROTOCOLID_ROCE;
1152                 break;
1153         default:
1154                 p_ramrod->conn_type = PROTOCOLID_ETH;
1155                 DP_NOTICE(p_hwfn, "Unknown connection type: %d\n", conn_type);
1156         }
1157
1158         p_ramrod->gsi_offload_flag = p_ll2_conn->conn.gsi_enable;
1159         return qed_spq_post(p_hwfn, p_ent, NULL);
1160 }
1161
1162 static int qed_sp_ll2_rx_queue_stop(struct qed_hwfn *p_hwfn,
1163                                     struct qed_ll2_info *p_ll2_conn)
1164 {
1165         struct core_rx_stop_ramrod_data *p_ramrod = NULL;
1166         struct qed_spq_entry *p_ent = NULL;
1167         struct qed_sp_init_data init_data;
1168         int rc = -EINVAL;
1169
1170         /* Get SPQ entry */
1171         memset(&init_data, 0, sizeof(init_data));
1172         init_data.cid = p_ll2_conn->cid;
1173         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
1174         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
1175
1176         rc = qed_sp_init_request(p_hwfn, &p_ent,
1177                                  CORE_RAMROD_RX_QUEUE_STOP,
1178                                  PROTOCOLID_CORE, &init_data);
1179         if (rc)
1180                 return rc;
1181
1182         p_ramrod = &p_ent->ramrod.core_rx_queue_stop;
1183
1184         p_ramrod->complete_event_flg = 1;
1185         p_ramrod->queue_id = p_ll2_conn->queue_id;
1186
1187         return qed_spq_post(p_hwfn, p_ent, NULL);
1188 }
1189
1190 static int qed_sp_ll2_tx_queue_stop(struct qed_hwfn *p_hwfn,
1191                                     struct qed_ll2_info *p_ll2_conn)
1192 {
1193         struct qed_spq_entry *p_ent = NULL;
1194         struct qed_sp_init_data init_data;
1195         int rc = -EINVAL;
1196
1197         /* Get SPQ entry */
1198         memset(&init_data, 0, sizeof(init_data));
1199         init_data.cid = p_ll2_conn->cid;
1200         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
1201         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
1202
1203         rc = qed_sp_init_request(p_hwfn, &p_ent,
1204                                  CORE_RAMROD_TX_QUEUE_STOP,
1205                                  PROTOCOLID_CORE, &init_data);
1206         if (rc)
1207                 return rc;
1208
1209         return qed_spq_post(p_hwfn, p_ent, NULL);
1210 }
1211
1212 static int
1213 qed_ll2_acquire_connection_rx(struct qed_hwfn *p_hwfn,
1214                               struct qed_ll2_info *p_ll2_info, u16 rx_num_desc)
1215 {
1216         struct qed_ll2_rx_packet *p_descq;
1217         u32 capacity;
1218         int rc = 0;
1219
1220         if (!rx_num_desc)
1221                 goto out;
1222
1223         rc = qed_chain_alloc(p_hwfn->cdev,
1224                              QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1225                              QED_CHAIN_MODE_NEXT_PTR,
1226                              QED_CHAIN_CNT_TYPE_U16,
1227                              rx_num_desc,
1228                              sizeof(struct core_rx_bd),
1229                              &p_ll2_info->rx_queue.rxq_chain);
1230         if (rc) {
1231                 DP_NOTICE(p_hwfn, "Failed to allocate ll2 rxq chain\n");
1232                 goto out;
1233         }
1234
1235         capacity = qed_chain_get_capacity(&p_ll2_info->rx_queue.rxq_chain);
1236         p_descq = kcalloc(capacity, sizeof(struct qed_ll2_rx_packet),
1237                           GFP_KERNEL);
1238         if (!p_descq) {
1239                 rc = -ENOMEM;
1240                 DP_NOTICE(p_hwfn, "Failed to allocate ll2 Rx desc\n");
1241                 goto out;
1242         }
1243         p_ll2_info->rx_queue.descq_array = p_descq;
1244
1245         rc = qed_chain_alloc(p_hwfn->cdev,
1246                              QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1247                              QED_CHAIN_MODE_PBL,
1248                              QED_CHAIN_CNT_TYPE_U16,
1249                              rx_num_desc,
1250                              sizeof(struct core_rx_fast_path_cqe),
1251                              &p_ll2_info->rx_queue.rcq_chain);
1252         if (rc) {
1253                 DP_NOTICE(p_hwfn, "Failed to allocate ll2 rcq chain\n");
1254                 goto out;
1255         }
1256
1257         DP_VERBOSE(p_hwfn, QED_MSG_LL2,
1258                    "Allocated LL2 Rxq [Type %08x] with 0x%08x buffers\n",
1259                    p_ll2_info->conn.conn_type, rx_num_desc);
1260
1261 out:
1262         return rc;
1263 }
1264
1265 static int qed_ll2_acquire_connection_tx(struct qed_hwfn *p_hwfn,
1266                                          struct qed_ll2_info *p_ll2_info,
1267                                          u16 tx_num_desc)
1268 {
1269         struct qed_ll2_tx_packet *p_descq;
1270         u32 capacity;
1271         int rc = 0;
1272
1273         if (!tx_num_desc)
1274                 goto out;
1275
1276         rc = qed_chain_alloc(p_hwfn->cdev,
1277                              QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1278                              QED_CHAIN_MODE_PBL,
1279                              QED_CHAIN_CNT_TYPE_U16,
1280                              tx_num_desc,
1281                              sizeof(struct core_tx_bd),
1282                              &p_ll2_info->tx_queue.txq_chain);
1283         if (rc)
1284                 goto out;
1285
1286         capacity = qed_chain_get_capacity(&p_ll2_info->tx_queue.txq_chain);
1287         p_descq = kcalloc(capacity, sizeof(struct qed_ll2_tx_packet),
1288                           GFP_KERNEL);
1289         if (!p_descq) {
1290                 rc = -ENOMEM;
1291                 goto out;
1292         }
1293         p_ll2_info->tx_queue.descq_array = p_descq;
1294
1295         DP_VERBOSE(p_hwfn, QED_MSG_LL2,
1296                    "Allocated LL2 Txq [Type %08x] with 0x%08x buffers\n",
1297                    p_ll2_info->conn.conn_type, tx_num_desc);
1298
1299 out:
1300         if (rc)
1301                 DP_NOTICE(p_hwfn,
1302                           "Can't allocate memory for Tx LL2 with 0x%08x buffers\n",
1303                           tx_num_desc);
1304         return rc;
1305 }
1306
1307 int qed_ll2_acquire_connection(struct qed_hwfn *p_hwfn,
1308                                struct qed_ll2_conn *p_params,
1309                                u16 rx_num_desc,
1310                                u16 tx_num_desc,
1311                                u8 *p_connection_handle)
1312 {
1313         qed_int_comp_cb_t comp_rx_cb, comp_tx_cb;
1314         struct qed_ll2_info *p_ll2_info = NULL;
1315         int rc;
1316         u8 i;
1317
1318         if (!p_connection_handle || !p_hwfn->p_ll2_info)
1319                 return -EINVAL;
1320
1321         /* Find a free connection to be used */
1322         for (i = 0; (i < QED_MAX_NUM_OF_LL2_CONNECTIONS); i++) {
1323                 mutex_lock(&p_hwfn->p_ll2_info[i].mutex);
1324                 if (p_hwfn->p_ll2_info[i].b_active) {
1325                         mutex_unlock(&p_hwfn->p_ll2_info[i].mutex);
1326                         continue;
1327                 }
1328
1329                 p_hwfn->p_ll2_info[i].b_active = true;
1330                 p_ll2_info = &p_hwfn->p_ll2_info[i];
1331                 mutex_unlock(&p_hwfn->p_ll2_info[i].mutex);
1332                 break;
1333         }
1334         if (!p_ll2_info)
1335                 return -EBUSY;
1336
1337         p_ll2_info->conn = *p_params;
1338
1339         rc = qed_ll2_acquire_connection_rx(p_hwfn, p_ll2_info, rx_num_desc);
1340         if (rc)
1341                 goto q_allocate_fail;
1342
1343         rc = qed_ll2_acquire_connection_tx(p_hwfn, p_ll2_info, tx_num_desc);
1344         if (rc)
1345                 goto q_allocate_fail;
1346
1347         rc = qed_ll2_acquire_connection_ooo(p_hwfn, p_ll2_info,
1348                                             rx_num_desc * 2, p_params->mtu);
1349         if (rc)
1350                 goto q_allocate_fail;
1351
1352         /* Register callbacks for the Rx/Tx queues */
1353         if (p_params->conn_type == QED_LL2_TYPE_ISCSI_OOO) {
1354                 comp_rx_cb = qed_ll2_lb_rxq_completion;
1355                 comp_tx_cb = qed_ll2_lb_txq_completion;
1356         } else {
1357                 comp_rx_cb = qed_ll2_rxq_completion;
1358                 comp_tx_cb = qed_ll2_txq_completion;
1359         }
1360
1361         if (rx_num_desc) {
1362                 qed_int_register_cb(p_hwfn, comp_rx_cb,
1363                                     &p_hwfn->p_ll2_info[i],
1364                                     &p_ll2_info->rx_queue.rx_sb_index,
1365                                     &p_ll2_info->rx_queue.p_fw_cons);
1366                 p_ll2_info->rx_queue.b_cb_registred = true;
1367         }
1368
1369         if (tx_num_desc) {
1370                 qed_int_register_cb(p_hwfn,
1371                                     comp_tx_cb,
1372                                     &p_hwfn->p_ll2_info[i],
1373                                     &p_ll2_info->tx_queue.tx_sb_index,
1374                                     &p_ll2_info->tx_queue.p_fw_cons);
1375                 p_ll2_info->tx_queue.b_cb_registred = true;
1376         }
1377
1378         *p_connection_handle = i;
1379         return rc;
1380
1381 q_allocate_fail:
1382         qed_ll2_release_connection(p_hwfn, i);
1383         return -ENOMEM;
1384 }
1385
1386 static int qed_ll2_establish_connection_rx(struct qed_hwfn *p_hwfn,
1387                                            struct qed_ll2_info *p_ll2_conn)
1388 {
1389         u8 action_on_error = 0;
1390
1391         if (!QED_LL2_RX_REGISTERED(p_ll2_conn))
1392                 return 0;
1393
1394         DIRECT_REG_WR(p_ll2_conn->rx_queue.set_prod_addr, 0x0);
1395
1396         SET_FIELD(action_on_error,
1397                   CORE_RX_ACTION_ON_ERROR_PACKET_TOO_BIG,
1398                   p_ll2_conn->conn.ai_err_packet_too_big);
1399         SET_FIELD(action_on_error,
1400                   CORE_RX_ACTION_ON_ERROR_NO_BUFF, p_ll2_conn->conn.ai_err_no_buf);
1401
1402         return qed_sp_ll2_rx_queue_start(p_hwfn, p_ll2_conn, action_on_error);
1403 }
1404
1405 int qed_ll2_establish_connection(struct qed_hwfn *p_hwfn, u8 connection_handle)
1406 {
1407         struct qed_ll2_info *p_ll2_conn;
1408         struct qed_ll2_rx_queue *p_rx;
1409         struct qed_ll2_tx_queue *p_tx;
1410         struct qed_ptt *p_ptt;
1411         int rc = -EINVAL;
1412         u32 i, capacity;
1413         u8 qid;
1414
1415         p_ptt = qed_ptt_acquire(p_hwfn);
1416         if (!p_ptt)
1417                 return -EAGAIN;
1418
1419         p_ll2_conn = qed_ll2_handle_sanity_lock(p_hwfn, connection_handle);
1420         if (!p_ll2_conn) {
1421                 rc = -EINVAL;
1422                 goto out;
1423         }
1424
1425         p_rx = &p_ll2_conn->rx_queue;
1426         p_tx = &p_ll2_conn->tx_queue;
1427
1428         qed_chain_reset(&p_rx->rxq_chain);
1429         qed_chain_reset(&p_rx->rcq_chain);
1430         INIT_LIST_HEAD(&p_rx->active_descq);
1431         INIT_LIST_HEAD(&p_rx->free_descq);
1432         INIT_LIST_HEAD(&p_rx->posting_descq);
1433         spin_lock_init(&p_rx->lock);
1434         capacity = qed_chain_get_capacity(&p_rx->rxq_chain);
1435         for (i = 0; i < capacity; i++)
1436                 list_add_tail(&p_rx->descq_array[i].list_entry,
1437                               &p_rx->free_descq);
1438         *p_rx->p_fw_cons = 0;
1439
1440         qed_chain_reset(&p_tx->txq_chain);
1441         INIT_LIST_HEAD(&p_tx->active_descq);
1442         INIT_LIST_HEAD(&p_tx->free_descq);
1443         INIT_LIST_HEAD(&p_tx->sending_descq);
1444         spin_lock_init(&p_tx->lock);
1445         capacity = qed_chain_get_capacity(&p_tx->txq_chain);
1446         for (i = 0; i < capacity; i++)
1447                 list_add_tail(&p_tx->descq_array[i].list_entry,
1448                               &p_tx->free_descq);
1449         p_tx->cur_completing_bd_idx = 0;
1450         p_tx->bds_idx = 0;
1451         p_tx->b_completing_packet = false;
1452         p_tx->cur_send_packet = NULL;
1453         p_tx->cur_send_frag_num = 0;
1454         p_tx->cur_completing_frag_num = 0;
1455         *p_tx->p_fw_cons = 0;
1456
1457         rc = qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_CORE, &p_ll2_conn->cid);
1458         if (rc)
1459                 goto out;
1460
1461         qid = p_hwfn->hw_info.resc_start[QED_LL2_QUEUE] + connection_handle;
1462         p_ll2_conn->queue_id = qid;
1463         p_ll2_conn->tx_stats_id = qid;
1464         p_rx->set_prod_addr = (u8 __iomem *)p_hwfn->regview +
1465                                             GTT_BAR0_MAP_REG_TSDM_RAM +
1466                                             TSTORM_LL2_RX_PRODS_OFFSET(qid);
1467         p_tx->doorbell_addr = (u8 __iomem *)p_hwfn->doorbells +
1468                                             qed_db_addr(p_ll2_conn->cid,
1469                                                         DQ_DEMS_LEGACY);
1470
1471         rc = qed_ll2_establish_connection_rx(p_hwfn, p_ll2_conn);
1472         if (rc)
1473                 goto out;
1474
1475         rc = qed_sp_ll2_tx_queue_start(p_hwfn, p_ll2_conn);
1476         if (rc)
1477                 goto out;
1478
1479         if (p_hwfn->hw_info.personality != QED_PCI_ETH_ROCE)
1480                 qed_wr(p_hwfn, p_ptt, PRS_REG_USE_LIGHT_L2, 1);
1481
1482         qed_ll2_establish_connection_ooo(p_hwfn, p_ll2_conn);
1483
1484         if (p_ll2_conn->conn.conn_type == QED_LL2_TYPE_FCOE) {
1485                 qed_llh_add_protocol_filter(p_hwfn, p_ptt,
1486                                             0x8906, 0,
1487                                             QED_LLH_FILTER_ETHERTYPE);
1488                 qed_llh_add_protocol_filter(p_hwfn, p_ptt,
1489                                             0x8914, 0,
1490                                             QED_LLH_FILTER_ETHERTYPE);
1491         }
1492
1493 out:
1494         qed_ptt_release(p_hwfn, p_ptt);
1495         return rc;
1496 }
1497
1498 static void qed_ll2_post_rx_buffer_notify_fw(struct qed_hwfn *p_hwfn,
1499                                              struct qed_ll2_rx_queue *p_rx,
1500                                              struct qed_ll2_rx_packet *p_curp)
1501 {
1502         struct qed_ll2_rx_packet *p_posting_packet = NULL;
1503         struct core_ll2_rx_prod rx_prod = { 0, 0, 0 };
1504         bool b_notify_fw = false;
1505         u16 bd_prod, cq_prod;
1506
1507         /* This handles the flushing of already posted buffers */
1508         while (!list_empty(&p_rx->posting_descq)) {
1509                 p_posting_packet = list_first_entry(&p_rx->posting_descq,
1510                                                     struct qed_ll2_rx_packet,
1511                                                     list_entry);
1512                 list_move_tail(&p_posting_packet->list_entry,
1513                                &p_rx->active_descq);
1514                 b_notify_fw = true;
1515         }
1516
1517         /* This handles the supplied packet [if there is one] */
1518         if (p_curp) {
1519                 list_add_tail(&p_curp->list_entry, &p_rx->active_descq);
1520                 b_notify_fw = true;
1521         }
1522
1523         if (!b_notify_fw)
1524                 return;
1525
1526         bd_prod = qed_chain_get_prod_idx(&p_rx->rxq_chain);
1527         cq_prod = qed_chain_get_prod_idx(&p_rx->rcq_chain);
1528         rx_prod.bd_prod = cpu_to_le16(bd_prod);
1529         rx_prod.cqe_prod = cpu_to_le16(cq_prod);
1530         DIRECT_REG_WR(p_rx->set_prod_addr, *((u32 *)&rx_prod));
1531 }
1532
1533 int qed_ll2_post_rx_buffer(struct qed_hwfn *p_hwfn,
1534                            u8 connection_handle,
1535                            dma_addr_t addr,
1536                            u16 buf_len, void *cookie, u8 notify_fw)
1537 {
1538         struct core_rx_bd_with_buff_len *p_curb = NULL;
1539         struct qed_ll2_rx_packet *p_curp = NULL;
1540         struct qed_ll2_info *p_ll2_conn;
1541         struct qed_ll2_rx_queue *p_rx;
1542         unsigned long flags;
1543         void *p_data;
1544         int rc = 0;
1545
1546         p_ll2_conn = qed_ll2_handle_sanity(p_hwfn, connection_handle);
1547         if (!p_ll2_conn)
1548                 return -EINVAL;
1549         p_rx = &p_ll2_conn->rx_queue;
1550
1551         spin_lock_irqsave(&p_rx->lock, flags);
1552         if (!list_empty(&p_rx->free_descq))
1553                 p_curp = list_first_entry(&p_rx->free_descq,
1554                                           struct qed_ll2_rx_packet, list_entry);
1555         if (p_curp) {
1556                 if (qed_chain_get_elem_left(&p_rx->rxq_chain) &&
1557                     qed_chain_get_elem_left(&p_rx->rcq_chain)) {
1558                         p_data = qed_chain_produce(&p_rx->rxq_chain);
1559                         p_curb = (struct core_rx_bd_with_buff_len *)p_data;
1560                         qed_chain_produce(&p_rx->rcq_chain);
1561                 }
1562         }
1563
1564         /* If we're lacking entires, let's try to flush buffers to FW */
1565         if (!p_curp || !p_curb) {
1566                 rc = -EBUSY;
1567                 p_curp = NULL;
1568                 goto out_notify;
1569         }
1570
1571         /* We have an Rx packet we can fill */
1572         DMA_REGPAIR_LE(p_curb->addr, addr);
1573         p_curb->buff_length = cpu_to_le16(buf_len);
1574         p_curp->rx_buf_addr = addr;
1575         p_curp->cookie = cookie;
1576         p_curp->rxq_bd = p_curb;
1577         p_curp->buf_length = buf_len;
1578         list_del(&p_curp->list_entry);
1579
1580         /* Check if we only want to enqueue this packet without informing FW */
1581         if (!notify_fw) {
1582                 list_add_tail(&p_curp->list_entry, &p_rx->posting_descq);
1583                 goto out;
1584         }
1585
1586 out_notify:
1587         qed_ll2_post_rx_buffer_notify_fw(p_hwfn, p_rx, p_curp);
1588 out:
1589         spin_unlock_irqrestore(&p_rx->lock, flags);
1590         return rc;
1591 }
1592
1593 static void qed_ll2_prepare_tx_packet_set(struct qed_hwfn *p_hwfn,
1594                                           struct qed_ll2_tx_queue *p_tx,
1595                                           struct qed_ll2_tx_packet *p_curp,
1596                                           u8 num_of_bds,
1597                                           dma_addr_t first_frag,
1598                                           u16 first_frag_len, void *p_cookie,
1599                                           u8 notify_fw)
1600 {
1601         list_del(&p_curp->list_entry);
1602         p_curp->cookie = p_cookie;
1603         p_curp->bd_used = num_of_bds;
1604         p_curp->notify_fw = notify_fw;
1605         p_tx->cur_send_packet = p_curp;
1606         p_tx->cur_send_frag_num = 0;
1607
1608         p_curp->bds_set[p_tx->cur_send_frag_num].tx_frag = first_frag;
1609         p_curp->bds_set[p_tx->cur_send_frag_num].frag_len = first_frag_len;
1610         p_tx->cur_send_frag_num++;
1611 }
1612
1613 static void
1614 qed_ll2_prepare_tx_packet_set_bd(struct qed_hwfn *p_hwfn,
1615                                  struct qed_ll2_info *p_ll2,
1616                                  struct qed_ll2_tx_packet *p_curp,
1617                                  u8 num_of_bds,
1618                                  enum core_tx_dest tx_dest,
1619                                  u16 vlan,
1620                                  u8 bd_flags,
1621                                  u16 l4_hdr_offset_w,
1622                                  enum core_roce_flavor_type roce_flavor,
1623                                  dma_addr_t first_frag,
1624                                  u16 first_frag_len)
1625 {
1626         struct qed_chain *p_tx_chain = &p_ll2->tx_queue.txq_chain;
1627         u16 prod_idx = qed_chain_get_prod_idx(p_tx_chain);
1628         struct core_tx_bd *start_bd = NULL;
1629         u16 bd_data = 0, frag_idx;
1630
1631         start_bd = (struct core_tx_bd *)qed_chain_produce(p_tx_chain);
1632         start_bd->nw_vlan_or_lb_echo = cpu_to_le16(vlan);
1633         SET_FIELD(start_bd->bitfield1, CORE_TX_BD_L4_HDR_OFFSET_W,
1634                   cpu_to_le16(l4_hdr_offset_w));
1635         SET_FIELD(start_bd->bitfield1, CORE_TX_BD_TX_DST, tx_dest);
1636         bd_data |= bd_flags;
1637         SET_FIELD(bd_data, CORE_TX_BD_DATA_START_BD, 0x1);
1638         SET_FIELD(bd_data, CORE_TX_BD_DATA_NBDS, num_of_bds);
1639         SET_FIELD(bd_data, CORE_TX_BD_DATA_ROCE_FLAV, roce_flavor);
1640         start_bd->bd_data.as_bitfield = cpu_to_le16(bd_data);
1641         DMA_REGPAIR_LE(start_bd->addr, first_frag);
1642         start_bd->nbytes = cpu_to_le16(first_frag_len);
1643
1644         DP_VERBOSE(p_hwfn,
1645                    (NETIF_MSG_TX_QUEUED | QED_MSG_LL2),
1646                    "LL2 [q 0x%02x cid 0x%08x type 0x%08x] Tx Producer at [0x%04x] - set with a %04x bytes %02x BDs buffer at %08x:%08x\n",
1647                    p_ll2->queue_id,
1648                    p_ll2->cid,
1649                    p_ll2->conn.conn_type,
1650                    prod_idx,
1651                    first_frag_len,
1652                    num_of_bds,
1653                    le32_to_cpu(start_bd->addr.hi),
1654                    le32_to_cpu(start_bd->addr.lo));
1655
1656         if (p_ll2->tx_queue.cur_send_frag_num == num_of_bds)
1657                 return;
1658
1659         /* Need to provide the packet with additional BDs for frags */
1660         for (frag_idx = p_ll2->tx_queue.cur_send_frag_num;
1661              frag_idx < num_of_bds; frag_idx++) {
1662                 struct core_tx_bd **p_bd = &p_curp->bds_set[frag_idx].txq_bd;
1663
1664                 *p_bd = (struct core_tx_bd *)qed_chain_produce(p_tx_chain);
1665                 (*p_bd)->bd_data.as_bitfield = 0;
1666                 (*p_bd)->bitfield1 = 0;
1667                 p_curp->bds_set[frag_idx].tx_frag = 0;
1668                 p_curp->bds_set[frag_idx].frag_len = 0;
1669         }
1670 }
1671
1672 /* This should be called while the Txq spinlock is being held */
1673 static void qed_ll2_tx_packet_notify(struct qed_hwfn *p_hwfn,
1674                                      struct qed_ll2_info *p_ll2_conn)
1675 {
1676         bool b_notify = p_ll2_conn->tx_queue.cur_send_packet->notify_fw;
1677         struct qed_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue;
1678         struct qed_ll2_tx_packet *p_pkt = NULL;
1679         struct core_db_data db_msg = { 0, 0, 0 };
1680         u16 bd_prod;
1681
1682         /* If there are missing BDs, don't do anything now */
1683         if (p_ll2_conn->tx_queue.cur_send_frag_num !=
1684             p_ll2_conn->tx_queue.cur_send_packet->bd_used)
1685                 return;
1686
1687         /* Push the current packet to the list and clean after it */
1688         list_add_tail(&p_ll2_conn->tx_queue.cur_send_packet->list_entry,
1689                       &p_ll2_conn->tx_queue.sending_descq);
1690         p_ll2_conn->tx_queue.cur_send_packet = NULL;
1691         p_ll2_conn->tx_queue.cur_send_frag_num = 0;
1692
1693         /* Notify FW of packet only if requested to */
1694         if (!b_notify)
1695                 return;
1696
1697         bd_prod = qed_chain_get_prod_idx(&p_ll2_conn->tx_queue.txq_chain);
1698
1699         while (!list_empty(&p_tx->sending_descq)) {
1700                 p_pkt = list_first_entry(&p_tx->sending_descq,
1701                                          struct qed_ll2_tx_packet, list_entry);
1702                 if (!p_pkt)
1703                         break;
1704
1705                 list_move_tail(&p_pkt->list_entry, &p_tx->active_descq);
1706         }
1707
1708         SET_FIELD(db_msg.params, CORE_DB_DATA_DEST, DB_DEST_XCM);
1709         SET_FIELD(db_msg.params, CORE_DB_DATA_AGG_CMD, DB_AGG_CMD_SET);
1710         SET_FIELD(db_msg.params, CORE_DB_DATA_AGG_VAL_SEL,
1711                   DQ_XCM_CORE_TX_BD_PROD_CMD);
1712         db_msg.agg_flags = DQ_XCM_CORE_DQ_CF_CMD;
1713         db_msg.spq_prod = cpu_to_le16(bd_prod);
1714
1715         /* Make sure the BDs data is updated before ringing the doorbell */
1716         wmb();
1717
1718         DIRECT_REG_WR(p_tx->doorbell_addr, *((u32 *)&db_msg));
1719
1720         DP_VERBOSE(p_hwfn,
1721                    (NETIF_MSG_TX_QUEUED | QED_MSG_LL2),
1722                    "LL2 [q 0x%02x cid 0x%08x type 0x%08x] Doorbelled [producer 0x%04x]\n",
1723                    p_ll2_conn->queue_id,
1724                    p_ll2_conn->cid, p_ll2_conn->conn.conn_type, db_msg.spq_prod);
1725 }
1726
1727 int qed_ll2_prepare_tx_packet(struct qed_hwfn *p_hwfn,
1728                               u8 connection_handle,
1729                               u8 num_of_bds,
1730                               u16 vlan,
1731                               u8 bd_flags,
1732                               u16 l4_hdr_offset_w,
1733                               enum qed_ll2_tx_dest e_tx_dest,
1734                               enum qed_ll2_roce_flavor_type qed_roce_flavor,
1735                               dma_addr_t first_frag,
1736                               u16 first_frag_len, void *cookie, u8 notify_fw)
1737 {
1738         struct qed_ll2_tx_packet *p_curp = NULL;
1739         struct qed_ll2_info *p_ll2_conn = NULL;
1740         enum core_roce_flavor_type roce_flavor;
1741         struct qed_ll2_tx_queue *p_tx;
1742         struct qed_chain *p_tx_chain;
1743         enum core_tx_dest tx_dest;
1744         unsigned long flags;
1745         int rc = 0;
1746
1747         p_ll2_conn = qed_ll2_handle_sanity(p_hwfn, connection_handle);
1748         if (!p_ll2_conn)
1749                 return -EINVAL;
1750         p_tx = &p_ll2_conn->tx_queue;
1751         p_tx_chain = &p_tx->txq_chain;
1752
1753         if (num_of_bds > CORE_LL2_TX_MAX_BDS_PER_PACKET)
1754                 return -EIO;
1755
1756         spin_lock_irqsave(&p_tx->lock, flags);
1757         if (p_tx->cur_send_packet) {
1758                 rc = -EEXIST;
1759                 goto out;
1760         }
1761
1762         /* Get entry, but only if we have tx elements for it */
1763         if (!list_empty(&p_tx->free_descq))
1764                 p_curp = list_first_entry(&p_tx->free_descq,
1765                                           struct qed_ll2_tx_packet, list_entry);
1766         if (p_curp && qed_chain_get_elem_left(p_tx_chain) < num_of_bds)
1767                 p_curp = NULL;
1768
1769         if (!p_curp) {
1770                 rc = -EBUSY;
1771                 goto out;
1772         }
1773
1774         tx_dest = e_tx_dest == QED_LL2_TX_DEST_NW ? CORE_TX_DEST_NW :
1775                                                     CORE_TX_DEST_LB;
1776         if (qed_roce_flavor == QED_LL2_ROCE) {
1777                 roce_flavor = CORE_ROCE;
1778         } else if (qed_roce_flavor == QED_LL2_RROCE) {
1779                 roce_flavor = CORE_RROCE;
1780         } else {
1781                 rc = -EINVAL;
1782                 goto out;
1783         }
1784
1785         /* Prepare packet and BD, and perhaps send a doorbell to FW */
1786         qed_ll2_prepare_tx_packet_set(p_hwfn, p_tx, p_curp,
1787                                       num_of_bds, first_frag,
1788                                       first_frag_len, cookie, notify_fw);
1789         qed_ll2_prepare_tx_packet_set_bd(p_hwfn, p_ll2_conn, p_curp,
1790                                          num_of_bds, tx_dest,
1791                                          vlan, bd_flags, l4_hdr_offset_w,
1792                                          roce_flavor,
1793                                          first_frag, first_frag_len);
1794
1795         qed_ll2_tx_packet_notify(p_hwfn, p_ll2_conn);
1796
1797 out:
1798         spin_unlock_irqrestore(&p_tx->lock, flags);
1799         return rc;
1800 }
1801
1802 int qed_ll2_set_fragment_of_tx_packet(struct qed_hwfn *p_hwfn,
1803                                       u8 connection_handle,
1804                                       dma_addr_t addr, u16 nbytes)
1805 {
1806         struct qed_ll2_tx_packet *p_cur_send_packet = NULL;
1807         struct qed_ll2_info *p_ll2_conn = NULL;
1808         u16 cur_send_frag_num = 0;
1809         struct core_tx_bd *p_bd;
1810         unsigned long flags;
1811
1812         p_ll2_conn = qed_ll2_handle_sanity(p_hwfn, connection_handle);
1813         if (!p_ll2_conn)
1814                 return -EINVAL;
1815
1816         if (!p_ll2_conn->tx_queue.cur_send_packet)
1817                 return -EINVAL;
1818
1819         p_cur_send_packet = p_ll2_conn->tx_queue.cur_send_packet;
1820         cur_send_frag_num = p_ll2_conn->tx_queue.cur_send_frag_num;
1821
1822         if (cur_send_frag_num >= p_cur_send_packet->bd_used)
1823                 return -EINVAL;
1824
1825         /* Fill the BD information, and possibly notify FW */
1826         p_bd = p_cur_send_packet->bds_set[cur_send_frag_num].txq_bd;
1827         DMA_REGPAIR_LE(p_bd->addr, addr);
1828         p_bd->nbytes = cpu_to_le16(nbytes);
1829         p_cur_send_packet->bds_set[cur_send_frag_num].tx_frag = addr;
1830         p_cur_send_packet->bds_set[cur_send_frag_num].frag_len = nbytes;
1831
1832         p_ll2_conn->tx_queue.cur_send_frag_num++;
1833
1834         spin_lock_irqsave(&p_ll2_conn->tx_queue.lock, flags);
1835         qed_ll2_tx_packet_notify(p_hwfn, p_ll2_conn);
1836         spin_unlock_irqrestore(&p_ll2_conn->tx_queue.lock, flags);
1837
1838         return 0;
1839 }
1840
1841 int qed_ll2_terminate_connection(struct qed_hwfn *p_hwfn, u8 connection_handle)
1842 {
1843         struct qed_ll2_info *p_ll2_conn = NULL;
1844         int rc = -EINVAL;
1845         struct qed_ptt *p_ptt;
1846
1847         p_ptt = qed_ptt_acquire(p_hwfn);
1848         if (!p_ptt)
1849                 return -EAGAIN;
1850
1851         p_ll2_conn = qed_ll2_handle_sanity_lock(p_hwfn, connection_handle);
1852         if (!p_ll2_conn) {
1853                 rc = -EINVAL;
1854                 goto out;
1855         }
1856
1857         /* Stop Tx & Rx of connection, if needed */
1858         if (QED_LL2_TX_REGISTERED(p_ll2_conn)) {
1859                 rc = qed_sp_ll2_tx_queue_stop(p_hwfn, p_ll2_conn);
1860                 if (rc)
1861                         goto out;
1862                 qed_ll2_txq_flush(p_hwfn, connection_handle);
1863         }
1864
1865         if (QED_LL2_RX_REGISTERED(p_ll2_conn)) {
1866                 rc = qed_sp_ll2_rx_queue_stop(p_hwfn, p_ll2_conn);
1867                 if (rc)
1868                         goto out;
1869                 qed_ll2_rxq_flush(p_hwfn, connection_handle);
1870         }
1871
1872         if (p_ll2_conn->conn.conn_type == QED_LL2_TYPE_ISCSI_OOO)
1873                 qed_ooo_release_all_isles(p_hwfn, p_hwfn->p_ooo_info);
1874
1875         if (p_ll2_conn->conn.conn_type == QED_LL2_TYPE_FCOE) {
1876                 qed_llh_remove_protocol_filter(p_hwfn, p_ptt,
1877                                                0x8906, 0,
1878                                                QED_LLH_FILTER_ETHERTYPE);
1879                 qed_llh_remove_protocol_filter(p_hwfn, p_ptt,
1880                                                0x8914, 0,
1881                                                QED_LLH_FILTER_ETHERTYPE);
1882         }
1883
1884 out:
1885         qed_ptt_release(p_hwfn, p_ptt);
1886         return rc;
1887 }
1888
1889 void qed_ll2_release_connection(struct qed_hwfn *p_hwfn, u8 connection_handle)
1890 {
1891         struct qed_ll2_info *p_ll2_conn = NULL;
1892
1893         p_ll2_conn = qed_ll2_handle_sanity(p_hwfn, connection_handle);
1894         if (!p_ll2_conn)
1895                 return;
1896
1897         if (QED_LL2_RX_REGISTERED(p_ll2_conn)) {
1898                 p_ll2_conn->rx_queue.b_cb_registred = false;
1899                 qed_int_unregister_cb(p_hwfn, p_ll2_conn->rx_queue.rx_sb_index);
1900         }
1901
1902         if (QED_LL2_TX_REGISTERED(p_ll2_conn)) {
1903                 p_ll2_conn->tx_queue.b_cb_registred = false;
1904                 qed_int_unregister_cb(p_hwfn, p_ll2_conn->tx_queue.tx_sb_index);
1905         }
1906
1907         kfree(p_ll2_conn->tx_queue.descq_array);
1908         qed_chain_free(p_hwfn->cdev, &p_ll2_conn->tx_queue.txq_chain);
1909
1910         kfree(p_ll2_conn->rx_queue.descq_array);
1911         qed_chain_free(p_hwfn->cdev, &p_ll2_conn->rx_queue.rxq_chain);
1912         qed_chain_free(p_hwfn->cdev, &p_ll2_conn->rx_queue.rcq_chain);
1913
1914         qed_cxt_release_cid(p_hwfn, p_ll2_conn->cid);
1915
1916         qed_ll2_release_connection_ooo(p_hwfn, p_ll2_conn);
1917
1918         mutex_lock(&p_ll2_conn->mutex);
1919         p_ll2_conn->b_active = false;
1920         mutex_unlock(&p_ll2_conn->mutex);
1921 }
1922
1923 struct qed_ll2_info *qed_ll2_alloc(struct qed_hwfn *p_hwfn)
1924 {
1925         struct qed_ll2_info *p_ll2_connections;
1926         u8 i;
1927
1928         /* Allocate LL2's set struct */
1929         p_ll2_connections = kcalloc(QED_MAX_NUM_OF_LL2_CONNECTIONS,
1930                                     sizeof(struct qed_ll2_info), GFP_KERNEL);
1931         if (!p_ll2_connections) {
1932                 DP_NOTICE(p_hwfn, "Failed to allocate `struct qed_ll2'\n");
1933                 return NULL;
1934         }
1935
1936         for (i = 0; i < QED_MAX_NUM_OF_LL2_CONNECTIONS; i++)
1937                 p_ll2_connections[i].my_id = i;
1938
1939         return p_ll2_connections;
1940 }
1941
1942 void qed_ll2_setup(struct qed_hwfn *p_hwfn,
1943                    struct qed_ll2_info *p_ll2_connections)
1944 {
1945         int i;
1946
1947         for (i = 0; i < QED_MAX_NUM_OF_LL2_CONNECTIONS; i++)
1948                 mutex_init(&p_ll2_connections[i].mutex);
1949 }
1950
1951 void qed_ll2_free(struct qed_hwfn *p_hwfn,
1952                   struct qed_ll2_info *p_ll2_connections)
1953 {
1954         kfree(p_ll2_connections);
1955 }
1956
1957 static void _qed_ll2_get_tstats(struct qed_hwfn *p_hwfn,
1958                                 struct qed_ptt *p_ptt,
1959                                 struct qed_ll2_info *p_ll2_conn,
1960                                 struct qed_ll2_stats *p_stats)
1961 {
1962         struct core_ll2_tstorm_per_queue_stat tstats;
1963         u8 qid = p_ll2_conn->queue_id;
1964         u32 tstats_addr;
1965
1966         memset(&tstats, 0, sizeof(tstats));
1967         tstats_addr = BAR0_MAP_REG_TSDM_RAM +
1968                       CORE_LL2_TSTORM_PER_QUEUE_STAT_OFFSET(qid);
1969         qed_memcpy_from(p_hwfn, p_ptt, &tstats, tstats_addr, sizeof(tstats));
1970
1971         p_stats->packet_too_big_discard =
1972                         HILO_64_REGPAIR(tstats.packet_too_big_discard);
1973         p_stats->no_buff_discard = HILO_64_REGPAIR(tstats.no_buff_discard);
1974 }
1975
1976 static void _qed_ll2_get_ustats(struct qed_hwfn *p_hwfn,
1977                                 struct qed_ptt *p_ptt,
1978                                 struct qed_ll2_info *p_ll2_conn,
1979                                 struct qed_ll2_stats *p_stats)
1980 {
1981         struct core_ll2_ustorm_per_queue_stat ustats;
1982         u8 qid = p_ll2_conn->queue_id;
1983         u32 ustats_addr;
1984
1985         memset(&ustats, 0, sizeof(ustats));
1986         ustats_addr = BAR0_MAP_REG_USDM_RAM +
1987                       CORE_LL2_USTORM_PER_QUEUE_STAT_OFFSET(qid);
1988         qed_memcpy_from(p_hwfn, p_ptt, &ustats, ustats_addr, sizeof(ustats));
1989
1990         p_stats->rcv_ucast_bytes = HILO_64_REGPAIR(ustats.rcv_ucast_bytes);
1991         p_stats->rcv_mcast_bytes = HILO_64_REGPAIR(ustats.rcv_mcast_bytes);
1992         p_stats->rcv_bcast_bytes = HILO_64_REGPAIR(ustats.rcv_bcast_bytes);
1993         p_stats->rcv_ucast_pkts = HILO_64_REGPAIR(ustats.rcv_ucast_pkts);
1994         p_stats->rcv_mcast_pkts = HILO_64_REGPAIR(ustats.rcv_mcast_pkts);
1995         p_stats->rcv_bcast_pkts = HILO_64_REGPAIR(ustats.rcv_bcast_pkts);
1996 }
1997
1998 static void _qed_ll2_get_pstats(struct qed_hwfn *p_hwfn,
1999                                 struct qed_ptt *p_ptt,
2000                                 struct qed_ll2_info *p_ll2_conn,
2001                                 struct qed_ll2_stats *p_stats)
2002 {
2003         struct core_ll2_pstorm_per_queue_stat pstats;
2004         u8 stats_id = p_ll2_conn->tx_stats_id;
2005         u32 pstats_addr;
2006
2007         memset(&pstats, 0, sizeof(pstats));
2008         pstats_addr = BAR0_MAP_REG_PSDM_RAM +
2009                       CORE_LL2_PSTORM_PER_QUEUE_STAT_OFFSET(stats_id);
2010         qed_memcpy_from(p_hwfn, p_ptt, &pstats, pstats_addr, sizeof(pstats));
2011
2012         p_stats->sent_ucast_bytes = HILO_64_REGPAIR(pstats.sent_ucast_bytes);
2013         p_stats->sent_mcast_bytes = HILO_64_REGPAIR(pstats.sent_mcast_bytes);
2014         p_stats->sent_bcast_bytes = HILO_64_REGPAIR(pstats.sent_bcast_bytes);
2015         p_stats->sent_ucast_pkts = HILO_64_REGPAIR(pstats.sent_ucast_pkts);
2016         p_stats->sent_mcast_pkts = HILO_64_REGPAIR(pstats.sent_mcast_pkts);
2017         p_stats->sent_bcast_pkts = HILO_64_REGPAIR(pstats.sent_bcast_pkts);
2018 }
2019
2020 int qed_ll2_get_stats(struct qed_hwfn *p_hwfn,
2021                       u8 connection_handle, struct qed_ll2_stats *p_stats)
2022 {
2023         struct qed_ll2_info *p_ll2_conn = NULL;
2024         struct qed_ptt *p_ptt;
2025
2026         memset(p_stats, 0, sizeof(*p_stats));
2027
2028         if ((connection_handle >= QED_MAX_NUM_OF_LL2_CONNECTIONS) ||
2029             !p_hwfn->p_ll2_info)
2030                 return -EINVAL;
2031
2032         p_ll2_conn = &p_hwfn->p_ll2_info[connection_handle];
2033
2034         p_ptt = qed_ptt_acquire(p_hwfn);
2035         if (!p_ptt) {
2036                 DP_ERR(p_hwfn, "Failed to acquire ptt\n");
2037                 return -EINVAL;
2038         }
2039
2040         _qed_ll2_get_tstats(p_hwfn, p_ptt, p_ll2_conn, p_stats);
2041         _qed_ll2_get_ustats(p_hwfn, p_ptt, p_ll2_conn, p_stats);
2042         if (p_ll2_conn->tx_stats_en)
2043                 _qed_ll2_get_pstats(p_hwfn, p_ptt, p_ll2_conn, p_stats);
2044
2045         qed_ptt_release(p_hwfn, p_ptt);
2046         return 0;
2047 }
2048
2049 static void qed_ll2_register_cb_ops(struct qed_dev *cdev,
2050                                     const struct qed_ll2_cb_ops *ops,
2051                                     void *cookie)
2052 {
2053         cdev->ll2->cbs = ops;
2054         cdev->ll2->cb_cookie = cookie;
2055 }
2056
2057 static int qed_ll2_start(struct qed_dev *cdev, struct qed_ll2_params *params)
2058 {
2059         struct qed_ll2_conn ll2_info;
2060         struct qed_ll2_buffer *buffer, *tmp_buffer;
2061         enum qed_ll2_conn_type conn_type;
2062         struct qed_ptt *p_ptt;
2063         int rc, i;
2064         u8 gsi_enable = 1;
2065
2066         /* Initialize LL2 locks & lists */
2067         INIT_LIST_HEAD(&cdev->ll2->list);
2068         spin_lock_init(&cdev->ll2->lock);
2069         cdev->ll2->rx_size = NET_SKB_PAD + ETH_HLEN +
2070                              L1_CACHE_BYTES + params->mtu;
2071         cdev->ll2->frags_mapped = params->frags_mapped;
2072
2073         /*Allocate memory for LL2 */
2074         DP_INFO(cdev, "Allocating LL2 buffers of size %08x bytes\n",
2075                 cdev->ll2->rx_size);
2076         for (i = 0; i < QED_LL2_RX_SIZE; i++) {
2077                 buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
2078                 if (!buffer) {
2079                         DP_INFO(cdev, "Failed to allocate LL2 buffers\n");
2080                         goto fail;
2081                 }
2082
2083                 rc = qed_ll2_alloc_buffer(cdev, (u8 **)&buffer->data,
2084                                           &buffer->phys_addr);
2085                 if (rc) {
2086                         kfree(buffer);
2087                         goto fail;
2088                 }
2089
2090                 list_add_tail(&buffer->list, &cdev->ll2->list);
2091         }
2092
2093         switch (QED_LEADING_HWFN(cdev)->hw_info.personality) {
2094         case QED_PCI_FCOE:
2095                 conn_type = QED_LL2_TYPE_FCOE;
2096                 gsi_enable = 0;
2097                 break;
2098         case QED_PCI_ISCSI:
2099                 conn_type = QED_LL2_TYPE_ISCSI;
2100                 gsi_enable = 0;
2101                 break;
2102         case QED_PCI_ETH_ROCE:
2103                 conn_type = QED_LL2_TYPE_ROCE;
2104                 break;
2105         default:
2106                 conn_type = QED_LL2_TYPE_TEST;
2107         }
2108
2109         /* Prepare the temporary ll2 information */
2110         memset(&ll2_info, 0, sizeof(ll2_info));
2111
2112         ll2_info.conn_type = conn_type;
2113         ll2_info.mtu = params->mtu;
2114         ll2_info.rx_drop_ttl0_flg = params->drop_ttl0_packets;
2115         ll2_info.rx_vlan_removal_en = params->rx_vlan_stripping;
2116         ll2_info.tx_tc = 0;
2117         ll2_info.tx_dest = CORE_TX_DEST_NW;
2118         ll2_info.gsi_enable = gsi_enable;
2119
2120         rc = qed_ll2_acquire_connection(QED_LEADING_HWFN(cdev), &ll2_info,
2121                                         QED_LL2_RX_SIZE, QED_LL2_TX_SIZE,
2122                                         &cdev->ll2->handle);
2123         if (rc) {
2124                 DP_INFO(cdev, "Failed to acquire LL2 connection\n");
2125                 goto fail;
2126         }
2127
2128         rc = qed_ll2_establish_connection(QED_LEADING_HWFN(cdev),
2129                                           cdev->ll2->handle);
2130         if (rc) {
2131                 DP_INFO(cdev, "Failed to establish LL2 connection\n");
2132                 goto release_fail;
2133         }
2134
2135         /* Post all Rx buffers to FW */
2136         spin_lock_bh(&cdev->ll2->lock);
2137         list_for_each_entry_safe(buffer, tmp_buffer, &cdev->ll2->list, list) {
2138                 rc = qed_ll2_post_rx_buffer(QED_LEADING_HWFN(cdev),
2139                                             cdev->ll2->handle,
2140                                             buffer->phys_addr, 0, buffer, 1);
2141                 if (rc) {
2142                         DP_INFO(cdev,
2143                                 "Failed to post an Rx buffer; Deleting it\n");
2144                         dma_unmap_single(&cdev->pdev->dev, buffer->phys_addr,
2145                                          cdev->ll2->rx_size, DMA_FROM_DEVICE);
2146                         kfree(buffer->data);
2147                         list_del(&buffer->list);
2148                         kfree(buffer);
2149                 } else {
2150                         cdev->ll2->rx_cnt++;
2151                 }
2152         }
2153         spin_unlock_bh(&cdev->ll2->lock);
2154
2155         if (!cdev->ll2->rx_cnt) {
2156                 DP_INFO(cdev, "Failed passing even a single Rx buffer\n");
2157                 goto release_terminate;
2158         }
2159
2160         if (!is_valid_ether_addr(params->ll2_mac_address)) {
2161                 DP_INFO(cdev, "Invalid Ethernet address\n");
2162                 goto release_terminate;
2163         }
2164
2165         if (cdev->hwfns[0].hw_info.personality == QED_PCI_ISCSI &&
2166             cdev->hwfns[0].pf_params.iscsi_pf_params.ooo_enable) {
2167                 DP_VERBOSE(cdev, QED_MSG_STORAGE, "Starting OOO LL2 queue\n");
2168                 rc = qed_ll2_start_ooo(cdev, params);
2169                 if (rc) {
2170                         DP_INFO(cdev,
2171                                 "Failed to initialize the OOO LL2 queue\n");
2172                         goto release_terminate;
2173                 }
2174         }
2175
2176         p_ptt = qed_ptt_acquire(QED_LEADING_HWFN(cdev));
2177         if (!p_ptt) {
2178                 DP_INFO(cdev, "Failed to acquire PTT\n");
2179                 goto release_terminate;
2180         }
2181
2182         rc = qed_llh_add_mac_filter(QED_LEADING_HWFN(cdev), p_ptt,
2183                                     params->ll2_mac_address);
2184         qed_ptt_release(QED_LEADING_HWFN(cdev), p_ptt);
2185         if (rc) {
2186                 DP_ERR(cdev, "Failed to allocate LLH filter\n");
2187                 goto release_terminate_all;
2188         }
2189
2190         ether_addr_copy(cdev->ll2_mac_address, params->ll2_mac_address);
2191         return 0;
2192
2193 release_terminate_all:
2194
2195 release_terminate:
2196         qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev), cdev->ll2->handle);
2197 release_fail:
2198         qed_ll2_release_connection(QED_LEADING_HWFN(cdev), cdev->ll2->handle);
2199 fail:
2200         qed_ll2_kill_buffers(cdev);
2201         cdev->ll2->handle = QED_LL2_UNUSED_HANDLE;
2202         return -EINVAL;
2203 }
2204
2205 static int qed_ll2_stop(struct qed_dev *cdev)
2206 {
2207         struct qed_ptt *p_ptt;
2208         int rc;
2209
2210         if (cdev->ll2->handle == QED_LL2_UNUSED_HANDLE)
2211                 return 0;
2212
2213         p_ptt = qed_ptt_acquire(QED_LEADING_HWFN(cdev));
2214         if (!p_ptt) {
2215                 DP_INFO(cdev, "Failed to acquire PTT\n");
2216                 goto fail;
2217         }
2218
2219         qed_llh_remove_mac_filter(QED_LEADING_HWFN(cdev), p_ptt,
2220                                   cdev->ll2_mac_address);
2221         qed_ptt_release(QED_LEADING_HWFN(cdev), p_ptt);
2222         eth_zero_addr(cdev->ll2_mac_address);
2223
2224         if (cdev->hwfns[0].hw_info.personality == QED_PCI_ISCSI &&
2225             cdev->hwfns[0].pf_params.iscsi_pf_params.ooo_enable)
2226                 qed_ll2_stop_ooo(cdev);
2227
2228         rc = qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev),
2229                                           cdev->ll2->handle);
2230         if (rc)
2231                 DP_INFO(cdev, "Failed to terminate LL2 connection\n");
2232
2233         qed_ll2_kill_buffers(cdev);
2234
2235         qed_ll2_release_connection(QED_LEADING_HWFN(cdev), cdev->ll2->handle);
2236         cdev->ll2->handle = QED_LL2_UNUSED_HANDLE;
2237
2238         return rc;
2239 fail:
2240         return -EINVAL;
2241 }
2242
2243 static int qed_ll2_start_xmit(struct qed_dev *cdev, struct sk_buff *skb)
2244 {
2245         const skb_frag_t *frag;
2246         int rc = -EINVAL, i;
2247         dma_addr_t mapping;
2248         u16 vlan = 0;
2249         u8 flags = 0;
2250
2251         if (unlikely(skb->ip_summed != CHECKSUM_NONE)) {
2252                 DP_INFO(cdev, "Cannot transmit a checksumed packet\n");
2253                 return -EINVAL;
2254         }
2255
2256         if (1 + skb_shinfo(skb)->nr_frags > CORE_LL2_TX_MAX_BDS_PER_PACKET) {
2257                 DP_ERR(cdev, "Cannot transmit a packet with %d fragments\n",
2258                        1 + skb_shinfo(skb)->nr_frags);
2259                 return -EINVAL;
2260         }
2261
2262         mapping = dma_map_single(&cdev->pdev->dev, skb->data,
2263                                  skb->len, DMA_TO_DEVICE);
2264         if (unlikely(dma_mapping_error(&cdev->pdev->dev, mapping))) {
2265                 DP_NOTICE(cdev, "SKB mapping failed\n");
2266                 return -EINVAL;
2267         }
2268
2269         /* Request HW to calculate IP csum */
2270         if (!((vlan_get_protocol(skb) == htons(ETH_P_IPV6)) &&
2271               ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2272                 flags |= BIT(CORE_TX_BD_DATA_IP_CSUM_SHIFT);
2273
2274         if (skb_vlan_tag_present(skb)) {
2275                 vlan = skb_vlan_tag_get(skb);
2276                 flags |= BIT(CORE_TX_BD_DATA_VLAN_INSERTION_SHIFT);
2277         }
2278
2279         rc = qed_ll2_prepare_tx_packet(QED_LEADING_HWFN(cdev),
2280                                        cdev->ll2->handle,
2281                                        1 + skb_shinfo(skb)->nr_frags,
2282                                        vlan, flags, 0, QED_LL2_TX_DEST_NW,
2283                                        0 /* RoCE FLAVOR */,
2284                                        mapping, skb->len, skb, 1);
2285         if (rc)
2286                 goto err;
2287
2288         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2289                 frag = &skb_shinfo(skb)->frags[i];
2290                 if (!cdev->ll2->frags_mapped) {
2291                         mapping = skb_frag_dma_map(&cdev->pdev->dev, frag, 0,
2292                                                    skb_frag_size(frag),
2293                                                    DMA_TO_DEVICE);
2294
2295                         if (unlikely(dma_mapping_error(&cdev->pdev->dev,
2296                                                        mapping))) {
2297                                 DP_NOTICE(cdev,
2298                                           "Unable to map frag - dropping packet\n");
2299                                 rc = -ENOMEM;
2300                                 goto err;
2301                         }
2302                 } else {
2303                         mapping = page_to_phys(skb_frag_page(frag)) |
2304                             frag->page_offset;
2305                 }
2306
2307                 rc = qed_ll2_set_fragment_of_tx_packet(QED_LEADING_HWFN(cdev),
2308                                                        cdev->ll2->handle,
2309                                                        mapping,
2310                                                        skb_frag_size(frag));
2311
2312                 /* if failed not much to do here, partial packet has been posted
2313                  * we can't free memory, will need to wait for completion.
2314                  */
2315                 if (rc)
2316                         goto err2;
2317         }
2318
2319         return 0;
2320
2321 err:
2322         dma_unmap_single(&cdev->pdev->dev, mapping, skb->len, DMA_TO_DEVICE);
2323
2324 err2:
2325         return rc;
2326 }
2327
2328 static int qed_ll2_stats(struct qed_dev *cdev, struct qed_ll2_stats *stats)
2329 {
2330         if (!cdev->ll2)
2331                 return -EINVAL;
2332
2333         return qed_ll2_get_stats(QED_LEADING_HWFN(cdev),
2334                                  cdev->ll2->handle, stats);
2335 }
2336
2337 const struct qed_ll2_ops qed_ll2_ops_pass = {
2338         .start = &qed_ll2_start,
2339         .stop = &qed_ll2_stop,
2340         .start_xmit = &qed_ll2_start_xmit,
2341         .register_cb_ops = &qed_ll2_register_cb_ops,
2342         .get_stats = &qed_ll2_stats,
2343 };
2344
2345 int qed_ll2_alloc_if(struct qed_dev *cdev)
2346 {
2347         cdev->ll2 = kzalloc(sizeof(*cdev->ll2), GFP_KERNEL);
2348         return cdev->ll2 ? 0 : -ENOMEM;
2349 }
2350
2351 void qed_ll2_dealloc_if(struct qed_dev *cdev)
2352 {
2353         kfree(cdev->ll2);
2354         cdev->ll2 = NULL;
2355 }