drivers/target/iscsi/cxgbit/cxgbit_main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2016 Chelsio Communications, Inc.
   4  */
   5
   6 #define DRV_NAME "cxgbit"
   7 #define DRV_VERSION "1.0.0-ko"
   8 #define pr_fmt(fmt) DRV_NAME ": " fmt
   9
  10 #include "cxgbit.h"
  11
  12 #ifdef CONFIG_CHELSIO_T4_DCB
  13 #include <net/dcbevent.h>
  14 #include "cxgb4_dcb.h"
  15 #endif
  16
  17 LIST_HEAD(cdev_list_head);
  18 /* cdev list lock */
  19 DEFINE_MUTEX(cdev_list_lock);
  20
  21 void _cxgbit_free_cdev(struct kref *kref)
  22 {
  23         struct cxgbit_device *cdev;
  24
  25         cdev = container_of(kref, struct cxgbit_device, kref);
  26
  27         cxgbi_ppm_release(cdev2ppm(cdev));
  28         kfree(cdev);
  29 }
  30
  31 static void cxgbit_set_mdsl(struct cxgbit_device *cdev)
  32 {
  33         struct cxgb4_lld_info *lldi = &cdev->lldi;
  34         u32 mdsl;
  35
  36 #define CXGBIT_T5_MAX_PDU_LEN 16224
  37 #define CXGBIT_PDU_NONPAYLOAD_LEN 312 /* 48(BHS) + 256(AHS) + 8(Digest) */
  38         if (is_t5(lldi->adapter_type)) {
  39                 mdsl = min_t(u32, lldi->iscsi_iolen - CXGBIT_PDU_NONPAYLOAD_LEN,
  40                              CXGBIT_T5_MAX_PDU_LEN - CXGBIT_PDU_NONPAYLOAD_LEN);
  41         } else {
  42                 mdsl = lldi->iscsi_iolen - CXGBIT_PDU_NONPAYLOAD_LEN;
  43                 mdsl = min(mdsl, 16384U);
  44         }
  45
  46         mdsl = round_down(mdsl, 4);
  47         mdsl = min_t(u32, mdsl, 4 * PAGE_SIZE);
  48         mdsl = min_t(u32, mdsl, (MAX_SKB_FRAGS - 1) * PAGE_SIZE);
  49
  50         cdev->mdsl = mdsl;
  51 }
  52
  53 static void *cxgbit_uld_add(const struct cxgb4_lld_info *lldi)
  54 {
  55         struct cxgbit_device *cdev;
  56
  57         if (is_t4(lldi->adapter_type))
  58                 return ERR_PTR(-ENODEV);
  59
  60         cdev = kzalloc(sizeof(*cdev), GFP_KERNEL);
  61         if (!cdev)
  62                 return ERR_PTR(-ENOMEM);
  63
  64         kref_init(&cdev->kref);
  65         spin_lock_init(&cdev->np_lock);
  66
  67         cdev->lldi = *lldi;
  68
  69         cxgbit_set_mdsl(cdev);
  70
  71         if (cxgbit_ddp_init(cdev) < 0) {
  72                 kfree(cdev);
  73                 return ERR_PTR(-EINVAL);
  74         }
  75
  76         if (!test_bit(CDEV_DDP_ENABLE, &cdev->flags))
  77                 pr_info("cdev %s ddp init failed\n",
  78                         pci_name(lldi->pdev));
  79
  80         if (lldi->fw_vers >= 0x10d2b00)
  81                 set_bit(CDEV_ISO_ENABLE, &cdev->flags);
  82
  83         spin_lock_init(&cdev->cskq.lock);
  84         INIT_LIST_HEAD(&cdev->cskq.list);
  85
  86         mutex_lock(&cdev_list_lock);
  87         list_add_tail(&cdev->list, &cdev_list_head);
  88         mutex_unlock(&cdev_list_lock);
  89
  90         pr_info("cdev %s added for iSCSI target transport\n",
  91                 pci_name(lldi->pdev));
  92
  93         return cdev;
  94 }
  95
  96 static void cxgbit_close_conn(struct cxgbit_device *cdev)
  97 {
  98         struct cxgbit_sock *csk;
  99         struct sk_buff *skb;
 100         bool wakeup_thread = false;
 101
 102         spin_lock_bh(&cdev->cskq.lock);
 103         list_for_each_entry(csk, &cdev->cskq.list, list) {
 104                 skb = alloc_skb(0, GFP_ATOMIC);
 105                 if (!skb)
 106                         continue;
 107
 108                 spin_lock_bh(&csk->rxq.lock);
 109                 __skb_queue_tail(&csk->rxq, skb);
 110                 if (skb_queue_len(&csk->rxq) == 1)
 111                         wakeup_thread = true;
 112                 spin_unlock_bh(&csk->rxq.lock);
 113
 114                 if (wakeup_thread) {
 115                         wake_up(&csk->waitq);
 116                         wakeup_thread = false;
 117                 }
 118         }
 119         spin_unlock_bh(&cdev->cskq.lock);
 120 }
 121
 122 static void cxgbit_detach_cdev(struct cxgbit_device *cdev)
 123 {
 124         bool free_cdev = false;
 125
 126         spin_lock_bh(&cdev->cskq.lock);
 127         if (list_empty(&cdev->cskq.list))
 128                 free_cdev = true;
 129         spin_unlock_bh(&cdev->cskq.lock);
 130
 131         if (free_cdev) {
 132                 mutex_lock(&cdev_list_lock);
 133                 list_del(&cdev->list);
 134                 mutex_unlock(&cdev_list_lock);
 135
 136                 cxgbit_put_cdev(cdev);
 137         } else {
 138                 cxgbit_close_conn(cdev);
 139         }
 140 }
 141
 142 static int cxgbit_uld_state_change(void *handle, enum cxgb4_state state)
 143 {
 144         struct cxgbit_device *cdev = handle;
 145
 146         switch (state) {
 147         case CXGB4_STATE_UP:
 148                 set_bit(CDEV_STATE_UP, &cdev->flags);
 149                 pr_info("cdev %s state UP.\n", pci_name(cdev->lldi.pdev));
 150                 break;
 151         case CXGB4_STATE_START_RECOVERY:
 152                 clear_bit(CDEV_STATE_UP, &cdev->flags);
 153                 cxgbit_close_conn(cdev);
 154                 pr_info("cdev %s state RECOVERY.\n", pci_name(cdev->lldi.pdev));
 155                 break;
 156         case CXGB4_STATE_DOWN:
 157                 pr_info("cdev %s state DOWN.\n", pci_name(cdev->lldi.pdev));
 158                 break;
 159         case CXGB4_STATE_DETACH:
 160                 clear_bit(CDEV_STATE_UP, &cdev->flags);
 161                 pr_info("cdev %s state DETACH.\n", pci_name(cdev->lldi.pdev));
 162                 cxgbit_detach_cdev(cdev);
 163                 break;
 164         default:
 165                 pr_info("cdev %s unknown state %d.\n",
 166                         pci_name(cdev->lldi.pdev), state);
 167                 break;
 168         }
 169         return 0;
 170 }
 171
 172 static void
 173 cxgbit_process_ddpvld(struct cxgbit_sock *csk, struct cxgbit_lro_pdu_cb *pdu_cb,
 174                       u32 ddpvld)
 175 {
 176
 177         if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_HCRC_SHIFT)) {
 178                 pr_info("tid 0x%x, status 0x%x, hcrc bad.\n", csk->tid, ddpvld);
 179                 pdu_cb->flags |= PDUCBF_RX_HCRC_ERR;
 180         }
 181
 182         if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_DCRC_SHIFT)) {
 183                 pr_info("tid 0x%x, status 0x%x, dcrc bad.\n", csk->tid, ddpvld);
 184                 pdu_cb->flags |= PDUCBF_RX_DCRC_ERR;
 185         }
 186
 187         if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_PAD_SHIFT))
 188                 pr_info("tid 0x%x, status 0x%x, pad bad.\n", csk->tid, ddpvld);
 189
 190         if ((ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_DDP_SHIFT)) &&
 191             (!(pdu_cb->flags & PDUCBF_RX_DATA))) {
 192                 pdu_cb->flags |= PDUCBF_RX_DATA_DDPD;
 193         }
 194 }
 195
 196 static void
 197 cxgbit_lro_add_packet_rsp(struct sk_buff *skb, u8 op, const __be64 *rsp)
 198 {
 199         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
 200         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb,
 201                                                 lro_cb->pdu_idx);
 202         struct cpl_rx_iscsi_ddp *cpl = (struct cpl_rx_iscsi_ddp *)(rsp + 1);
 203
 204         cxgbit_process_ddpvld(lro_cb->csk, pdu_cb, be32_to_cpu(cpl->ddpvld));
 205
 206         pdu_cb->flags |= PDUCBF_RX_STATUS;
 207         pdu_cb->ddigest = ntohl(cpl->ulp_crc);
 208         pdu_cb->pdulen = ntohs(cpl->len);
 209
 210         if (pdu_cb->flags & PDUCBF_RX_HDR)
 211                 pdu_cb->complete = true;
 212
 213         lro_cb->pdu_totallen += pdu_cb->pdulen;
 214         lro_cb->complete = true;
 215         lro_cb->pdu_idx++;
 216 }
 217
 218 static void
 219 cxgbit_copy_frags(struct sk_buff *skb, const struct pkt_gl *gl,
 220                   unsigned int offset)
 221 {
 222         u8 skb_frag_idx = skb_shinfo(skb)->nr_frags;
 223         u8 i;
 224
 225         /* usually there's just one frag */
 226         __skb_fill_page_desc(skb, skb_frag_idx, gl->frags[0].page,
 227                              gl->frags[0].offset + offset,
 228                              gl->frags[0].size - offset);
 229         for (i = 1; i < gl->nfrags; i++)
 230                 __skb_fill_page_desc(skb, skb_frag_idx + i,
 231                                      gl->frags[i].page,
 232                                      gl->frags[i].offset,
 233                                      gl->frags[i].size);
 234
 235         skb_shinfo(skb)->nr_frags += gl->nfrags;
 236
 237         /* get a reference to the last page, we don't own it */
 238         get_page(gl->frags[gl->nfrags - 1].page);
 239 }
 240
 241 static void
 242 cxgbit_lro_add_packet_gl(struct sk_buff *skb, u8 op, const struct pkt_gl *gl)
 243 {
 244         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
 245         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb,
 246                                                 lro_cb->pdu_idx);
 247         u32 len, offset;
 248
 249         if (op == CPL_ISCSI_HDR) {
 250                 struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)gl->va;
 251
 252                 offset = sizeof(struct cpl_iscsi_hdr);
 253                 pdu_cb->flags |= PDUCBF_RX_HDR;
 254                 pdu_cb->seq = ntohl(cpl->seq);
 255                 len = ntohs(cpl->len);
 256                 pdu_cb->hdr = gl->va + offset;
 257                 pdu_cb->hlen = len;
 258                 pdu_cb->hfrag_idx = skb_shinfo(skb)->nr_frags;
 259
 260                 if (unlikely(gl->nfrags > 1))
 261                         cxgbit_skcb_flags(skb) = 0;
 262
 263                 lro_cb->complete = false;
 264         } else if (op == CPL_ISCSI_DATA) {
 265                 struct cpl_iscsi_data *cpl = (struct cpl_iscsi_data *)gl->va;
 266
 267                 offset = sizeof(struct cpl_iscsi_data);
 268                 pdu_cb->flags |= PDUCBF_RX_DATA;
 269                 len = ntohs(cpl->len);
 270                 pdu_cb->dlen = len;
 271                 pdu_cb->doffset = lro_cb->offset;
 272                 pdu_cb->nr_dfrags = gl->nfrags;
 273                 pdu_cb->dfrag_idx = skb_shinfo(skb)->nr_frags;
 274                 lro_cb->complete = false;
 275         } else {
 276                 struct cpl_rx_iscsi_cmp *cpl;
 277
 278                 cpl = (struct cpl_rx_iscsi_cmp *)gl->va;
 279                 offset = sizeof(struct cpl_rx_iscsi_cmp);
 280                 pdu_cb->flags |= (PDUCBF_RX_HDR | PDUCBF_RX_STATUS);
 281                 len = be16_to_cpu(cpl->len);
 282                 pdu_cb->hdr = gl->va + offset;
 283                 pdu_cb->hlen = len;
 284                 pdu_cb->hfrag_idx = skb_shinfo(skb)->nr_frags;
 285                 pdu_cb->ddigest = be32_to_cpu(cpl->ulp_crc);
 286                 pdu_cb->pdulen = ntohs(cpl->len);
 287
 288                 if (unlikely(gl->nfrags > 1))
 289                         cxgbit_skcb_flags(skb) = 0;
 290
 291                 cxgbit_process_ddpvld(lro_cb->csk, pdu_cb,
 292                                       be32_to_cpu(cpl->ddpvld));
 293
 294                 if (pdu_cb->flags & PDUCBF_RX_DATA_DDPD) {
 295                         pdu_cb->flags |= PDUCBF_RX_DDP_CMP;
 296                         pdu_cb->complete = true;
 297                 } else if (pdu_cb->flags & PDUCBF_RX_DATA) {
 298                         pdu_cb->complete = true;
 299                 }
 300
 301                 lro_cb->pdu_totallen += pdu_cb->hlen + pdu_cb->dlen;
 302                 lro_cb->complete = true;
 303                 lro_cb->pdu_idx++;
 304         }
 305
 306         cxgbit_copy_frags(skb, gl, offset);
 307
 308         pdu_cb->frags += gl->nfrags;
 309         lro_cb->offset += len;
 310         skb->len += len;
 311         skb->data_len += len;
 312         skb->truesize += len;
 313 }
 314
 315 static struct sk_buff *
 316 cxgbit_lro_init_skb(struct cxgbit_sock *csk, u8 op, const struct pkt_gl *gl,
 317                     const __be64 *rsp, struct napi_struct *napi)
 318 {
 319         struct sk_buff *skb;
 320         struct cxgbit_lro_cb *lro_cb;
 321
 322         skb = napi_alloc_skb(napi, LRO_SKB_MAX_HEADROOM);
 323
 324         if (unlikely(!skb))
 325                 return NULL;
 326
 327         memset(skb->data, 0, LRO_SKB_MAX_HEADROOM);
 328
 329         cxgbit_skcb_flags(skb) |= SKCBF_RX_LRO;
 330
 331         lro_cb = cxgbit_skb_lro_cb(skb);
 332
 333         cxgbit_get_csk(csk);
 334
 335         lro_cb->csk = csk;
 336
 337         return skb;
 338 }
 339
 340 static void cxgbit_queue_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
 341 {
 342         bool wakeup_thread = false;
 343
 344         spin_lock(&csk->rxq.lock);
 345         __skb_queue_tail(&csk->rxq, skb);
 346         if (skb_queue_len(&csk->rxq) == 1)
 347                 wakeup_thread = true;
 348         spin_unlock(&csk->rxq.lock);
 349
 350         if (wakeup_thread)
 351                 wake_up(&csk->waitq);
 352 }
 353
 354 static void cxgbit_lro_flush(struct t4_lro_mgr *lro_mgr, struct sk_buff *skb)
 355 {
 356         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
 357         struct cxgbit_sock *csk = lro_cb->csk;
 358
 359         csk->lro_skb = NULL;
 360
 361         __skb_unlink(skb, &lro_mgr->lroq);
 362         cxgbit_queue_lro_skb(csk, skb);
 363
 364         cxgbit_put_csk(csk);
 365
 366         lro_mgr->lro_pkts++;
 367         lro_mgr->lro_session_cnt--;
 368 }
 369
 370 static void cxgbit_uld_lro_flush(struct t4_lro_mgr *lro_mgr)
 371 {
 372         struct sk_buff *skb;
 373
 374         while ((skb = skb_peek(&lro_mgr->lroq)))
 375                 cxgbit_lro_flush(lro_mgr, skb);
 376 }
 377
 378 static int
 379 cxgbit_lro_receive(struct cxgbit_sock *csk, u8 op, const __be64 *rsp,
 380                    const struct pkt_gl *gl, struct t4_lro_mgr *lro_mgr,
 381                    struct napi_struct *napi)
 382 {
 383         struct sk_buff *skb;
 384         struct cxgbit_lro_cb *lro_cb;
 385
 386         if (!csk) {
 387                 pr_err("%s: csk NULL, op 0x%x.\n", __func__, op);
 388                 goto out;
 389         }
 390
 391         if (csk->lro_skb)
 392                 goto add_packet;
 393
 394 start_lro:
 395         if (lro_mgr->lro_session_cnt >= MAX_LRO_SESSIONS) {
 396                 cxgbit_uld_lro_flush(lro_mgr);
 397                 goto start_lro;
 398         }
 399
 400         skb = cxgbit_lro_init_skb(csk, op, gl, rsp, napi);
 401         if (unlikely(!skb))
 402                 goto out;
 403
 404         csk->lro_skb = skb;
 405
 406         __skb_queue_tail(&lro_mgr->lroq, skb);
 407         lro_mgr->lro_session_cnt++;
 408
 409 add_packet:
 410         skb = csk->lro_skb;
 411         lro_cb = cxgbit_skb_lro_cb(skb);
 412
 413         if ((gl && (((skb_shinfo(skb)->nr_frags + gl->nfrags) >
 414             MAX_SKB_FRAGS) || (lro_cb->pdu_totallen >= LRO_FLUSH_LEN_MAX))) ||
 415             (lro_cb->pdu_idx >= MAX_SKB_FRAGS)) {
 416                 cxgbit_lro_flush(lro_mgr, skb);
 417                 goto start_lro;
 418         }
 419
 420         if (gl)
 421                 cxgbit_lro_add_packet_gl(skb, op, gl);
 422         else
 423                 cxgbit_lro_add_packet_rsp(skb, op, rsp);
 424
 425         lro_mgr->lro_merged++;
 426
 427         return 0;
 428
 429 out:
 430         return -1;
 431 }
 432
 433 static int
 434 cxgbit_uld_lro_rx_handler(void *hndl, const __be64 *rsp,
 435                           const struct pkt_gl *gl, struct t4_lro_mgr *lro_mgr,
 436                           struct napi_struct *napi)
 437 {
 438         struct cxgbit_device *cdev = hndl;
 439         struct cxgb4_lld_info *lldi = &cdev->lldi;
 440         struct cpl_tx_data *rpl = NULL;
 441         struct cxgbit_sock *csk = NULL;
 442         unsigned int tid = 0;
 443         struct sk_buff *skb;
 444         unsigned int op = *(u8 *)rsp;
 445         bool lro_flush = true;
 446
 447         switch (op) {
 448         case CPL_ISCSI_HDR:
 449         case CPL_ISCSI_DATA:
 450         case CPL_RX_ISCSI_CMP:
 451         case CPL_RX_ISCSI_DDP:
 452         case CPL_FW4_ACK:
 453                 lro_flush = false;
 454                 fallthrough;
 455         case CPL_ABORT_RPL_RSS:
 456         case CPL_PASS_ESTABLISH:
 457         case CPL_PEER_CLOSE:
 458         case CPL_CLOSE_CON_RPL:
 459         case CPL_ABORT_REQ_RSS:
 460         case CPL_SET_TCB_RPL:
 461         case CPL_RX_DATA:
 462                 rpl = gl ? (struct cpl_tx_data *)gl->va :
 463                            (struct cpl_tx_data *)(rsp + 1);
 464                 tid = GET_TID(rpl);
 465                 csk = lookup_tid(lldi->tids, tid);
 466                 break;
 467         default:
 468                 break;
 469         }
 470
 471         if (csk && csk->lro_skb && lro_flush)
 472                 cxgbit_lro_flush(lro_mgr, csk->lro_skb);
 473
 474         if (!gl) {
 475                 unsigned int len;
 476
 477                 if (op == CPL_RX_ISCSI_DDP) {
 478                         if (!cxgbit_lro_receive(csk, op, rsp, NULL, lro_mgr,
 479                                                 napi))
 480                                 return 0;
 481                 }
 482
 483                 len = 64 - sizeof(struct rsp_ctrl) - 8;
 484                 skb = napi_alloc_skb(napi, len);
 485                 if (!skb)
 486                         goto nomem;
 487                 __skb_put(skb, len);
 488                 skb_copy_to_linear_data(skb, &rsp[1], len);
 489         } else {
 490                 if (unlikely(op != *(u8 *)gl->va)) {
 491                         pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n",
 492                                 gl->va, be64_to_cpu(*rsp),
 493                                 get_unaligned_be64(gl->va),
 494                                 gl->tot_len);
 495                         return 0;
 496                 }
 497
 498                 if ((op == CPL_ISCSI_HDR) || (op == CPL_ISCSI_DATA) ||
 499                     (op == CPL_RX_ISCSI_CMP)) {
 500                         if (!cxgbit_lro_receive(csk, op, rsp, gl, lro_mgr,
 501                                                 napi))
 502                                 return 0;
 503                 }
 504
 505 #define RX_PULL_LEN 128
 506                 skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN);
 507                 if (unlikely(!skb))
 508                         goto nomem;
 509         }
 510
 511         rpl = (struct cpl_tx_data *)skb->data;
 512         op = rpl->ot.opcode;
 513         cxgbit_skcb_rx_opcode(skb) = op;
 514
 515         pr_debug("cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n",
 516                  cdev, op, rpl->ot.opcode_tid,
 517                  ntohl(rpl->ot.opcode_tid), skb);
 518
 519         if (op < NUM_CPL_CMDS && cxgbit_cplhandlers[op]) {
 520                 cxgbit_cplhandlers[op](cdev, skb);
 521         } else {
 522                 pr_err("No handler for opcode 0x%x.\n", op);
 523                 __kfree_skb(skb);
 524         }
 525         return 0;
 526 nomem:
 527         pr_err("%s OOM bailing out.\n", __func__);
 528         return 1;
 529 }
 530
 531 #ifdef CONFIG_CHELSIO_T4_DCB
 532 struct cxgbit_dcb_work {
 533         struct dcb_app_type dcb_app;
 534         struct work_struct work;
 535 };
 536
 537 static void
 538 cxgbit_update_dcb_priority(struct cxgbit_device *cdev, u8 port_id,
 539                            u8 dcb_priority, u16 port_num)
 540 {
 541         struct cxgbit_sock *csk;
 542         struct sk_buff *skb;
 543         u16 local_port;
 544         bool wakeup_thread = false;
 545
 546         spin_lock_bh(&cdev->cskq.lock);
 547         list_for_each_entry(csk, &cdev->cskq.list, list) {
 548                 if (csk->port_id != port_id)
 549                         continue;
 550
 551                 if (csk->com.local_addr.ss_family == AF_INET6) {
 552                         struct sockaddr_in6 *sock_in6;
 553
 554                         sock_in6 = (struct sockaddr_in6 *)&csk->com.local_addr;
 555                         local_port = ntohs(sock_in6->sin6_port);
 556                 } else {
 557                         struct sockaddr_in *sock_in;
 558
 559                         sock_in = (struct sockaddr_in *)&csk->com.local_addr;
 560                         local_port = ntohs(sock_in->sin_port);
 561                 }
 562
 563                 if (local_port != port_num)
 564                         continue;
 565
 566                 if (csk->dcb_priority == dcb_priority)
 567                         continue;
 568
 569                 skb = alloc_skb(0, GFP_ATOMIC);
 570                 if (!skb)
 571                         continue;
 572
 573                 spin_lock(&csk->rxq.lock);
 574                 __skb_queue_tail(&csk->rxq, skb);
 575                 if (skb_queue_len(&csk->rxq) == 1)
 576                         wakeup_thread = true;
 577                 spin_unlock(&csk->rxq.lock);
 578
 579                 if (wakeup_thread) {
 580                         wake_up(&csk->waitq);
 581                         wakeup_thread = false;
 582                 }
 583         }
 584         spin_unlock_bh(&cdev->cskq.lock);
 585 }
 586
 587 static void cxgbit_dcb_workfn(struct work_struct *work)
 588 {
 589         struct cxgbit_dcb_work *dcb_work;
 590         struct net_device *ndev;
 591         struct cxgbit_device *cdev = NULL;
 592         struct dcb_app_type *iscsi_app;
 593         u8 priority, port_id = 0xff;
 594
 595         dcb_work = container_of(work, struct cxgbit_dcb_work, work);
 596         iscsi_app = &dcb_work->dcb_app;
 597
 598         if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_IEEE) {
 599                 if ((iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_STREAM) &&
 600                     (iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_ANY))
 601                         goto out;
 602
 603                 priority = iscsi_app->app.priority;
 604
 605         } else if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_CEE) {
 606                 if (iscsi_app->app.selector != DCB_APP_IDTYPE_PORTNUM)
 607                         goto out;
 608
 609                 if (!iscsi_app->app.priority)
 610                         goto out;
 611
 612                 priority = ffs(iscsi_app->app.priority) - 1;
 613         } else {
 614                 goto out;
 615         }
 616
 617         pr_debug("priority for ifid %d is %u\n",
 618                  iscsi_app->ifindex, priority);
 619
 620         ndev = dev_get_by_index(&init_net, iscsi_app->ifindex);
 621
 622         if (!ndev)
 623                 goto out;
 624
 625         mutex_lock(&cdev_list_lock);
 626         cdev = cxgbit_find_device(ndev, &port_id);
 627
 628         dev_put(ndev);
 629
 630         if (!cdev) {
 631                 mutex_unlock(&cdev_list_lock);
 632                 goto out;
 633         }
 634
 635         cxgbit_update_dcb_priority(cdev, port_id, priority,
 636                                    iscsi_app->app.protocol);
 637         mutex_unlock(&cdev_list_lock);
 638 out:
 639         kfree(dcb_work);
 640 }
 641
 642 static int
 643 cxgbit_dcbevent_notify(struct notifier_block *nb, unsigned long action,
 644                        void *data)
 645 {
 646         struct cxgbit_dcb_work *dcb_work;
 647         struct dcb_app_type *dcb_app = data;
 648
 649         dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC);
 650         if (!dcb_work)
 651                 return NOTIFY_DONE;
 652
 653         dcb_work->dcb_app = *dcb_app;
 654         INIT_WORK(&dcb_work->work, cxgbit_dcb_workfn);
 655         schedule_work(&dcb_work->work);
 656         return NOTIFY_OK;
 657 }
 658 #endif
 659
 660 static enum target_prot_op cxgbit_get_sup_prot_ops(struct iscsi_conn *conn)
 661 {
 662         return TARGET_PROT_NORMAL;
 663 }
 664
 665 static struct iscsit_transport cxgbit_transport = {
 666         .name                   = DRV_NAME,
 667         .transport_type         = ISCSI_CXGBIT,
 668         .rdma_shutdown          = false,
 669         .priv_size              = sizeof(struct cxgbit_cmd),
 670         .owner                  = THIS_MODULE,
 671         .iscsit_setup_np        = cxgbit_setup_np,
 672         .iscsit_accept_np       = cxgbit_accept_np,
 673         .iscsit_free_np         = cxgbit_free_np,
 674         .iscsit_free_conn       = cxgbit_free_conn,
 675         .iscsit_get_login_rx    = cxgbit_get_login_rx,
 676         .iscsit_put_login_tx    = cxgbit_put_login_tx,
 677         .iscsit_immediate_queue = iscsit_immediate_queue,
 678         .iscsit_response_queue  = iscsit_response_queue,
 679         .iscsit_get_dataout     = iscsit_build_r2ts_for_cmd,
 680         .iscsit_queue_data_in   = iscsit_queue_rsp,
 681         .iscsit_queue_status    = iscsit_queue_rsp,
 682         .iscsit_xmit_pdu        = cxgbit_xmit_pdu,
 683         .iscsit_get_r2t_ttt     = cxgbit_get_r2t_ttt,
 684         .iscsit_get_rx_pdu      = cxgbit_get_rx_pdu,
 685         .iscsit_validate_params = cxgbit_validate_params,
 686         .iscsit_unmap_cmd       = cxgbit_unmap_cmd,
 687         .iscsit_aborted_task    = iscsit_aborted_task,
 688         .iscsit_get_sup_prot_ops = cxgbit_get_sup_prot_ops,
 689 };
 690
 691 static struct cxgb4_uld_info cxgbit_uld_info = {
 692         .name           = DRV_NAME,
 693         .nrxq           = MAX_ULD_QSETS,
 694         .ntxq           = MAX_ULD_QSETS,
 695         .rxq_size       = 1024,
 696         .lro            = true,
 697         .add            = cxgbit_uld_add,
 698         .state_change   = cxgbit_uld_state_change,
 699         .lro_rx_handler = cxgbit_uld_lro_rx_handler,
 700         .lro_flush      = cxgbit_uld_lro_flush,
 701 };
 702
 703 #ifdef CONFIG_CHELSIO_T4_DCB
 704 static struct notifier_block cxgbit_dcbevent_nb = {
 705         .notifier_call = cxgbit_dcbevent_notify,
 706 };
 707 #endif
 708
 709 static int __init cxgbit_init(void)
 710 {
 711         cxgb4_register_uld(CXGB4_ULD_ISCSIT, &cxgbit_uld_info);
 712         iscsit_register_transport(&cxgbit_transport);
 713
 714 #ifdef CONFIG_CHELSIO_T4_DCB
 715         pr_info("%s dcb enabled.\n", DRV_NAME);
 716         register_dcbevent_notifier(&cxgbit_dcbevent_nb);
 717 #endif
 718         BUILD_BUG_ON(sizeof_field(struct sk_buff, cb) <
 719                      sizeof(union cxgbit_skb_cb));
 720         return 0;
 721 }
 722
 723 static void __exit cxgbit_exit(void)
 724 {
 725         struct cxgbit_device *cdev, *tmp;
 726
 727 #ifdef CONFIG_CHELSIO_T4_DCB
 728         unregister_dcbevent_notifier(&cxgbit_dcbevent_nb);
 729 #endif
 730         mutex_lock(&cdev_list_lock);
 731         list_for_each_entry_safe(cdev, tmp, &cdev_list_head, list) {
 732                 list_del(&cdev->list);
 733                 cxgbit_put_cdev(cdev);
 734         }
 735         mutex_unlock(&cdev_list_lock);
 736         iscsit_unregister_transport(&cxgbit_transport);
 737         cxgb4_unregister_uld(CXGB4_ULD_ISCSIT);
 738 }
 739
 740 module_init(cxgbit_init);
 741 module_exit(cxgbit_exit);
 742
 743 MODULE_DESCRIPTION("Chelsio iSCSI target offload driver");
 744 MODULE_AUTHOR("Chelsio Communications");
 745 MODULE_VERSION(DRV_VERSION);
 746 MODULE_LICENSE("GPL");