net/smc/smc_llc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  Link Layer Control (LLC)
   6  *
   7  *  Copyright IBM Corp. 2016
   8  *
   9  *  Author(s):  Klaus Wacker <Klaus.Wacker@de.ibm.com>
  10  *              Ursula Braun <ubraun@linux.vnet.ibm.com>
  11  */
  12
  13 #include <net/tcp.h>
  14 #include <rdma/ib_verbs.h>
  15
  16 #include "smc.h"
  17 #include "smc_core.h"
  18 #include "smc_clc.h"
  19 #include "smc_llc.h"
  20 #include "smc_pnet.h"
  21
  22 #define SMC_LLC_DATA_LEN                40
  23
  24 struct smc_llc_hdr {
  25         struct smc_wr_rx_hdr common;
  26         u8 length;      /* 44 */
  27 #if defined(__BIG_ENDIAN_BITFIELD)
  28         u8 reserved:4,
  29            add_link_rej_rsn:4;
  30 #elif defined(__LITTLE_ENDIAN_BITFIELD)
  31         u8 add_link_rej_rsn:4,
  32            reserved:4;
  33 #endif
  34         u8 flags;
  35 };
  36
  37 #define SMC_LLC_FLAG_NO_RMBE_EYEC       0x03
  38
  39 struct smc_llc_msg_confirm_link {       /* type 0x01 */
  40         struct smc_llc_hdr hd;
  41         u8 sender_mac[ETH_ALEN];
  42         u8 sender_gid[SMC_GID_SIZE];
  43         u8 sender_qp_num[3];
  44         u8 link_num;
  45         u8 link_uid[SMC_LGR_ID_SIZE];
  46         u8 max_links;
  47         u8 reserved[9];
  48 };
  49
  50 #define SMC_LLC_FLAG_ADD_LNK_REJ        0x40
  51 #define SMC_LLC_REJ_RSN_NO_ALT_PATH     1
  52
  53 #define SMC_LLC_ADD_LNK_MAX_LINKS       2
  54
  55 struct smc_llc_msg_add_link {           /* type 0x02 */
  56         struct smc_llc_hdr hd;
  57         u8 sender_mac[ETH_ALEN];
  58         u8 reserved2[2];
  59         u8 sender_gid[SMC_GID_SIZE];
  60         u8 sender_qp_num[3];
  61         u8 link_num;
  62 #if defined(__BIG_ENDIAN_BITFIELD)
  63         u8 reserved3 : 4,
  64            qp_mtu   : 4;
  65 #elif defined(__LITTLE_ENDIAN_BITFIELD)
  66         u8 qp_mtu   : 4,
  67            reserved3 : 4;
  68 #endif
  69         u8 initial_psn[3];
  70         u8 reserved[8];
  71 };
  72
  73 struct smc_llc_msg_add_link_cont_rt {
  74         __be32 rmb_key;
  75         __be32 rmb_key_new;
  76         __be64 rmb_vaddr_new;
  77 };
  78
  79 #define SMC_LLC_RKEYS_PER_CONT_MSG      2
  80
  81 struct smc_llc_msg_add_link_cont {      /* type 0x03 */
  82         struct smc_llc_hdr hd;
  83         u8 link_num;
  84         u8 num_rkeys;
  85         u8 reserved2[2];
  86         struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG];
  87         u8 reserved[4];
  88 } __packed;                     /* format defined in RFC7609 */
  89
  90 #define SMC_LLC_FLAG_DEL_LINK_ALL       0x40
  91 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY   0x20
  92
  93 struct smc_llc_msg_del_link {           /* type 0x04 */
  94         struct smc_llc_hdr hd;
  95         u8 link_num;
  96         __be32 reason;
  97         u8 reserved[35];
  98 } __packed;                     /* format defined in RFC7609 */
  99
 100 struct smc_llc_msg_test_link {          /* type 0x07 */
 101         struct smc_llc_hdr hd;
 102         u8 user_data[16];
 103         u8 reserved[24];
 104 };
 105
 106 struct smc_rmb_rtoken {
 107         union {
 108                 u8 num_rkeys;   /* first rtoken byte of CONFIRM LINK msg */
 109                                 /* is actually the num of rtokens, first */
 110                                 /* rtoken is always for the current link */
 111                 u8 link_id;     /* link id of the rtoken */
 112         };
 113         __be32 rmb_key;
 114         __be64 rmb_vaddr;
 115 } __packed;                     /* format defined in RFC7609 */
 116
 117 #define SMC_LLC_RKEYS_PER_MSG   3
 118
 119 struct smc_llc_msg_confirm_rkey {       /* type 0x06 */
 120         struct smc_llc_hdr hd;
 121         struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
 122         u8 reserved;
 123 };
 124
 125 #define SMC_LLC_DEL_RKEY_MAX    8
 126 #define SMC_LLC_FLAG_RKEY_RETRY 0x10
 127 #define SMC_LLC_FLAG_RKEY_NEG   0x20
 128
 129 struct smc_llc_msg_delete_rkey {        /* type 0x09 */
 130         struct smc_llc_hdr hd;
 131         u8 num_rkeys;
 132         u8 err_mask;
 133         u8 reserved[2];
 134         __be32 rkey[8];
 135         u8 reserved2[4];
 136 };
 137
 138 union smc_llc_msg {
 139         struct smc_llc_msg_confirm_link confirm_link;
 140         struct smc_llc_msg_add_link add_link;
 141         struct smc_llc_msg_add_link_cont add_link_cont;
 142         struct smc_llc_msg_del_link delete_link;
 143
 144         struct smc_llc_msg_confirm_rkey confirm_rkey;
 145         struct smc_llc_msg_delete_rkey delete_rkey;
 146
 147         struct smc_llc_msg_test_link test_link;
 148         struct {
 149                 struct smc_llc_hdr hdr;
 150                 u8 data[SMC_LLC_DATA_LEN];
 151         } raw;
 152 };
 153
 154 #define SMC_LLC_FLAG_RESP               0x80
 155
 156 struct smc_llc_qentry {
 157         struct list_head list;
 158         struct smc_link *link;
 159         union smc_llc_msg msg;
 160 };
 161
 162 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc);
 163
 164 struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow)
 165 {
 166         struct smc_llc_qentry *qentry = flow->qentry;
 167
 168         flow->qentry = NULL;
 169         return qentry;
 170 }
 171
 172 void smc_llc_flow_qentry_del(struct smc_llc_flow *flow)
 173 {
 174         struct smc_llc_qentry *qentry;
 175
 176         if (flow->qentry) {
 177                 qentry = flow->qentry;
 178                 flow->qentry = NULL;
 179                 kfree(qentry);
 180         }
 181 }
 182
 183 static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow,
 184                                            struct smc_llc_qentry *qentry)
 185 {
 186         flow->qentry = qentry;
 187 }
 188
 189 static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type,
 190                                   struct smc_llc_qentry *qentry)
 191 {
 192         u8 msg_type = qentry->msg.raw.hdr.common.type;
 193
 194         if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) &&
 195             flow_type != msg_type && !lgr->delayed_event) {
 196                 lgr->delayed_event = qentry;
 197                 return;
 198         }
 199         /* drop parallel or already-in-progress llc requests */
 200         if (flow_type != msg_type)
 201                 pr_warn_once("smc: SMC-R lg %*phN dropped parallel "
 202                              "LLC msg: msg %d flow %d role %d\n",
 203                              SMC_LGR_ID_SIZE, &lgr->id,
 204                              qentry->msg.raw.hdr.common.type,
 205                              flow_type, lgr->role);
 206         kfree(qentry);
 207 }
 208
 209 /* try to start a new llc flow, initiated by an incoming llc msg */
 210 static bool smc_llc_flow_start(struct smc_llc_flow *flow,
 211                                struct smc_llc_qentry *qentry)
 212 {
 213         struct smc_link_group *lgr = qentry->link->lgr;
 214
 215         spin_lock_bh(&lgr->llc_flow_lock);
 216         if (flow->type) {
 217                 /* a flow is already active */
 218                 smc_llc_flow_parallel(lgr, flow->type, qentry);
 219                 spin_unlock_bh(&lgr->llc_flow_lock);
 220                 return false;
 221         }
 222         switch (qentry->msg.raw.hdr.common.type) {
 223         case SMC_LLC_ADD_LINK:
 224                 flow->type = SMC_LLC_FLOW_ADD_LINK;
 225                 break;
 226         case SMC_LLC_DELETE_LINK:
 227                 flow->type = SMC_LLC_FLOW_DEL_LINK;
 228                 break;
 229         case SMC_LLC_CONFIRM_RKEY:
 230         case SMC_LLC_DELETE_RKEY:
 231                 flow->type = SMC_LLC_FLOW_RKEY;
 232                 break;
 233         default:
 234                 flow->type = SMC_LLC_FLOW_NONE;
 235         }
 236         if (qentry == lgr->delayed_event)
 237                 lgr->delayed_event = NULL;
 238         smc_llc_flow_qentry_set(flow, qentry);
 239         spin_unlock_bh(&lgr->llc_flow_lock);
 240         return true;
 241 }
 242
 243 /* start a new local llc flow, wait till current flow finished */
 244 int smc_llc_flow_initiate(struct smc_link_group *lgr,
 245                           enum smc_llc_flowtype type)
 246 {
 247         enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE;
 248         int rc;
 249
 250         /* all flows except confirm_rkey and delete_rkey are exclusive,
 251          * confirm/delete rkey flows can run concurrently (local and remote)
 252          */
 253         if (type == SMC_LLC_FLOW_RKEY)
 254                 allowed_remote = SMC_LLC_FLOW_RKEY;
 255 again:
 256         if (list_empty(&lgr->list))
 257                 return -ENODEV;
 258         spin_lock_bh(&lgr->llc_flow_lock);
 259         if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
 260             (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
 261              lgr->llc_flow_rmt.type == allowed_remote)) {
 262                 lgr->llc_flow_lcl.type = type;
 263                 spin_unlock_bh(&lgr->llc_flow_lock);
 264                 return 0;
 265         }
 266         spin_unlock_bh(&lgr->llc_flow_lock);
 267         rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) ||
 268                                 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
 269                                  (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
 270                                   lgr->llc_flow_rmt.type == allowed_remote))),
 271                                 SMC_LLC_WAIT_TIME * 10);
 272         if (!rc)
 273                 return -ETIMEDOUT;
 274         goto again;
 275 }
 276
 277 /* finish the current llc flow */
 278 void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)
 279 {
 280         spin_lock_bh(&lgr->llc_flow_lock);
 281         memset(flow, 0, sizeof(*flow));
 282         flow->type = SMC_LLC_FLOW_NONE;
 283         spin_unlock_bh(&lgr->llc_flow_lock);
 284         if (!list_empty(&lgr->list) && lgr->delayed_event &&
 285             flow == &lgr->llc_flow_lcl)
 286                 schedule_work(&lgr->llc_event_work);
 287         else
 288                 wake_up(&lgr->llc_flow_waiter);
 289 }
 290
 291 /* lnk is optional and used for early wakeup when link goes down, useful in
 292  * cases where we wait for a response on the link after we sent a request
 293  */
 294 struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
 295                                     struct smc_link *lnk,
 296                                     int time_out, u8 exp_msg)
 297 {
 298         struct smc_llc_flow *flow = &lgr->llc_flow_lcl;
 299         u8 rcv_msg;
 300
 301         wait_event_timeout(lgr->llc_msg_waiter,
 302                            (flow->qentry ||
 303                             (lnk && !smc_link_usable(lnk)) ||
 304                             list_empty(&lgr->list)),
 305                            time_out);
 306         if (!flow->qentry ||
 307             (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) {
 308                 smc_llc_flow_qentry_del(flow);
 309                 goto out;
 310         }
 311         rcv_msg = flow->qentry->msg.raw.hdr.common.type;
 312         if (exp_msg && rcv_msg != exp_msg) {
 313                 if (exp_msg == SMC_LLC_ADD_LINK &&
 314                     rcv_msg == SMC_LLC_DELETE_LINK) {
 315                         /* flow_start will delay the unexpected msg */
 316                         smc_llc_flow_start(&lgr->llc_flow_lcl,
 317                                            smc_llc_flow_qentry_clr(flow));
 318                         return NULL;
 319                 }
 320                 pr_warn_once("smc: SMC-R lg %*phN dropped unexpected LLC msg: "
 321                              "msg %d exp %d flow %d role %d flags %x\n",
 322                              SMC_LGR_ID_SIZE, &lgr->id, rcv_msg, exp_msg,
 323                              flow->type, lgr->role,
 324                              flow->qentry->msg.raw.hdr.flags);
 325                 smc_llc_flow_qentry_del(flow);
 326         }
 327 out:
 328         return flow->qentry;
 329 }
 330
 331 /********************************** send *************************************/
 332
 333 struct smc_llc_tx_pend {
 334 };
 335
 336 /* handler for send/transmission completion of an LLC msg */
 337 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
 338                                struct smc_link *link,
 339                                enum ib_wc_status wc_status)
 340 {
 341         /* future work: handle wc_status error for recovery and failover */
 342 }
 343
 344 /**
 345  * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
 346  * @link: Pointer to SMC link used for sending LLC control message.
 347  * @wr_buf: Out variable returning pointer to work request payload buffer.
 348  * @pend: Out variable returning pointer to private pending WR tracking.
 349  *        It's the context the transmit complete handler will get.
 350  *
 351  * Reserves and pre-fills an entry for a pending work request send/tx.
 352  * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
 353  * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
 354  *
 355  * Return: 0 on success, otherwise an error value.
 356  */
 357 static int smc_llc_add_pending_send(struct smc_link *link,
 358                                     struct smc_wr_buf **wr_buf,
 359                                     struct smc_wr_tx_pend_priv **pend)
 360 {
 361         int rc;
 362
 363         rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
 364                                      pend);
 365         if (rc < 0)
 366                 return rc;
 367         BUILD_BUG_ON_MSG(
 368                 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
 369                 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
 370         BUILD_BUG_ON_MSG(
 371                 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
 372                 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
 373         BUILD_BUG_ON_MSG(
 374                 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
 375                 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
 376         return 0;
 377 }
 378
 379 /* high-level API to send LLC confirm link */
 380 int smc_llc_send_confirm_link(struct smc_link *link,
 381                               enum smc_llc_reqresp reqresp)
 382 {
 383         struct smc_llc_msg_confirm_link *confllc;
 384         struct smc_wr_tx_pend_priv *pend;
 385         struct smc_wr_buf *wr_buf;
 386         int rc;
 387
 388         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 389         if (rc)
 390                 return rc;
 391         confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
 392         memset(confllc, 0, sizeof(*confllc));
 393         confllc->hd.common.type = SMC_LLC_CONFIRM_LINK;
 394         confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link);
 395         confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
 396         if (reqresp == SMC_LLC_RESP)
 397                 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
 398         memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
 399                ETH_ALEN);
 400         memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
 401         hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
 402         confllc->link_num = link->link_id;
 403         memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE);
 404         confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS;
 405         /* send llc message */
 406         rc = smc_wr_tx_send(link, pend);
 407         return rc;
 408 }
 409
 410 /* send LLC confirm rkey request */
 411 static int smc_llc_send_confirm_rkey(struct smc_link *send_link,
 412                                      struct smc_buf_desc *rmb_desc)
 413 {
 414         struct smc_llc_msg_confirm_rkey *rkeyllc;
 415         struct smc_wr_tx_pend_priv *pend;
 416         struct smc_wr_buf *wr_buf;
 417         struct smc_link *link;
 418         int i, rc, rtok_ix;
 419
 420         rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend);
 421         if (rc)
 422                 return rc;
 423         rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
 424         memset(rkeyllc, 0, sizeof(*rkeyllc));
 425         rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY;
 426         rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey);
 427
 428         rtok_ix = 1;
 429         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 430                 link = &send_link->lgr->lnk[i];
 431                 if (smc_link_active(link) && link != send_link) {
 432                         rkeyllc->rtoken[rtok_ix].link_id = link->link_id;
 433                         rkeyllc->rtoken[rtok_ix].rmb_key =
 434                                 htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
 435                         rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64(
 436                                 (u64)sg_dma_address(
 437                                         rmb_desc->sgt[link->link_idx].sgl));
 438                         rtok_ix++;
 439                 }
 440         }
 441         /* rkey of send_link is in rtoken[0] */
 442         rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1;
 443         rkeyllc->rtoken[0].rmb_key =
 444                 htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey);
 445         rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64(
 446                 (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl));
 447         /* send llc message */
 448         rc = smc_wr_tx_send(send_link, pend);
 449         return rc;
 450 }
 451
 452 /* send LLC delete rkey request */
 453 static int smc_llc_send_delete_rkey(struct smc_link *link,
 454                                     struct smc_buf_desc *rmb_desc)
 455 {
 456         struct smc_llc_msg_delete_rkey *rkeyllc;
 457         struct smc_wr_tx_pend_priv *pend;
 458         struct smc_wr_buf *wr_buf;
 459         int rc;
 460
 461         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 462         if (rc)
 463                 return rc;
 464         rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
 465         memset(rkeyllc, 0, sizeof(*rkeyllc));
 466         rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY;
 467         rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey);
 468         rkeyllc->num_rkeys = 1;
 469         rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
 470         /* send llc message */
 471         rc = smc_wr_tx_send(link, pend);
 472         return rc;
 473 }
 474
 475 /* send ADD LINK request or response */
 476 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
 477                           struct smc_link *link_new,
 478                           enum smc_llc_reqresp reqresp)
 479 {
 480         struct smc_llc_msg_add_link *addllc;
 481         struct smc_wr_tx_pend_priv *pend;
 482         struct smc_wr_buf *wr_buf;
 483         int rc;
 484
 485         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 486         if (rc)
 487                 return rc;
 488         addllc = (struct smc_llc_msg_add_link *)wr_buf;
 489
 490         memset(addllc, 0, sizeof(*addllc));
 491         addllc->hd.common.type = SMC_LLC_ADD_LINK;
 492         addllc->hd.length = sizeof(struct smc_llc_msg_add_link);
 493         if (reqresp == SMC_LLC_RESP)
 494                 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
 495         memcpy(addllc->sender_mac, mac, ETH_ALEN);
 496         memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
 497         if (link_new) {
 498                 addllc->link_num = link_new->link_id;
 499                 hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num);
 500                 hton24(addllc->initial_psn, link_new->psn_initial);
 501                 if (reqresp == SMC_LLC_REQ)
 502                         addllc->qp_mtu = link_new->path_mtu;
 503                 else
 504                         addllc->qp_mtu = min(link_new->path_mtu,
 505                                              link_new->peer_mtu);
 506         }
 507         /* send llc message */
 508         rc = smc_wr_tx_send(link, pend);
 509         return rc;
 510 }
 511
 512 /* send DELETE LINK request or response */
 513 int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id,
 514                              enum smc_llc_reqresp reqresp, bool orderly,
 515                              u32 reason)
 516 {
 517         struct smc_llc_msg_del_link *delllc;
 518         struct smc_wr_tx_pend_priv *pend;
 519         struct smc_wr_buf *wr_buf;
 520         int rc;
 521
 522         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 523         if (rc)
 524                 return rc;
 525         delllc = (struct smc_llc_msg_del_link *)wr_buf;
 526
 527         memset(delllc, 0, sizeof(*delllc));
 528         delllc->hd.common.type = SMC_LLC_DELETE_LINK;
 529         delllc->hd.length = sizeof(struct smc_llc_msg_del_link);
 530         if (reqresp == SMC_LLC_RESP)
 531                 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
 532         if (orderly)
 533                 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
 534         if (link_del_id)
 535                 delllc->link_num = link_del_id;
 536         else
 537                 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
 538         delllc->reason = htonl(reason);
 539         /* send llc message */
 540         rc = smc_wr_tx_send(link, pend);
 541         return rc;
 542 }
 543
 544 /* send LLC test link request */
 545 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
 546 {
 547         struct smc_llc_msg_test_link *testllc;
 548         struct smc_wr_tx_pend_priv *pend;
 549         struct smc_wr_buf *wr_buf;
 550         int rc;
 551
 552         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 553         if (rc)
 554                 return rc;
 555         testllc = (struct smc_llc_msg_test_link *)wr_buf;
 556         memset(testllc, 0, sizeof(*testllc));
 557         testllc->hd.common.type = SMC_LLC_TEST_LINK;
 558         testllc->hd.length = sizeof(struct smc_llc_msg_test_link);
 559         memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
 560         /* send llc message */
 561         rc = smc_wr_tx_send(link, pend);
 562         return rc;
 563 }
 564
 565 /* schedule an llc send on link, may wait for buffers */
 566 static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
 567 {
 568         struct smc_wr_tx_pend_priv *pend;
 569         struct smc_wr_buf *wr_buf;
 570         int rc;
 571
 572         if (!smc_link_usable(link))
 573                 return -ENOLINK;
 574         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 575         if (rc)
 576                 return rc;
 577         memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
 578         return smc_wr_tx_send(link, pend);
 579 }
 580
 581 /* schedule an llc send on link, may wait for buffers,
 582  * and wait for send completion notification.
 583  * @return 0 on success
 584  */
 585 static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf)
 586 {
 587         struct smc_wr_tx_pend_priv *pend;
 588         struct smc_wr_buf *wr_buf;
 589         int rc;
 590
 591         if (!smc_link_usable(link))
 592                 return -ENOLINK;
 593         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 594         if (rc)
 595                 return rc;
 596         memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
 597         return smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME);
 598 }
 599
 600 /********************************* receive ***********************************/
 601
 602 static int smc_llc_alloc_alt_link(struct smc_link_group *lgr,
 603                                   enum smc_lgr_type lgr_new_t)
 604 {
 605         int i;
 606
 607         if (lgr->type == SMC_LGR_SYMMETRIC ||
 608             (lgr->type != SMC_LGR_SINGLE &&
 609              (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
 610               lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)))
 611                 return -EMLINK;
 612
 613         if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
 614             lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) {
 615                 for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--)
 616                         if (lgr->lnk[i].state == SMC_LNK_UNUSED)
 617                                 return i;
 618         } else {
 619                 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++)
 620                         if (lgr->lnk[i].state == SMC_LNK_UNUSED)
 621                                 return i;
 622         }
 623         return -EMLINK;
 624 }
 625
 626 /* return first buffer from any of the next buf lists */
 627 static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr,
 628                                                   int *buf_lst)
 629 {
 630         struct smc_buf_desc *buf_pos;
 631
 632         while (*buf_lst < SMC_RMBE_SIZES) {
 633                 buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst],
 634                                                    struct smc_buf_desc, list);
 635                 if (buf_pos)
 636                         return buf_pos;
 637                 (*buf_lst)++;
 638         }
 639         return NULL;
 640 }
 641
 642 /* return next rmb from buffer lists */
 643 static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr,
 644                                                  int *buf_lst,
 645                                                  struct smc_buf_desc *buf_pos)
 646 {
 647         struct smc_buf_desc *buf_next;
 648
 649         if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) {
 650                 (*buf_lst)++;
 651                 return _smc_llc_get_next_rmb(lgr, buf_lst);
 652         }
 653         buf_next = list_next_entry(buf_pos, list);
 654         return buf_next;
 655 }
 656
 657 static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr,
 658                                                   int *buf_lst)
 659 {
 660         *buf_lst = 0;
 661         return smc_llc_get_next_rmb(lgr, buf_lst, NULL);
 662 }
 663
 664 /* send one add_link_continue msg */
 665 static int smc_llc_add_link_cont(struct smc_link *link,
 666                                  struct smc_link *link_new, u8 *num_rkeys_todo,
 667                                  int *buf_lst, struct smc_buf_desc **buf_pos)
 668 {
 669         struct smc_llc_msg_add_link_cont *addc_llc;
 670         struct smc_link_group *lgr = link->lgr;
 671         int prim_lnk_idx, lnk_idx, i, rc;
 672         struct smc_wr_tx_pend_priv *pend;
 673         struct smc_wr_buf *wr_buf;
 674         struct smc_buf_desc *rmb;
 675         u8 n;
 676
 677         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 678         if (rc)
 679                 return rc;
 680         addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf;
 681         memset(addc_llc, 0, sizeof(*addc_llc));
 682
 683         prim_lnk_idx = link->link_idx;
 684         lnk_idx = link_new->link_idx;
 685         addc_llc->link_num = link_new->link_id;
 686         addc_llc->num_rkeys = *num_rkeys_todo;
 687         n = *num_rkeys_todo;
 688         for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) {
 689                 if (!*buf_pos) {
 690                         addc_llc->num_rkeys = addc_llc->num_rkeys -
 691                                               *num_rkeys_todo;
 692                         *num_rkeys_todo = 0;
 693                         break;
 694                 }
 695                 rmb = *buf_pos;
 696
 697                 addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey);
 698                 addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey);
 699                 addc_llc->rt[i].rmb_vaddr_new =
 700                         cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
 701
 702                 (*num_rkeys_todo)--;
 703                 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
 704                 while (*buf_pos && !(*buf_pos)->used)
 705                         *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
 706         }
 707         addc_llc->hd.common.type = SMC_LLC_ADD_LINK_CONT;
 708         addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont);
 709         if (lgr->role == SMC_CLNT)
 710                 addc_llc->hd.flags |= SMC_LLC_FLAG_RESP;
 711         return smc_wr_tx_send(link, pend);
 712 }
 713
 714 static int smc_llc_cli_rkey_exchange(struct smc_link *link,
 715                                      struct smc_link *link_new)
 716 {
 717         struct smc_llc_msg_add_link_cont *addc_llc;
 718         struct smc_link_group *lgr = link->lgr;
 719         u8 max, num_rkeys_send, num_rkeys_recv;
 720         struct smc_llc_qentry *qentry;
 721         struct smc_buf_desc *buf_pos;
 722         int buf_lst;
 723         int rc = 0;
 724         int i;
 725
 726         mutex_lock(&lgr->rmbs_lock);
 727         num_rkeys_send = lgr->conns_num;
 728         buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
 729         do {
 730                 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME,
 731                                       SMC_LLC_ADD_LINK_CONT);
 732                 if (!qentry) {
 733                         rc = -ETIMEDOUT;
 734                         break;
 735                 }
 736                 addc_llc = &qentry->msg.add_link_cont;
 737                 num_rkeys_recv = addc_llc->num_rkeys;
 738                 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
 739                 for (i = 0; i < max; i++) {
 740                         smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
 741                                        addc_llc->rt[i].rmb_key,
 742                                        addc_llc->rt[i].rmb_vaddr_new,
 743                                        addc_llc->rt[i].rmb_key_new);
 744                         num_rkeys_recv--;
 745                 }
 746                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 747                 rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
 748                                            &buf_lst, &buf_pos);
 749                 if (rc)
 750                         break;
 751         } while (num_rkeys_send || num_rkeys_recv);
 752
 753         mutex_unlock(&lgr->rmbs_lock);
 754         return rc;
 755 }
 756
 757 /* prepare and send an add link reject response */
 758 static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry)
 759 {
 760         qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
 761         qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
 762         qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
 763         return smc_llc_send_message(qentry->link, &qentry->msg);
 764 }
 765
 766 static int smc_llc_cli_conf_link(struct smc_link *link,
 767                                  struct smc_init_info *ini,
 768                                  struct smc_link *link_new,
 769                                  enum smc_lgr_type lgr_new_t)
 770 {
 771         struct smc_link_group *lgr = link->lgr;
 772         struct smc_llc_qentry *qentry = NULL;
 773         int rc = 0;
 774
 775         /* receive CONFIRM LINK request over RoCE fabric */
 776         qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0);
 777         if (!qentry) {
 778                 rc = smc_llc_send_delete_link(link, link_new->link_id,
 779                                               SMC_LLC_REQ, false,
 780                                               SMC_LLC_DEL_LOST_PATH);
 781                 return -ENOLINK;
 782         }
 783         if (qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) {
 784                 /* received DELETE_LINK instead */
 785                 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
 786                 smc_llc_send_message(link, &qentry->msg);
 787                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 788                 return -ENOLINK;
 789         }
 790         smc_llc_save_peer_uid(qentry);
 791         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 792
 793         rc = smc_ib_modify_qp_rts(link_new);
 794         if (rc) {
 795                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
 796                                          false, SMC_LLC_DEL_LOST_PATH);
 797                 return -ENOLINK;
 798         }
 799         smc_wr_remember_qp_attr(link_new);
 800
 801         rc = smcr_buf_reg_lgr(link_new);
 802         if (rc) {
 803                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
 804                                          false, SMC_LLC_DEL_LOST_PATH);
 805                 return -ENOLINK;
 806         }
 807
 808         /* send CONFIRM LINK response over RoCE fabric */
 809         rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP);
 810         if (rc) {
 811                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
 812                                          false, SMC_LLC_DEL_LOST_PATH);
 813                 return -ENOLINK;
 814         }
 815         smc_llc_link_active(link_new);
 816         if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
 817             lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
 818                 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
 819         else
 820                 smcr_lgr_set_type(lgr, lgr_new_t);
 821         return 0;
 822 }
 823
 824 static void smc_llc_save_add_link_info(struct smc_link *link,
 825                                        struct smc_llc_msg_add_link *add_llc)
 826 {
 827         link->peer_qpn = ntoh24(add_llc->sender_qp_num);
 828         memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE);
 829         memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN);
 830         link->peer_psn = ntoh24(add_llc->initial_psn);
 831         link->peer_mtu = add_llc->qp_mtu;
 832 }
 833
 834 /* as an SMC client, process an add link request */
 835 int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry)
 836 {
 837         struct smc_llc_msg_add_link *llc = &qentry->msg.add_link;
 838         enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
 839         struct smc_link_group *lgr = smc_get_lgr(link);
 840         struct smc_link *lnk_new = NULL;
 841         struct smc_init_info ini;
 842         int lnk_idx, rc = 0;
 843
 844         if (!llc->qp_mtu)
 845                 goto out_reject;
 846
 847         ini.vlan_id = lgr->vlan_id;
 848         smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev);
 849         if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
 850             !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN)) {
 851                 if (!ini.ib_dev)
 852                         goto out_reject;
 853                 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
 854         }
 855         if (!ini.ib_dev) {
 856                 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
 857                 ini.ib_dev = link->smcibdev;
 858                 ini.ib_port = link->ibport;
 859         }
 860         lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
 861         if (lnk_idx < 0)
 862                 goto out_reject;
 863         lnk_new = &lgr->lnk[lnk_idx];
 864         rc = smcr_link_init(lgr, lnk_new, lnk_idx, &ini);
 865         if (rc)
 866                 goto out_reject;
 867         smc_llc_save_add_link_info(lnk_new, llc);
 868         lnk_new->link_id = llc->link_num;       /* SMC server assigns link id */
 869         smc_llc_link_set_uid(lnk_new);
 870
 871         rc = smc_ib_ready_link(lnk_new);
 872         if (rc)
 873                 goto out_clear_lnk;
 874
 875         rc = smcr_buf_map_lgr(lnk_new);
 876         if (rc)
 877                 goto out_clear_lnk;
 878
 879         rc = smc_llc_send_add_link(link,
 880                                    lnk_new->smcibdev->mac[ini.ib_port - 1],
 881                                    lnk_new->gid, lnk_new, SMC_LLC_RESP);
 882         if (rc)
 883                 goto out_clear_lnk;
 884         rc = smc_llc_cli_rkey_exchange(link, lnk_new);
 885         if (rc) {
 886                 rc = 0;
 887                 goto out_clear_lnk;
 888         }
 889         rc = smc_llc_cli_conf_link(link, &ini, lnk_new, lgr_new_t);
 890         if (!rc)
 891                 goto out;
 892 out_clear_lnk:
 893         smcr_link_clear(lnk_new, false);
 894 out_reject:
 895         smc_llc_cli_add_link_reject(qentry);
 896 out:
 897         kfree(qentry);
 898         return rc;
 899 }
 900
 901 /* as an SMC client, invite server to start the add_link processing */
 902 static void smc_llc_cli_add_link_invite(struct smc_link *link,
 903                                         struct smc_llc_qentry *qentry)
 904 {
 905         struct smc_link_group *lgr = smc_get_lgr(link);
 906         struct smc_init_info ini;
 907
 908         if (lgr->type == SMC_LGR_SYMMETRIC ||
 909             lgr->type == SMC_LGR_ASYMMETRIC_PEER)
 910                 goto out;
 911
 912         ini.vlan_id = lgr->vlan_id;
 913         smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev);
 914         if (!ini.ib_dev)
 915                 goto out;
 916
 917         smc_llc_send_add_link(link, ini.ib_dev->mac[ini.ib_port - 1],
 918                               ini.ib_gid, NULL, SMC_LLC_REQ);
 919 out:
 920         kfree(qentry);
 921 }
 922
 923 static bool smc_llc_is_empty_llc_message(union smc_llc_msg *llc)
 924 {
 925         int i;
 926
 927         for (i = 0; i < ARRAY_SIZE(llc->raw.data); i++)
 928                 if (llc->raw.data[i])
 929                         return false;
 930         return true;
 931 }
 932
 933 static bool smc_llc_is_local_add_link(union smc_llc_msg *llc)
 934 {
 935         if (llc->raw.hdr.common.type == SMC_LLC_ADD_LINK &&
 936             smc_llc_is_empty_llc_message(llc))
 937                 return true;
 938         return false;
 939 }
 940
 941 static void smc_llc_process_cli_add_link(struct smc_link_group *lgr)
 942 {
 943         struct smc_llc_qentry *qentry;
 944
 945         qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
 946
 947         mutex_lock(&lgr->llc_conf_mutex);
 948         if (smc_llc_is_local_add_link(&qentry->msg))
 949                 smc_llc_cli_add_link_invite(qentry->link, qentry);
 950         else
 951                 smc_llc_cli_add_link(qentry->link, qentry);
 952         mutex_unlock(&lgr->llc_conf_mutex);
 953 }
 954
 955 static int smc_llc_active_link_count(struct smc_link_group *lgr)
 956 {
 957         int i, link_count = 0;
 958
 959         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 960                 if (!smc_link_active(&lgr->lnk[i]))
 961                         continue;
 962                 link_count++;
 963         }
 964         return link_count;
 965 }
 966
 967 /* find the asymmetric link when 3 links are established  */
 968 static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr)
 969 {
 970         int asym_idx = -ENOENT;
 971         int i, j, k;
 972         bool found;
 973
 974         /* determine asymmetric link */
 975         found = false;
 976         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 977                 for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) {
 978                         if (!smc_link_usable(&lgr->lnk[i]) ||
 979                             !smc_link_usable(&lgr->lnk[j]))
 980                                 continue;
 981                         if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid,
 982                                     SMC_GID_SIZE)) {
 983                                 found = true;   /* asym_lnk is i or j */
 984                                 break;
 985                         }
 986                 }
 987                 if (found)
 988                         break;
 989         }
 990         if (!found)
 991                 goto out; /* no asymmetric link */
 992         for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) {
 993                 if (!smc_link_usable(&lgr->lnk[k]))
 994                         continue;
 995                 if (k != i &&
 996                     !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid,
 997                             SMC_GID_SIZE)) {
 998                         asym_idx = i;
 999                         break;
1000                 }
1001                 if (k != j &&
1002                     !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid,
1003                             SMC_GID_SIZE)) {
1004                         asym_idx = j;
1005                         break;
1006                 }
1007         }
1008 out:
1009         return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx];
1010 }
1011
1012 static void smc_llc_delete_asym_link(struct smc_link_group *lgr)
1013 {
1014         struct smc_link *lnk_new = NULL, *lnk_asym;
1015         struct smc_llc_qentry *qentry;
1016         int rc;
1017
1018         lnk_asym = smc_llc_find_asym_link(lgr);
1019         if (!lnk_asym)
1020                 return; /* no asymmetric link */
1021         if (!smc_link_downing(&lnk_asym->state))
1022                 return;
1023         lnk_new = smc_switch_conns(lgr, lnk_asym, false);
1024         smc_wr_tx_wait_no_pending_sends(lnk_asym);
1025         if (!lnk_new)
1026                 goto out_free;
1027         /* change flow type from ADD_LINK into DEL_LINK */
1028         lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK;
1029         rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ,
1030                                       true, SMC_LLC_DEL_NO_ASYM_NEEDED);
1031         if (rc) {
1032                 smcr_link_down_cond(lnk_new);
1033                 goto out_free;
1034         }
1035         qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME,
1036                               SMC_LLC_DELETE_LINK);
1037         if (!qentry) {
1038                 smcr_link_down_cond(lnk_new);
1039                 goto out_free;
1040         }
1041         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1042 out_free:
1043         smcr_link_clear(lnk_asym, true);
1044 }
1045
1046 static int smc_llc_srv_rkey_exchange(struct smc_link *link,
1047                                      struct smc_link *link_new)
1048 {
1049         struct smc_llc_msg_add_link_cont *addc_llc;
1050         struct smc_link_group *lgr = link->lgr;
1051         u8 max, num_rkeys_send, num_rkeys_recv;
1052         struct smc_llc_qentry *qentry = NULL;
1053         struct smc_buf_desc *buf_pos;
1054         int buf_lst;
1055         int rc = 0;
1056         int i;
1057
1058         mutex_lock(&lgr->rmbs_lock);
1059         num_rkeys_send = lgr->conns_num;
1060         buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
1061         do {
1062                 smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
1063                                       &buf_lst, &buf_pos);
1064                 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME,
1065                                       SMC_LLC_ADD_LINK_CONT);
1066                 if (!qentry) {
1067                         rc = -ETIMEDOUT;
1068                         goto out;
1069                 }
1070                 addc_llc = &qentry->msg.add_link_cont;
1071                 num_rkeys_recv = addc_llc->num_rkeys;
1072                 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
1073                 for (i = 0; i < max; i++) {
1074                         smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
1075                                        addc_llc->rt[i].rmb_key,
1076                                        addc_llc->rt[i].rmb_vaddr_new,
1077                                        addc_llc->rt[i].rmb_key_new);
1078                         num_rkeys_recv--;
1079                 }
1080                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1081         } while (num_rkeys_send || num_rkeys_recv);
1082 out:
1083         mutex_unlock(&lgr->rmbs_lock);
1084         return rc;
1085 }
1086
1087 static int smc_llc_srv_conf_link(struct smc_link *link,
1088                                  struct smc_link *link_new,
1089                                  enum smc_lgr_type lgr_new_t)
1090 {
1091         struct smc_link_group *lgr = link->lgr;
1092         struct smc_llc_qentry *qentry = NULL;
1093         int rc;
1094
1095         /* send CONFIRM LINK request over the RoCE fabric */
1096         rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ);
1097         if (rc)
1098                 return -ENOLINK;
1099         /* receive CONFIRM LINK response over the RoCE fabric */
1100         qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 0);
1101         if (!qentry ||
1102             qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) {
1103                 /* send DELETE LINK */
1104                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
1105                                          false, SMC_LLC_DEL_LOST_PATH);
1106                 if (qentry)
1107                         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1108                 return -ENOLINK;
1109         }
1110         smc_llc_save_peer_uid(qentry);
1111         smc_llc_link_active(link_new);
1112         if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
1113             lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
1114                 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
1115         else
1116                 smcr_lgr_set_type(lgr, lgr_new_t);
1117         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1118         return 0;
1119 }
1120
1121 int smc_llc_srv_add_link(struct smc_link *link)
1122 {
1123         enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1124         struct smc_link_group *lgr = link->lgr;
1125         struct smc_llc_msg_add_link *add_llc;
1126         struct smc_llc_qentry *qentry = NULL;
1127         struct smc_link *link_new;
1128         struct smc_init_info ini;
1129         int lnk_idx, rc = 0;
1130
1131         /* ignore client add link recommendation, start new flow */
1132         ini.vlan_id = lgr->vlan_id;
1133         smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev);
1134         if (!ini.ib_dev) {
1135                 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1136                 ini.ib_dev = link->smcibdev;
1137                 ini.ib_port = link->ibport;
1138         }
1139         lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1140         if (lnk_idx < 0)
1141                 return 0;
1142
1143         rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, &ini);
1144         if (rc)
1145                 return rc;
1146         link_new = &lgr->lnk[lnk_idx];
1147         rc = smc_llc_send_add_link(link,
1148                                    link_new->smcibdev->mac[ini.ib_port - 1],
1149                                    link_new->gid, link_new, SMC_LLC_REQ);
1150         if (rc)
1151                 goto out_err;
1152         /* receive ADD LINK response over the RoCE fabric */
1153         qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK);
1154         if (!qentry) {
1155                 rc = -ETIMEDOUT;
1156                 goto out_err;
1157         }
1158         add_llc = &qentry->msg.add_link;
1159         if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) {
1160                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1161                 rc = -ENOLINK;
1162                 goto out_err;
1163         }
1164         if (lgr->type == SMC_LGR_SINGLE &&
1165             (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
1166              !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN))) {
1167                 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1168         }
1169         smc_llc_save_add_link_info(link_new, add_llc);
1170         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1171
1172         rc = smc_ib_ready_link(link_new);
1173         if (rc)
1174                 goto out_err;
1175         rc = smcr_buf_map_lgr(link_new);
1176         if (rc)
1177                 goto out_err;
1178         rc = smcr_buf_reg_lgr(link_new);
1179         if (rc)
1180                 goto out_err;
1181         rc = smc_llc_srv_rkey_exchange(link, link_new);
1182         if (rc)
1183                 goto out_err;
1184         rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t);
1185         if (rc)
1186                 goto out_err;
1187         return 0;
1188 out_err:
1189         smcr_link_clear(link_new, false);
1190         return rc;
1191 }
1192
1193 static void smc_llc_process_srv_add_link(struct smc_link_group *lgr)
1194 {
1195         struct smc_link *link = lgr->llc_flow_lcl.qentry->link;
1196         int rc;
1197
1198         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1199
1200         mutex_lock(&lgr->llc_conf_mutex);
1201         rc = smc_llc_srv_add_link(link);
1202         if (!rc && lgr->type == SMC_LGR_SYMMETRIC) {
1203                 /* delete any asymmetric link */
1204                 smc_llc_delete_asym_link(lgr);
1205         }
1206         mutex_unlock(&lgr->llc_conf_mutex);
1207 }
1208
1209 /* enqueue a local add_link req to trigger a new add_link flow */
1210 void smc_llc_add_link_local(struct smc_link *link)
1211 {
1212         struct smc_llc_msg_add_link add_llc = {0};
1213
1214         add_llc.hd.length = sizeof(add_llc);
1215         add_llc.hd.common.type = SMC_LLC_ADD_LINK;
1216         /* no dev and port needed */
1217         smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc);
1218 }
1219
1220 /* worker to process an add link message */
1221 static void smc_llc_add_link_work(struct work_struct *work)
1222 {
1223         struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1224                                                   llc_add_link_work);
1225
1226         if (list_empty(&lgr->list)) {
1227                 /* link group is terminating */
1228                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1229                 goto out;
1230         }
1231
1232         if (lgr->role == SMC_CLNT)
1233                 smc_llc_process_cli_add_link(lgr);
1234         else
1235                 smc_llc_process_srv_add_link(lgr);
1236 out:
1237         smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1238 }
1239
1240 /* enqueue a local del_link msg to trigger a new del_link flow,
1241  * called only for role SMC_SERV
1242  */
1243 void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id)
1244 {
1245         struct smc_llc_msg_del_link del_llc = {0};
1246
1247         del_llc.hd.length = sizeof(del_llc);
1248         del_llc.hd.common.type = SMC_LLC_DELETE_LINK;
1249         del_llc.link_num = del_link_id;
1250         del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH);
1251         del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1252         smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc);
1253 }
1254
1255 static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr)
1256 {
1257         struct smc_link *lnk_del = NULL, *lnk_asym, *lnk;
1258         struct smc_llc_msg_del_link *del_llc;
1259         struct smc_llc_qentry *qentry;
1260         int active_links;
1261         int lnk_idx;
1262
1263         qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1264         lnk = qentry->link;
1265         del_llc = &qentry->msg.delete_link;
1266
1267         if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1268                 smc_lgr_terminate_sched(lgr);
1269                 goto out;
1270         }
1271         mutex_lock(&lgr->llc_conf_mutex);
1272         /* delete single link */
1273         for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) {
1274                 if (lgr->lnk[lnk_idx].link_id != del_llc->link_num)
1275                         continue;
1276                 lnk_del = &lgr->lnk[lnk_idx];
1277                 break;
1278         }
1279         del_llc->hd.flags |= SMC_LLC_FLAG_RESP;
1280         if (!lnk_del) {
1281                 /* link was not found */
1282                 del_llc->reason = htonl(SMC_LLC_DEL_NOLNK);
1283                 smc_llc_send_message(lnk, &qentry->msg);
1284                 goto out_unlock;
1285         }
1286         lnk_asym = smc_llc_find_asym_link(lgr);
1287
1288         del_llc->reason = 0;
1289         smc_llc_send_message(lnk, &qentry->msg); /* response */
1290
1291         if (smc_link_downing(&lnk_del->state)) {
1292                 if (smc_switch_conns(lgr, lnk_del, false))
1293                         smc_wr_tx_wait_no_pending_sends(lnk_del);
1294         }
1295         smcr_link_clear(lnk_del, true);
1296
1297         active_links = smc_llc_active_link_count(lgr);
1298         if (lnk_del == lnk_asym) {
1299                 /* expected deletion of asym link, don't change lgr state */
1300         } else if (active_links == 1) {
1301                 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
1302         } else if (!active_links) {
1303                 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
1304                 smc_lgr_terminate_sched(lgr);
1305         }
1306 out_unlock:
1307         mutex_unlock(&lgr->llc_conf_mutex);
1308 out:
1309         kfree(qentry);
1310 }
1311
1312 /* try to send a DELETE LINK ALL request on any active link,
1313  * waiting for send completion
1314  */
1315 void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn)
1316 {
1317         struct smc_llc_msg_del_link delllc = {0};
1318         int i;
1319
1320         delllc.hd.common.type = SMC_LLC_DELETE_LINK;
1321         delllc.hd.length = sizeof(delllc);
1322         if (ord)
1323                 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1324         delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
1325         delllc.reason = htonl(rsn);
1326
1327         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1328                 if (!smc_link_usable(&lgr->lnk[i]))
1329                         continue;
1330                 if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc))
1331                         break;
1332         }
1333 }
1334
1335 static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr)
1336 {
1337         struct smc_llc_msg_del_link *del_llc;
1338         struct smc_link *lnk, *lnk_del;
1339         struct smc_llc_qentry *qentry;
1340         int active_links;
1341         int i;
1342
1343         mutex_lock(&lgr->llc_conf_mutex);
1344         qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1345         lnk = qentry->link;
1346         del_llc = &qentry->msg.delete_link;
1347
1348         if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1349                 /* delete entire lgr */
1350                 smc_llc_send_link_delete_all(lgr, true, ntohl(
1351                                               qentry->msg.delete_link.reason));
1352                 smc_lgr_terminate_sched(lgr);
1353                 goto out;
1354         }
1355         /* delete single link */
1356         lnk_del = NULL;
1357         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1358                 if (lgr->lnk[i].link_id == del_llc->link_num) {
1359                         lnk_del = &lgr->lnk[i];
1360                         break;
1361                 }
1362         }
1363         if (!lnk_del)
1364                 goto out; /* asymmetric link already deleted */
1365
1366         if (smc_link_downing(&lnk_del->state)) {
1367                 if (smc_switch_conns(lgr, lnk_del, false))
1368                         smc_wr_tx_wait_no_pending_sends(lnk_del);
1369         }
1370         if (!list_empty(&lgr->list)) {
1371                 /* qentry is either a request from peer (send it back to
1372                  * initiate the DELETE_LINK processing), or a locally
1373                  * enqueued DELETE_LINK request (forward it)
1374                  */
1375                 if (!smc_llc_send_message(lnk, &qentry->msg)) {
1376                         struct smc_llc_qentry *qentry2;
1377
1378                         qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME,
1379                                                SMC_LLC_DELETE_LINK);
1380                         if (qentry2)
1381                                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1382                 }
1383         }
1384         smcr_link_clear(lnk_del, true);
1385
1386         active_links = smc_llc_active_link_count(lgr);
1387         if (active_links == 1) {
1388                 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
1389         } else if (!active_links) {
1390                 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
1391                 smc_lgr_terminate_sched(lgr);
1392         }
1393
1394         if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) {
1395                 /* trigger setup of asymm alt link */
1396                 smc_llc_add_link_local(lnk);
1397         }
1398 out:
1399         mutex_unlock(&lgr->llc_conf_mutex);
1400         kfree(qentry);
1401 }
1402
1403 static void smc_llc_delete_link_work(struct work_struct *work)
1404 {
1405         struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1406                                                   llc_del_link_work);
1407
1408         if (list_empty(&lgr->list)) {
1409                 /* link group is terminating */
1410                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1411                 goto out;
1412         }
1413
1414         if (lgr->role == SMC_CLNT)
1415                 smc_llc_process_cli_delete_link(lgr);
1416         else
1417                 smc_llc_process_srv_delete_link(lgr);
1418 out:
1419         smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1420 }
1421
1422 /* process a confirm_rkey request from peer, remote flow */
1423 static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr)
1424 {
1425         struct smc_llc_msg_confirm_rkey *llc;
1426         struct smc_llc_qentry *qentry;
1427         struct smc_link *link;
1428         int num_entries;
1429         int rk_idx;
1430         int i;
1431
1432         qentry = lgr->llc_flow_rmt.qentry;
1433         llc = &qentry->msg.confirm_rkey;
1434         link = qentry->link;
1435
1436         num_entries = llc->rtoken[0].num_rkeys;
1437         /* first rkey entry is for receiving link */
1438         rk_idx = smc_rtoken_add(link,
1439                                 llc->rtoken[0].rmb_vaddr,
1440                                 llc->rtoken[0].rmb_key);
1441         if (rk_idx < 0)
1442                 goto out_err;
1443
1444         for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++)
1445                 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id,
1446                                 llc->rtoken[i].rmb_vaddr,
1447                                 llc->rtoken[i].rmb_key);
1448         /* max links is 3 so there is no need to support conf_rkey_cont msgs */
1449         goto out;
1450 out_err:
1451         llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1452         llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY;
1453 out:
1454         llc->hd.flags |= SMC_LLC_FLAG_RESP;
1455         smc_llc_send_message(link, &qentry->msg);
1456         smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1457 }
1458
1459 /* process a delete_rkey request from peer, remote flow */
1460 static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr)
1461 {
1462         struct smc_llc_msg_delete_rkey *llc;
1463         struct smc_llc_qentry *qentry;
1464         struct smc_link *link;
1465         u8 err_mask = 0;
1466         int i, max;
1467
1468         qentry = lgr->llc_flow_rmt.qentry;
1469         llc = &qentry->msg.delete_rkey;
1470         link = qentry->link;
1471
1472         max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
1473         for (i = 0; i < max; i++) {
1474                 if (smc_rtoken_delete(link, llc->rkey[i]))
1475                         err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
1476         }
1477         if (err_mask) {
1478                 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1479                 llc->err_mask = err_mask;
1480         }
1481         llc->hd.flags |= SMC_LLC_FLAG_RESP;
1482         smc_llc_send_message(link, &qentry->msg);
1483         smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1484 }
1485
1486 static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type)
1487 {
1488         pr_warn_ratelimited("smc: SMC-R lg %*phN LLC protocol violation: "
1489                             "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id, type);
1490         smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL);
1491         smc_lgr_terminate_sched(lgr);
1492 }
1493
1494 /* flush the llc event queue */
1495 static void smc_llc_event_flush(struct smc_link_group *lgr)
1496 {
1497         struct smc_llc_qentry *qentry, *q;
1498
1499         spin_lock_bh(&lgr->llc_event_q_lock);
1500         list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
1501                 list_del_init(&qentry->list);
1502                 kfree(qentry);
1503         }
1504         spin_unlock_bh(&lgr->llc_event_q_lock);
1505 }
1506
1507 static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
1508 {
1509         union smc_llc_msg *llc = &qentry->msg;
1510         struct smc_link *link = qentry->link;
1511         struct smc_link_group *lgr = link->lgr;
1512
1513         if (!smc_link_usable(link))
1514                 goto out;
1515
1516         switch (llc->raw.hdr.common.type) {
1517         case SMC_LLC_TEST_LINK:
1518                 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP;
1519                 smc_llc_send_message(link, llc);
1520                 break;
1521         case SMC_LLC_ADD_LINK:
1522                 if (list_empty(&lgr->list))
1523                         goto out;       /* lgr is terminating */
1524                 if (lgr->role == SMC_CLNT) {
1525                         if (smc_llc_is_local_add_link(llc)) {
1526                                 if (lgr->llc_flow_lcl.type ==
1527                                     SMC_LLC_FLOW_ADD_LINK)
1528                                         break;  /* add_link in progress */
1529                                 if (smc_llc_flow_start(&lgr->llc_flow_lcl,
1530                                                        qentry)) {
1531                                         schedule_work(&lgr->llc_add_link_work);
1532                                 }
1533                                 return;
1534                         }
1535                         if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1536                             !lgr->llc_flow_lcl.qentry) {
1537                                 /* a flow is waiting for this message */
1538                                 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1539                                                         qentry);
1540                                 wake_up(&lgr->llc_msg_waiter);
1541                         } else if (smc_llc_flow_start(&lgr->llc_flow_lcl,
1542                                                       qentry)) {
1543                                 schedule_work(&lgr->llc_add_link_work);
1544                         }
1545                 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1546                         /* as smc server, handle client suggestion */
1547                         schedule_work(&lgr->llc_add_link_work);
1548                 }
1549                 return;
1550         case SMC_LLC_CONFIRM_LINK:
1551         case SMC_LLC_ADD_LINK_CONT:
1552                 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
1553                         /* a flow is waiting for this message */
1554                         smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1555                         wake_up(&lgr->llc_msg_waiter);
1556                         return;
1557                 }
1558                 break;
1559         case SMC_LLC_DELETE_LINK:
1560                 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1561                     !lgr->llc_flow_lcl.qentry) {
1562                         /* DEL LINK REQ during ADD LINK SEQ */
1563                         smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1564                         wake_up(&lgr->llc_msg_waiter);
1565                 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1566                         schedule_work(&lgr->llc_del_link_work);
1567                 }
1568                 return;
1569         case SMC_LLC_CONFIRM_RKEY:
1570                 /* new request from remote, assign to remote flow */
1571                 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1572                         /* process here, does not wait for more llc msgs */
1573                         smc_llc_rmt_conf_rkey(lgr);
1574                         smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1575                 }
1576                 return;
1577         case SMC_LLC_CONFIRM_RKEY_CONT:
1578                 /* not used because max links is 3, and 3 rkeys fit into
1579                  * one CONFIRM_RKEY message
1580                  */
1581                 break;
1582         case SMC_LLC_DELETE_RKEY:
1583                 /* new request from remote, assign to remote flow */
1584                 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1585                         /* process here, does not wait for more llc msgs */
1586                         smc_llc_rmt_delete_rkey(lgr);
1587                         smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1588                 }
1589                 return;
1590         default:
1591                 smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type);
1592                 break;
1593         }
1594 out:
1595         kfree(qentry);
1596 }
1597
1598 /* worker to process llc messages on the event queue */
1599 static void smc_llc_event_work(struct work_struct *work)
1600 {
1601         struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1602                                                   llc_event_work);
1603         struct smc_llc_qentry *qentry;
1604
1605         if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {
1606                 if (smc_link_usable(lgr->delayed_event->link)) {
1607                         smc_llc_event_handler(lgr->delayed_event);
1608                 } else {
1609                         qentry = lgr->delayed_event;
1610                         lgr->delayed_event = NULL;
1611                         kfree(qentry);
1612                 }
1613         }
1614
1615 again:
1616         spin_lock_bh(&lgr->llc_event_q_lock);
1617         if (!list_empty(&lgr->llc_event_q)) {
1618                 qentry = list_first_entry(&lgr->llc_event_q,
1619                                           struct smc_llc_qentry, list);
1620                 list_del_init(&qentry->list);
1621                 spin_unlock_bh(&lgr->llc_event_q_lock);
1622                 smc_llc_event_handler(qentry);
1623                 goto again;
1624         }
1625         spin_unlock_bh(&lgr->llc_event_q_lock);
1626 }
1627
1628 /* process llc responses in tasklet context */
1629 static void smc_llc_rx_response(struct smc_link *link,
1630                                 struct smc_llc_qentry *qentry)
1631 {
1632         enum smc_llc_flowtype flowtype = link->lgr->llc_flow_lcl.type;
1633         struct smc_llc_flow *flow = &link->lgr->llc_flow_lcl;
1634         u8 llc_type = qentry->msg.raw.hdr.common.type;
1635
1636         switch (llc_type) {
1637         case SMC_LLC_TEST_LINK:
1638                 if (smc_link_active(link))
1639                         complete(&link->llc_testlink_resp);
1640                 break;
1641         case SMC_LLC_ADD_LINK:
1642         case SMC_LLC_ADD_LINK_CONT:
1643         case SMC_LLC_CONFIRM_LINK:
1644                 if (flowtype != SMC_LLC_FLOW_ADD_LINK || flow->qentry)
1645                         break;  /* drop out-of-flow response */
1646                 goto assign;
1647         case SMC_LLC_DELETE_LINK:
1648                 if (flowtype != SMC_LLC_FLOW_DEL_LINK || flow->qentry)
1649                         break;  /* drop out-of-flow response */
1650                 goto assign;
1651         case SMC_LLC_CONFIRM_RKEY:
1652         case SMC_LLC_DELETE_RKEY:
1653                 if (flowtype != SMC_LLC_FLOW_RKEY || flow->qentry)
1654                         break;  /* drop out-of-flow response */
1655                 goto assign;
1656         case SMC_LLC_CONFIRM_RKEY_CONT:
1657                 /* not used because max links is 3 */
1658                 break;
1659         default:
1660                 smc_llc_protocol_violation(link->lgr, llc_type);
1661                 break;
1662         }
1663         kfree(qentry);
1664         return;
1665 assign:
1666         /* assign responses to the local flow, we requested them */
1667         smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry);
1668         wake_up(&link->lgr->llc_msg_waiter);
1669 }
1670
1671 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc)
1672 {
1673         struct smc_link_group *lgr = link->lgr;
1674         struct smc_llc_qentry *qentry;
1675         unsigned long flags;
1676
1677         qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC);
1678         if (!qentry)
1679                 return;
1680         qentry->link = link;
1681         INIT_LIST_HEAD(&qentry->list);
1682         memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
1683
1684         /* process responses immediately */
1685         if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) {
1686                 smc_llc_rx_response(link, qentry);
1687                 return;
1688         }
1689
1690         /* add requests to event queue */
1691         spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
1692         list_add_tail(&qentry->list, &lgr->llc_event_q);
1693         spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);
1694         schedule_work(&lgr->llc_event_work);
1695 }
1696
1697 /* copy received msg and add it to the event queue */
1698 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
1699 {
1700         struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
1701         union smc_llc_msg *llc = buf;
1702
1703         if (wc->byte_len < sizeof(*llc))
1704                 return; /* short message */
1705         if (llc->raw.hdr.length != sizeof(*llc))
1706                 return; /* invalid message */
1707
1708         smc_llc_enqueue(link, llc);
1709 }
1710
1711 /***************************** worker, utils *********************************/
1712
1713 static void smc_llc_testlink_work(struct work_struct *work)
1714 {
1715         struct smc_link *link = container_of(to_delayed_work(work),
1716                                              struct smc_link, llc_testlink_wrk);
1717         unsigned long next_interval;
1718         unsigned long expire_time;
1719         u8 user_data[16] = { 0 };
1720         int rc;
1721
1722         if (!smc_link_active(link))
1723                 return;         /* don't reschedule worker */
1724         expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
1725         if (time_is_after_jiffies(expire_time)) {
1726                 next_interval = expire_time - jiffies;
1727                 goto out;
1728         }
1729         reinit_completion(&link->llc_testlink_resp);
1730         smc_llc_send_test_link(link, user_data);
1731         /* receive TEST LINK response over RoCE fabric */
1732         rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
1733                                                        SMC_LLC_WAIT_TIME);
1734         if (!smc_link_active(link))
1735                 return;         /* link state changed */
1736         if (rc <= 0) {
1737                 smcr_link_down_cond_sched(link);
1738                 return;
1739         }
1740         next_interval = link->llc_testlink_time;
1741 out:
1742         schedule_delayed_work(&link->llc_testlink_wrk, next_interval);
1743 }
1744
1745 void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
1746 {
1747         struct net *net = sock_net(smc->clcsock->sk);
1748
1749         INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);
1750         INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work);
1751         INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work);
1752         INIT_LIST_HEAD(&lgr->llc_event_q);
1753         spin_lock_init(&lgr->llc_event_q_lock);
1754         spin_lock_init(&lgr->llc_flow_lock);
1755         init_waitqueue_head(&lgr->llc_flow_waiter);
1756         init_waitqueue_head(&lgr->llc_msg_waiter);
1757         mutex_init(&lgr->llc_conf_mutex);
1758         lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time;
1759 }
1760
1761 /* called after lgr was removed from lgr_list */
1762 void smc_llc_lgr_clear(struct smc_link_group *lgr)
1763 {
1764         smc_llc_event_flush(lgr);
1765         wake_up_all(&lgr->llc_flow_waiter);
1766         wake_up_all(&lgr->llc_msg_waiter);
1767         cancel_work_sync(&lgr->llc_event_work);
1768         cancel_work_sync(&lgr->llc_add_link_work);
1769         cancel_work_sync(&lgr->llc_del_link_work);
1770         if (lgr->delayed_event) {
1771                 kfree(lgr->delayed_event);
1772                 lgr->delayed_event = NULL;
1773         }
1774 }
1775
1776 int smc_llc_link_init(struct smc_link *link)
1777 {
1778         init_completion(&link->llc_testlink_resp);
1779         INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
1780         return 0;
1781 }
1782
1783 void smc_llc_link_active(struct smc_link *link)
1784 {
1785         pr_warn_ratelimited("smc: SMC-R lg %*phN link added: id %*phN, "
1786                             "peerid %*phN, ibdev %s, ibport %d\n",
1787                             SMC_LGR_ID_SIZE, &link->lgr->id,
1788                             SMC_LGR_ID_SIZE, &link->link_uid,
1789                             SMC_LGR_ID_SIZE, &link->peer_link_uid,
1790                             link->smcibdev->ibdev->name, link->ibport);
1791         link->state = SMC_LNK_ACTIVE;
1792         if (link->lgr->llc_testlink_time) {
1793                 link->llc_testlink_time = link->lgr->llc_testlink_time * HZ;
1794                 schedule_delayed_work(&link->llc_testlink_wrk,
1795                                       link->llc_testlink_time);
1796         }
1797 }
1798
1799 /* called in worker context */
1800 void smc_llc_link_clear(struct smc_link *link, bool log)
1801 {
1802         if (log)
1803                 pr_warn_ratelimited("smc: SMC-R lg %*phN link removed: id %*phN"
1804                                     ", peerid %*phN, ibdev %s, ibport %d\n",
1805                                     SMC_LGR_ID_SIZE, &link->lgr->id,
1806                                     SMC_LGR_ID_SIZE, &link->link_uid,
1807                                     SMC_LGR_ID_SIZE, &link->peer_link_uid,
1808                                     link->smcibdev->ibdev->name, link->ibport);
1809         complete(&link->llc_testlink_resp);
1810         cancel_delayed_work_sync(&link->llc_testlink_wrk);
1811         smc_wr_wakeup_reg_wait(link);
1812         smc_wr_wakeup_tx_wait(link);
1813 }
1814
1815 /* register a new rtoken at the remote peer (for all links) */
1816 int smc_llc_do_confirm_rkey(struct smc_link *send_link,
1817                             struct smc_buf_desc *rmb_desc)
1818 {
1819         struct smc_link_group *lgr = send_link->lgr;
1820         struct smc_llc_qentry *qentry = NULL;
1821         int rc = 0;
1822
1823         rc = smc_llc_send_confirm_rkey(send_link, rmb_desc);
1824         if (rc)
1825                 goto out;
1826         /* receive CONFIRM RKEY response from server over RoCE fabric */
1827         qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
1828                               SMC_LLC_CONFIRM_RKEY);
1829         if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
1830                 rc = -EFAULT;
1831 out:
1832         if (qentry)
1833                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1834         return rc;
1835 }
1836
1837 /* unregister an rtoken at the remote peer */
1838 int smc_llc_do_delete_rkey(struct smc_link_group *lgr,
1839                            struct smc_buf_desc *rmb_desc)
1840 {
1841         struct smc_llc_qentry *qentry = NULL;
1842         struct smc_link *send_link;
1843         int rc = 0;
1844
1845         send_link = smc_llc_usable_link(lgr);
1846         if (!send_link)
1847                 return -ENOLINK;
1848
1849         /* protected by llc_flow control */
1850         rc = smc_llc_send_delete_rkey(send_link, rmb_desc);
1851         if (rc)
1852                 goto out;
1853         /* receive DELETE RKEY response from server over RoCE fabric */
1854         qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
1855                               SMC_LLC_DELETE_RKEY);
1856         if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
1857                 rc = -EFAULT;
1858 out:
1859         if (qentry)
1860                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1861         return rc;
1862 }
1863
1864 void smc_llc_link_set_uid(struct smc_link *link)
1865 {
1866         __be32 link_uid;
1867
1868         link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id);
1869         memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE);
1870 }
1871
1872 /* save peers link user id, used for debug purposes */
1873 void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry)
1874 {
1875         memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid,
1876                SMC_LGR_ID_SIZE);
1877 }
1878
1879 /* evaluate confirm link request or response */
1880 int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry,
1881                            enum smc_llc_reqresp type)
1882 {
1883         if (type == SMC_LLC_REQ) {      /* SMC server assigns link_id */
1884                 qentry->link->link_id = qentry->msg.confirm_link.link_num;
1885                 smc_llc_link_set_uid(qentry->link);
1886         }
1887         if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
1888                 return -ENOTSUPP;
1889         return 0;
1890 }
1891
1892 /***************************** init, exit, misc ******************************/
1893
1894 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
1895         {
1896                 .handler        = smc_llc_rx_handler,
1897                 .type           = SMC_LLC_CONFIRM_LINK
1898         },
1899         {
1900                 .handler        = smc_llc_rx_handler,
1901                 .type           = SMC_LLC_TEST_LINK
1902         },
1903         {
1904                 .handler        = smc_llc_rx_handler,
1905                 .type           = SMC_LLC_ADD_LINK
1906         },
1907         {
1908                 .handler        = smc_llc_rx_handler,
1909                 .type           = SMC_LLC_ADD_LINK_CONT
1910         },
1911         {
1912                 .handler        = smc_llc_rx_handler,
1913                 .type           = SMC_LLC_DELETE_LINK
1914         },
1915         {
1916                 .handler        = smc_llc_rx_handler,
1917                 .type           = SMC_LLC_CONFIRM_RKEY
1918         },
1919         {
1920                 .handler        = smc_llc_rx_handler,
1921                 .type           = SMC_LLC_CONFIRM_RKEY_CONT
1922         },
1923         {
1924                 .handler        = smc_llc_rx_handler,
1925                 .type           = SMC_LLC_DELETE_RKEY
1926         },
1927         {
1928                 .handler        = NULL,
1929         }
1930 };
1931
1932 int __init smc_llc_init(void)
1933 {
1934         struct smc_wr_rx_handler *handler;
1935         int rc = 0;
1936
1937         for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
1938                 INIT_HLIST_NODE(&handler->list);
1939                 rc = smc_wr_rx_register_handler(handler);
1940                 if (rc)
1941                         break;
1942         }
1943         return rc;
1944 }