net/smc/smc_llc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  Link Layer Control (LLC)
   6  *
   7  *  Copyright IBM Corp. 2016
   8  *
   9  *  Author(s):  Klaus Wacker <Klaus.Wacker@de.ibm.com>
  10  *              Ursula Braun <ubraun@linux.vnet.ibm.com>
  11  */
  12
  13 #include <net/tcp.h>
  14 #include <rdma/ib_verbs.h>
  15
  16 #include "smc.h"
  17 #include "smc_core.h"
  18 #include "smc_clc.h"
  19 #include "smc_llc.h"
  20
  21 #define SMC_LLC_DATA_LEN                40
  22
  23 struct smc_llc_hdr {
  24         struct smc_wr_rx_hdr common;
  25         u8 length;      /* 44 */
  26 #if defined(__BIG_ENDIAN_BITFIELD)
  27         u8 reserved:4,
  28            add_link_rej_rsn:4;
  29 #elif defined(__LITTLE_ENDIAN_BITFIELD)
  30         u8 add_link_rej_rsn:4,
  31            reserved:4;
  32 #endif
  33         u8 flags;
  34 };
  35
  36 #define SMC_LLC_FLAG_NO_RMBE_EYEC       0x03
  37
  38 struct smc_llc_msg_confirm_link {       /* type 0x01 */
  39         struct smc_llc_hdr hd;
  40         u8 sender_mac[ETH_ALEN];
  41         u8 sender_gid[SMC_GID_SIZE];
  42         u8 sender_qp_num[3];
  43         u8 link_num;
  44         u8 link_uid[SMC_LGR_ID_SIZE];
  45         u8 max_links;
  46         u8 reserved[9];
  47 };
  48
  49 #define SMC_LLC_FLAG_ADD_LNK_REJ        0x40
  50 #define SMC_LLC_REJ_RSN_NO_ALT_PATH     1
  51
  52 #define SMC_LLC_ADD_LNK_MAX_LINKS       2
  53
  54 struct smc_llc_msg_add_link {           /* type 0x02 */
  55         struct smc_llc_hdr hd;
  56         u8 sender_mac[ETH_ALEN];
  57         u8 reserved2[2];
  58         u8 sender_gid[SMC_GID_SIZE];
  59         u8 sender_qp_num[3];
  60         u8 link_num;
  61         u8 flags2;      /* QP mtu */
  62         u8 initial_psn[3];
  63         u8 reserved[8];
  64 };
  65
  66 #define SMC_LLC_FLAG_DEL_LINK_ALL       0x40
  67 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY   0x20
  68
  69 struct smc_llc_msg_del_link {           /* type 0x04 */
  70         struct smc_llc_hdr hd;
  71         u8 link_num;
  72         __be32 reason;
  73         u8 reserved[35];
  74 } __packed;                     /* format defined in RFC7609 */
  75
  76 struct smc_llc_msg_test_link {          /* type 0x07 */
  77         struct smc_llc_hdr hd;
  78         u8 user_data[16];
  79         u8 reserved[24];
  80 };
  81
  82 struct smc_rmb_rtoken {
  83         union {
  84                 u8 num_rkeys;   /* first rtoken byte of CONFIRM LINK msg */
  85                                 /* is actually the num of rtokens, first */
  86                                 /* rtoken is always for the current link */
  87                 u8 link_id;     /* link id of the rtoken */
  88         };
  89         __be32 rmb_key;
  90         __be64 rmb_vaddr;
  91 } __packed;                     /* format defined in RFC7609 */
  92
  93 #define SMC_LLC_RKEYS_PER_MSG   3
  94
  95 struct smc_llc_msg_confirm_rkey {       /* type 0x06 */
  96         struct smc_llc_hdr hd;
  97         struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
  98         u8 reserved;
  99 };
 100
 101 #define SMC_LLC_DEL_RKEY_MAX    8
 102 #define SMC_LLC_FLAG_RKEY_RETRY 0x10
 103 #define SMC_LLC_FLAG_RKEY_NEG   0x20
 104
 105 struct smc_llc_msg_delete_rkey {        /* type 0x09 */
 106         struct smc_llc_hdr hd;
 107         u8 num_rkeys;
 108         u8 err_mask;
 109         u8 reserved[2];
 110         __be32 rkey[8];
 111         u8 reserved2[4];
 112 };
 113
 114 union smc_llc_msg {
 115         struct smc_llc_msg_confirm_link confirm_link;
 116         struct smc_llc_msg_add_link add_link;
 117         struct smc_llc_msg_del_link delete_link;
 118
 119         struct smc_llc_msg_confirm_rkey confirm_rkey;
 120         struct smc_llc_msg_delete_rkey delete_rkey;
 121
 122         struct smc_llc_msg_test_link test_link;
 123         struct {
 124                 struct smc_llc_hdr hdr;
 125                 u8 data[SMC_LLC_DATA_LEN];
 126         } raw;
 127 };
 128
 129 #define SMC_LLC_FLAG_RESP               0x80
 130
 131 struct smc_llc_qentry {
 132         struct list_head list;
 133         struct smc_link *link;
 134         union smc_llc_msg msg;
 135 };
 136
 137 struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow)
 138 {
 139         struct smc_llc_qentry *qentry = flow->qentry;
 140
 141         flow->qentry = NULL;
 142         return qentry;
 143 }
 144
 145 void smc_llc_flow_qentry_del(struct smc_llc_flow *flow)
 146 {
 147         struct smc_llc_qentry *qentry;
 148
 149         if (flow->qentry) {
 150                 qentry = flow->qentry;
 151                 flow->qentry = NULL;
 152                 kfree(qentry);
 153         }
 154 }
 155
 156 static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow,
 157                                            struct smc_llc_qentry *qentry)
 158 {
 159         flow->qentry = qentry;
 160 }
 161
 162 /* try to start a new llc flow, initiated by an incoming llc msg */
 163 static bool smc_llc_flow_start(struct smc_llc_flow *flow,
 164                                struct smc_llc_qentry *qentry)
 165 {
 166         struct smc_link_group *lgr = qentry->link->lgr;
 167
 168         spin_lock_bh(&lgr->llc_flow_lock);
 169         if (flow->type) {
 170                 /* a flow is already active */
 171                 if ((qentry->msg.raw.hdr.common.type == SMC_LLC_ADD_LINK ||
 172                      qentry->msg.raw.hdr.common.type == SMC_LLC_DELETE_LINK) &&
 173                     !lgr->delayed_event) {
 174                         lgr->delayed_event = qentry;
 175                 } else {
 176                         /* forget this llc request */
 177                         kfree(qentry);
 178                 }
 179                 spin_unlock_bh(&lgr->llc_flow_lock);
 180                 return false;
 181         }
 182         switch (qentry->msg.raw.hdr.common.type) {
 183         case SMC_LLC_ADD_LINK:
 184                 flow->type = SMC_LLC_FLOW_ADD_LINK;
 185                 break;
 186         case SMC_LLC_DELETE_LINK:
 187                 flow->type = SMC_LLC_FLOW_DEL_LINK;
 188                 break;
 189         case SMC_LLC_CONFIRM_RKEY:
 190         case SMC_LLC_DELETE_RKEY:
 191                 flow->type = SMC_LLC_FLOW_RKEY;
 192                 break;
 193         default:
 194                 flow->type = SMC_LLC_FLOW_NONE;
 195         }
 196         if (qentry == lgr->delayed_event)
 197                 lgr->delayed_event = NULL;
 198         spin_unlock_bh(&lgr->llc_flow_lock);
 199         smc_llc_flow_qentry_set(flow, qentry);
 200         return true;
 201 }
 202
 203 /* start a new local llc flow, wait till current flow finished */
 204 int smc_llc_flow_initiate(struct smc_link_group *lgr,
 205                           enum smc_llc_flowtype type)
 206 {
 207         enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE;
 208         int rc;
 209
 210         /* all flows except confirm_rkey and delete_rkey are exclusive,
 211          * confirm/delete rkey flows can run concurrently (local and remote)
 212          */
 213         if (type == SMC_LLC_FLOW_RKEY)
 214                 allowed_remote = SMC_LLC_FLOW_RKEY;
 215 again:
 216         if (list_empty(&lgr->list))
 217                 return -ENODEV;
 218         spin_lock_bh(&lgr->llc_flow_lock);
 219         if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
 220             (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
 221              lgr->llc_flow_rmt.type == allowed_remote)) {
 222                 lgr->llc_flow_lcl.type = type;
 223                 spin_unlock_bh(&lgr->llc_flow_lock);
 224                 return 0;
 225         }
 226         spin_unlock_bh(&lgr->llc_flow_lock);
 227         rc = wait_event_interruptible_timeout(lgr->llc_waiter,
 228                         (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
 229                          (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
 230                           lgr->llc_flow_rmt.type == allowed_remote)),
 231                         SMC_LLC_WAIT_TIME);
 232         if (!rc)
 233                 return -ETIMEDOUT;
 234         goto again;
 235 }
 236
 237 /* finish the current llc flow */
 238 void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)
 239 {
 240         spin_lock_bh(&lgr->llc_flow_lock);
 241         memset(flow, 0, sizeof(*flow));
 242         flow->type = SMC_LLC_FLOW_NONE;
 243         spin_unlock_bh(&lgr->llc_flow_lock);
 244         if (!list_empty(&lgr->list) && lgr->delayed_event &&
 245             flow == &lgr->llc_flow_lcl)
 246                 schedule_work(&lgr->llc_event_work);
 247         else
 248                 wake_up_interruptible(&lgr->llc_waiter);
 249 }
 250
 251 /* lnk is optional and used for early wakeup when link goes down, useful in
 252  * cases where we wait for a response on the link after we sent a request
 253  */
 254 struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
 255                                     struct smc_link *lnk,
 256                                     int time_out, u8 exp_msg)
 257 {
 258         struct smc_llc_flow *flow = &lgr->llc_flow_lcl;
 259
 260         wait_event_interruptible_timeout(lgr->llc_waiter,
 261                                          (flow->qentry ||
 262                                           (lnk && !smc_link_usable(lnk)) ||
 263                                           list_empty(&lgr->list)),
 264                                          time_out);
 265         if (!flow->qentry ||
 266             (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) {
 267                 smc_llc_flow_qentry_del(flow);
 268                 goto out;
 269         }
 270         if (exp_msg && flow->qentry->msg.raw.hdr.common.type != exp_msg) {
 271                 if (exp_msg == SMC_LLC_ADD_LINK &&
 272                     flow->qentry->msg.raw.hdr.common.type ==
 273                     SMC_LLC_DELETE_LINK) {
 274                         /* flow_start will delay the unexpected msg */
 275                         smc_llc_flow_start(&lgr->llc_flow_lcl,
 276                                            smc_llc_flow_qentry_clr(flow));
 277                         return NULL;
 278                 }
 279                 smc_llc_flow_qentry_del(flow);
 280         }
 281 out:
 282         return flow->qentry;
 283 }
 284
 285 /********************************** send *************************************/
 286
 287 struct smc_llc_tx_pend {
 288 };
 289
 290 /* handler for send/transmission completion of an LLC msg */
 291 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
 292                                struct smc_link *link,
 293                                enum ib_wc_status wc_status)
 294 {
 295         /* future work: handle wc_status error for recovery and failover */
 296 }
 297
 298 /**
 299  * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
 300  * @link: Pointer to SMC link used for sending LLC control message.
 301  * @wr_buf: Out variable returning pointer to work request payload buffer.
 302  * @pend: Out variable returning pointer to private pending WR tracking.
 303  *        It's the context the transmit complete handler will get.
 304  *
 305  * Reserves and pre-fills an entry for a pending work request send/tx.
 306  * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
 307  * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
 308  *
 309  * Return: 0 on success, otherwise an error value.
 310  */
 311 static int smc_llc_add_pending_send(struct smc_link *link,
 312                                     struct smc_wr_buf **wr_buf,
 313                                     struct smc_wr_tx_pend_priv **pend)
 314 {
 315         int rc;
 316
 317         rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
 318                                      pend);
 319         if (rc < 0)
 320                 return rc;
 321         BUILD_BUG_ON_MSG(
 322                 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
 323                 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
 324         BUILD_BUG_ON_MSG(
 325                 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
 326                 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
 327         BUILD_BUG_ON_MSG(
 328                 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
 329                 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
 330         return 0;
 331 }
 332
 333 /* high-level API to send LLC confirm link */
 334 int smc_llc_send_confirm_link(struct smc_link *link,
 335                               enum smc_llc_reqresp reqresp)
 336 {
 337         struct smc_link_group *lgr = smc_get_lgr(link);
 338         struct smc_llc_msg_confirm_link *confllc;
 339         struct smc_wr_tx_pend_priv *pend;
 340         struct smc_wr_buf *wr_buf;
 341         int rc;
 342
 343         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 344         if (rc)
 345                 return rc;
 346         confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
 347         memset(confllc, 0, sizeof(*confllc));
 348         confllc->hd.common.type = SMC_LLC_CONFIRM_LINK;
 349         confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link);
 350         confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
 351         if (reqresp == SMC_LLC_RESP)
 352                 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
 353         memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
 354                ETH_ALEN);
 355         memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
 356         hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
 357         confllc->link_num = link->link_id;
 358         memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE);
 359         confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */
 360         /* send llc message */
 361         rc = smc_wr_tx_send(link, pend);
 362         return rc;
 363 }
 364
 365 /* send LLC confirm rkey request */
 366 static int smc_llc_send_confirm_rkey(struct smc_link *send_link,
 367                                      struct smc_buf_desc *rmb_desc)
 368 {
 369         struct smc_llc_msg_confirm_rkey *rkeyllc;
 370         struct smc_wr_tx_pend_priv *pend;
 371         struct smc_wr_buf *wr_buf;
 372         struct smc_link *link;
 373         int i, rc, rtok_ix;
 374
 375         rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend);
 376         if (rc)
 377                 return rc;
 378         rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
 379         memset(rkeyllc, 0, sizeof(*rkeyllc));
 380         rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY;
 381         rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey);
 382
 383         rtok_ix = 1;
 384         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 385                 link = &send_link->lgr->lnk[i];
 386                 if (link->state == SMC_LNK_ACTIVE && link != send_link) {
 387                         rkeyllc->rtoken[rtok_ix].link_id = link->link_id;
 388                         rkeyllc->rtoken[rtok_ix].rmb_key =
 389                                 htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
 390                         rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64(
 391                                 (u64)sg_dma_address(
 392                                         rmb_desc->sgt[link->link_idx].sgl));
 393                         rtok_ix++;
 394                 }
 395         }
 396         /* rkey of send_link is in rtoken[0] */
 397         rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1;
 398         rkeyllc->rtoken[0].rmb_key =
 399                 htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey);
 400         rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64(
 401                 (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl));
 402         /* send llc message */
 403         rc = smc_wr_tx_send(send_link, pend);
 404         return rc;
 405 }
 406
 407 /* send LLC delete rkey request */
 408 static int smc_llc_send_delete_rkey(struct smc_link *link,
 409                                     struct smc_buf_desc *rmb_desc)
 410 {
 411         struct smc_llc_msg_delete_rkey *rkeyllc;
 412         struct smc_wr_tx_pend_priv *pend;
 413         struct smc_wr_buf *wr_buf;
 414         int rc;
 415
 416         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 417         if (rc)
 418                 return rc;
 419         rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
 420         memset(rkeyllc, 0, sizeof(*rkeyllc));
 421         rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY;
 422         rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey);
 423         rkeyllc->num_rkeys = 1;
 424         rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
 425         /* send llc message */
 426         rc = smc_wr_tx_send(link, pend);
 427         return rc;
 428 }
 429
 430 /* prepare an add link message */
 431 static void smc_llc_prep_add_link(struct smc_llc_msg_add_link *addllc,
 432                                   struct smc_link *link, u8 mac[], u8 gid[],
 433                                   enum smc_llc_reqresp reqresp)
 434 {
 435         memset(addllc, 0, sizeof(*addllc));
 436         addllc->hd.common.type = SMC_LLC_ADD_LINK;
 437         addllc->hd.length = sizeof(struct smc_llc_msg_add_link);
 438         if (reqresp == SMC_LLC_RESP) {
 439                 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
 440                 /* always reject more links for now */
 441                 addllc->hd.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
 442                 addllc->hd.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
 443         }
 444         memcpy(addllc->sender_mac, mac, ETH_ALEN);
 445         memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
 446 }
 447
 448 /* send ADD LINK request or response */
 449 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
 450                           enum smc_llc_reqresp reqresp)
 451 {
 452         struct smc_llc_msg_add_link *addllc;
 453         struct smc_wr_tx_pend_priv *pend;
 454         struct smc_wr_buf *wr_buf;
 455         int rc;
 456
 457         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 458         if (rc)
 459                 return rc;
 460         addllc = (struct smc_llc_msg_add_link *)wr_buf;
 461         smc_llc_prep_add_link(addllc, link, mac, gid, reqresp);
 462         /* send llc message */
 463         rc = smc_wr_tx_send(link, pend);
 464         return rc;
 465 }
 466
 467 /* prepare a delete link message */
 468 static void smc_llc_prep_delete_link(struct smc_llc_msg_del_link *delllc,
 469                                      struct smc_link *link,
 470                                      enum smc_llc_reqresp reqresp, bool orderly)
 471 {
 472         memset(delllc, 0, sizeof(*delllc));
 473         delllc->hd.common.type = SMC_LLC_DELETE_LINK;
 474         delllc->hd.length = sizeof(struct smc_llc_msg_add_link);
 475         if (reqresp == SMC_LLC_RESP)
 476                 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
 477         /* DEL_LINK_ALL because only 1 link supported */
 478         delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
 479         if (orderly)
 480                 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
 481         delllc->link_num = link->link_id;
 482 }
 483
 484 /* send DELETE LINK request or response */
 485 int smc_llc_send_delete_link(struct smc_link *link,
 486                              enum smc_llc_reqresp reqresp, bool orderly)
 487 {
 488         struct smc_llc_msg_del_link *delllc;
 489         struct smc_wr_tx_pend_priv *pend;
 490         struct smc_wr_buf *wr_buf;
 491         int rc;
 492
 493         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 494         if (rc)
 495                 return rc;
 496         delllc = (struct smc_llc_msg_del_link *)wr_buf;
 497         smc_llc_prep_delete_link(delllc, link, reqresp, orderly);
 498         /* send llc message */
 499         rc = smc_wr_tx_send(link, pend);
 500         return rc;
 501 }
 502
 503 /* send LLC test link request */
 504 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
 505 {
 506         struct smc_llc_msg_test_link *testllc;
 507         struct smc_wr_tx_pend_priv *pend;
 508         struct smc_wr_buf *wr_buf;
 509         int rc;
 510
 511         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 512         if (rc)
 513                 return rc;
 514         testllc = (struct smc_llc_msg_test_link *)wr_buf;
 515         memset(testllc, 0, sizeof(*testllc));
 516         testllc->hd.common.type = SMC_LLC_TEST_LINK;
 517         testllc->hd.length = sizeof(struct smc_llc_msg_test_link);
 518         memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
 519         /* send llc message */
 520         rc = smc_wr_tx_send(link, pend);
 521         return rc;
 522 }
 523
 524 /* schedule an llc send on link, may wait for buffers */
 525 static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
 526 {
 527         struct smc_wr_tx_pend_priv *pend;
 528         struct smc_wr_buf *wr_buf;
 529         int rc;
 530
 531         if (!smc_link_usable(link))
 532                 return -ENOLINK;
 533         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 534         if (rc)
 535                 return rc;
 536         memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
 537         return smc_wr_tx_send(link, pend);
 538 }
 539
 540 /********************************* receive ***********************************/
 541
 542 static void smc_llc_rx_delete_link(struct smc_link *link,
 543                                    struct smc_llc_msg_del_link *llc)
 544 {
 545         struct smc_link_group *lgr = smc_get_lgr(link);
 546
 547         smc_lgr_forget(lgr);
 548         smc_llc_link_deleting(link);
 549         if (lgr->role == SMC_SERV) {
 550                 /* client asks to delete this link, send request */
 551                 smc_llc_prep_delete_link(llc, link, SMC_LLC_REQ, true);
 552         } else {
 553                 /* server requests to delete this link, send response */
 554                 smc_llc_prep_delete_link(llc, link, SMC_LLC_RESP, true);
 555         }
 556         smc_llc_send_message(link, llc);
 557         smc_lgr_terminate_sched(lgr);
 558 }
 559
 560 /* process a confirm_rkey request from peer, remote flow */
 561 static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr)
 562 {
 563         struct smc_llc_msg_confirm_rkey *llc;
 564         struct smc_llc_qentry *qentry;
 565         struct smc_link *link;
 566         int num_entries;
 567         int rk_idx;
 568         int i;
 569
 570         qentry = lgr->llc_flow_rmt.qentry;
 571         llc = &qentry->msg.confirm_rkey;
 572         link = qentry->link;
 573
 574         num_entries = llc->rtoken[0].num_rkeys;
 575         /* first rkey entry is for receiving link */
 576         rk_idx = smc_rtoken_add(link,
 577                                 llc->rtoken[0].rmb_vaddr,
 578                                 llc->rtoken[0].rmb_key);
 579         if (rk_idx < 0)
 580                 goto out_err;
 581
 582         for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++)
 583                 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id,
 584                                 llc->rtoken[i].rmb_vaddr,
 585                                 llc->rtoken[i].rmb_key);
 586         /* max links is 3 so there is no need to support conf_rkey_cont msgs */
 587         goto out;
 588 out_err:
 589         llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
 590         llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY;
 591 out:
 592         llc->hd.flags |= SMC_LLC_FLAG_RESP;
 593         smc_llc_send_message(link, &qentry->msg);
 594         smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
 595 }
 596
 597 /* process a delete_rkey request from peer, remote flow */
 598 static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr)
 599 {
 600         struct smc_llc_msg_delete_rkey *llc;
 601         struct smc_llc_qentry *qentry;
 602         struct smc_link *link;
 603         u8 err_mask = 0;
 604         int i, max;
 605
 606         qentry = lgr->llc_flow_rmt.qentry;
 607         llc = &qentry->msg.delete_rkey;
 608         link = qentry->link;
 609
 610         max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
 611         for (i = 0; i < max; i++) {
 612                 if (smc_rtoken_delete(link, llc->rkey[i]))
 613                         err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
 614         }
 615         if (err_mask) {
 616                 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
 617                 llc->err_mask = err_mask;
 618         }
 619         llc->hd.flags |= SMC_LLC_FLAG_RESP;
 620         smc_llc_send_message(link, &qentry->msg);
 621         smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
 622 }
 623
 624 /* flush the llc event queue */
 625 static void smc_llc_event_flush(struct smc_link_group *lgr)
 626 {
 627         struct smc_llc_qentry *qentry, *q;
 628
 629         spin_lock_bh(&lgr->llc_event_q_lock);
 630         list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
 631                 list_del_init(&qentry->list);
 632                 kfree(qentry);
 633         }
 634         spin_unlock_bh(&lgr->llc_event_q_lock);
 635 }
 636
 637 static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
 638 {
 639         union smc_llc_msg *llc = &qentry->msg;
 640         struct smc_link *link = qentry->link;
 641         struct smc_link_group *lgr = link->lgr;
 642
 643         if (!smc_link_usable(link))
 644                 goto out;
 645
 646         switch (llc->raw.hdr.common.type) {
 647         case SMC_LLC_TEST_LINK:
 648                 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP;
 649                 smc_llc_send_message(link, llc);
 650                 break;
 651         case SMC_LLC_ADD_LINK:
 652                 if (list_empty(&lgr->list))
 653                         goto out;       /* lgr is terminating */
 654                 if (lgr->role == SMC_CLNT) {
 655                         if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) {
 656                                 /* a flow is waiting for this message */
 657                                 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
 658                                                         qentry);
 659                                 wake_up_interruptible(&lgr->llc_waiter);
 660                         } else if (smc_llc_flow_start(&lgr->llc_flow_lcl,
 661                                                       qentry)) {
 662                                 /* tbd: schedule_work(&lgr->llc_add_link_work); */
 663                         }
 664                 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
 665                         /* as smc server, handle client suggestion */
 666                         /* tbd: schedule_work(&lgr->llc_add_link_work); */
 667                 }
 668                 return;
 669         case SMC_LLC_CONFIRM_LINK:
 670                 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
 671                         /* a flow is waiting for this message */
 672                         smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
 673                         wake_up_interruptible(&lgr->llc_waiter);
 674                         return;
 675                 }
 676                 break;
 677         case SMC_LLC_DELETE_LINK:
 678                 smc_llc_rx_delete_link(link, &llc->delete_link);
 679                 break;
 680         case SMC_LLC_CONFIRM_RKEY:
 681                 /* new request from remote, assign to remote flow */
 682                 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
 683                         /* process here, does not wait for more llc msgs */
 684                         smc_llc_rmt_conf_rkey(lgr);
 685                         smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
 686                 }
 687                 return;
 688         case SMC_LLC_CONFIRM_RKEY_CONT:
 689                 /* not used because max links is 3, and 3 rkeys fit into
 690                  * one CONFIRM_RKEY message
 691                  */
 692                 break;
 693         case SMC_LLC_DELETE_RKEY:
 694                 /* new request from remote, assign to remote flow */
 695                 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
 696                         /* process here, does not wait for more llc msgs */
 697                         smc_llc_rmt_delete_rkey(lgr);
 698                         smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
 699                 }
 700                 return;
 701         }
 702 out:
 703         kfree(qentry);
 704 }
 705
 706 /* worker to process llc messages on the event queue */
 707 static void smc_llc_event_work(struct work_struct *work)
 708 {
 709         struct smc_link_group *lgr = container_of(work, struct smc_link_group,
 710                                                   llc_event_work);
 711         struct smc_llc_qentry *qentry;
 712
 713         if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {
 714                 if (smc_link_usable(lgr->delayed_event->link)) {
 715                         smc_llc_event_handler(lgr->delayed_event);
 716                 } else {
 717                         qentry = lgr->delayed_event;
 718                         lgr->delayed_event = NULL;
 719                         kfree(qentry);
 720                 }
 721         }
 722
 723 again:
 724         spin_lock_bh(&lgr->llc_event_q_lock);
 725         if (!list_empty(&lgr->llc_event_q)) {
 726                 qentry = list_first_entry(&lgr->llc_event_q,
 727                                           struct smc_llc_qentry, list);
 728                 list_del_init(&qentry->list);
 729                 spin_unlock_bh(&lgr->llc_event_q_lock);
 730                 smc_llc_event_handler(qentry);
 731                 goto again;
 732         }
 733         spin_unlock_bh(&lgr->llc_event_q_lock);
 734 }
 735
 736 /* process llc responses in tasklet context */
 737 static void smc_llc_rx_response(struct smc_link *link,
 738                                 struct smc_llc_qentry *qentry)
 739 {
 740         u8 llc_type = qentry->msg.raw.hdr.common.type;
 741
 742         switch (llc_type) {
 743         case SMC_LLC_TEST_LINK:
 744                 if (link->state == SMC_LNK_ACTIVE)
 745                         complete(&link->llc_testlink_resp);
 746                 break;
 747         case SMC_LLC_ADD_LINK:
 748         case SMC_LLC_CONFIRM_LINK:
 749         case SMC_LLC_CONFIRM_RKEY:
 750         case SMC_LLC_DELETE_RKEY:
 751                 /* assign responses to the local flow, we requested them */
 752                 smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry);
 753                 wake_up_interruptible(&link->lgr->llc_waiter);
 754                 return;
 755         case SMC_LLC_DELETE_LINK:
 756                 if (link->lgr->role == SMC_SERV)
 757                         smc_lgr_schedule_free_work_fast(link->lgr);
 758                 break;
 759         case SMC_LLC_CONFIRM_RKEY_CONT:
 760                 /* not used because max links is 3 */
 761                 break;
 762         }
 763         kfree(qentry);
 764 }
 765
 766 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc)
 767 {
 768         struct smc_link_group *lgr = link->lgr;
 769         struct smc_llc_qentry *qentry;
 770         unsigned long flags;
 771
 772         qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC);
 773         if (!qentry)
 774                 return;
 775         qentry->link = link;
 776         INIT_LIST_HEAD(&qentry->list);
 777         memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
 778
 779         /* process responses immediately */
 780         if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) {
 781                 smc_llc_rx_response(link, qentry);
 782                 return;
 783         }
 784
 785         /* add requests to event queue */
 786         spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
 787         list_add_tail(&qentry->list, &lgr->llc_event_q);
 788         spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);
 789         schedule_work(&link->lgr->llc_event_work);
 790 }
 791
 792 /* copy received msg and add it to the event queue */
 793 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
 794 {
 795         struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
 796         union smc_llc_msg *llc = buf;
 797
 798         if (wc->byte_len < sizeof(*llc))
 799                 return; /* short message */
 800         if (llc->raw.hdr.length != sizeof(*llc))
 801                 return; /* invalid message */
 802
 803         smc_llc_enqueue(link, llc);
 804 }
 805
 806 /***************************** worker, utils *********************************/
 807
 808 static void smc_llc_testlink_work(struct work_struct *work)
 809 {
 810         struct smc_link *link = container_of(to_delayed_work(work),
 811                                              struct smc_link, llc_testlink_wrk);
 812         unsigned long next_interval;
 813         unsigned long expire_time;
 814         u8 user_data[16] = { 0 };
 815         int rc;
 816
 817         if (link->state != SMC_LNK_ACTIVE)
 818                 return;         /* don't reschedule worker */
 819         expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
 820         if (time_is_after_jiffies(expire_time)) {
 821                 next_interval = expire_time - jiffies;
 822                 goto out;
 823         }
 824         reinit_completion(&link->llc_testlink_resp);
 825         smc_llc_send_test_link(link, user_data);
 826         /* receive TEST LINK response over RoCE fabric */
 827         rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
 828                                                        SMC_LLC_WAIT_TIME);
 829         if (link->state != SMC_LNK_ACTIVE)
 830                 return;         /* link state changed */
 831         if (rc <= 0) {
 832                 smc_lgr_terminate_sched(smc_get_lgr(link));
 833                 return;
 834         }
 835         next_interval = link->llc_testlink_time;
 836 out:
 837         schedule_delayed_work(&link->llc_testlink_wrk, next_interval);
 838 }
 839
 840 void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
 841 {
 842         struct net *net = sock_net(smc->clcsock->sk);
 843
 844         INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);
 845         INIT_LIST_HEAD(&lgr->llc_event_q);
 846         spin_lock_init(&lgr->llc_event_q_lock);
 847         spin_lock_init(&lgr->llc_flow_lock);
 848         init_waitqueue_head(&lgr->llc_waiter);
 849         lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time;
 850 }
 851
 852 /* called after lgr was removed from lgr_list */
 853 void smc_llc_lgr_clear(struct smc_link_group *lgr)
 854 {
 855         smc_llc_event_flush(lgr);
 856         wake_up_interruptible_all(&lgr->llc_waiter);
 857         cancel_work_sync(&lgr->llc_event_work);
 858         if (lgr->delayed_event) {
 859                 kfree(lgr->delayed_event);
 860                 lgr->delayed_event = NULL;
 861         }
 862 }
 863
 864 int smc_llc_link_init(struct smc_link *link)
 865 {
 866         init_completion(&link->llc_testlink_resp);
 867         INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
 868         return 0;
 869 }
 870
 871 void smc_llc_link_active(struct smc_link *link)
 872 {
 873         link->state = SMC_LNK_ACTIVE;
 874         if (link->lgr->llc_testlink_time) {
 875                 link->llc_testlink_time = link->lgr->llc_testlink_time * HZ;
 876                 schedule_delayed_work(&link->llc_testlink_wrk,
 877                                       link->llc_testlink_time);
 878         }
 879 }
 880
 881 void smc_llc_link_deleting(struct smc_link *link)
 882 {
 883         link->state = SMC_LNK_DELETING;
 884         smc_wr_wakeup_tx_wait(link);
 885 }
 886
 887 /* called in worker context */
 888 void smc_llc_link_clear(struct smc_link *link)
 889 {
 890         complete(&link->llc_testlink_resp);
 891         cancel_delayed_work_sync(&link->llc_testlink_wrk);
 892         smc_wr_wakeup_reg_wait(link);
 893         smc_wr_wakeup_tx_wait(link);
 894 }
 895
 896 /* register a new rtoken at the remote peer (for all links) */
 897 int smc_llc_do_confirm_rkey(struct smc_link *send_link,
 898                             struct smc_buf_desc *rmb_desc)
 899 {
 900         struct smc_link_group *lgr = send_link->lgr;
 901         struct smc_llc_qentry *qentry = NULL;
 902         int rc = 0;
 903
 904         rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
 905         if (rc)
 906                 return rc;
 907         rc = smc_llc_send_confirm_rkey(send_link, rmb_desc);
 908         if (rc)
 909                 goto out;
 910         /* receive CONFIRM RKEY response from server over RoCE fabric */
 911         qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
 912                               SMC_LLC_CONFIRM_RKEY);
 913         if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
 914                 rc = -EFAULT;
 915 out:
 916         if (qentry)
 917                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 918         smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
 919         return rc;
 920 }
 921
 922 /* unregister an rtoken at the remote peer */
 923 int smc_llc_do_delete_rkey(struct smc_link_group *lgr,
 924                            struct smc_buf_desc *rmb_desc)
 925 {
 926         struct smc_llc_qentry *qentry = NULL;
 927         struct smc_link *send_link;
 928         int rc = 0;
 929
 930         send_link = smc_llc_usable_link(lgr);
 931         if (!send_link)
 932                 return -ENOLINK;
 933
 934         rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
 935         if (rc)
 936                 return rc;
 937         /* protected by llc_flow control */
 938         rc = smc_llc_send_delete_rkey(send_link, rmb_desc);
 939         if (rc)
 940                 goto out;
 941         /* receive DELETE RKEY response from server over RoCE fabric */
 942         qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
 943                               SMC_LLC_DELETE_RKEY);
 944         if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
 945                 rc = -EFAULT;
 946 out:
 947         if (qentry)
 948                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 949         smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
 950         return rc;
 951 }
 952
 953 /* evaluate confirm link request or response */
 954 int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry,
 955                            enum smc_llc_reqresp type)
 956 {
 957         if (type == SMC_LLC_REQ)        /* SMC server assigns link_id */
 958                 qentry->link->link_id = qentry->msg.confirm_link.link_num;
 959         if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
 960                 return -ENOTSUPP;
 961         return 0;
 962 }
 963
 964 /***************************** init, exit, misc ******************************/
 965
 966 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
 967         {
 968                 .handler        = smc_llc_rx_handler,
 969                 .type           = SMC_LLC_CONFIRM_LINK
 970         },
 971         {
 972                 .handler        = smc_llc_rx_handler,
 973                 .type           = SMC_LLC_TEST_LINK
 974         },
 975         {
 976                 .handler        = smc_llc_rx_handler,
 977                 .type           = SMC_LLC_ADD_LINK
 978         },
 979         {
 980                 .handler        = smc_llc_rx_handler,
 981                 .type           = SMC_LLC_DELETE_LINK
 982         },
 983         {
 984                 .handler        = smc_llc_rx_handler,
 985                 .type           = SMC_LLC_CONFIRM_RKEY
 986         },
 987         {
 988                 .handler        = smc_llc_rx_handler,
 989                 .type           = SMC_LLC_CONFIRM_RKEY_CONT
 990         },
 991         {
 992                 .handler        = smc_llc_rx_handler,
 993                 .type           = SMC_LLC_DELETE_RKEY
 994         },
 995         {
 996                 .handler        = NULL,
 997         }
 998 };
 999
1000 int __init smc_llc_init(void)
1001 {
1002         struct smc_wr_rx_handler *handler;
1003         int rc = 0;
1004
1005         for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
1006                 INIT_HLIST_NODE(&handler->list);
1007                 rc = smc_wr_rx_register_handler(handler);
1008                 if (rc)
1009                         break;
1010         }
1011         return rc;
1012 }