net/smc/smc_llc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  Link Layer Control (LLC)
   6  *
   7  *  Copyright IBM Corp. 2016
   8  *
   9  *  Author(s):  Klaus Wacker <Klaus.Wacker@de.ibm.com>
  10  *              Ursula Braun <ubraun@linux.vnet.ibm.com>
  11  */
  12
  13 #include <net/tcp.h>
  14 #include <rdma/ib_verbs.h>
  15
  16 #include "smc.h"
  17 #include "smc_core.h"
  18 #include "smc_clc.h"
  19 #include "smc_llc.h"
  20
  21 #define SMC_LLC_DATA_LEN                40
  22
  23 struct smc_llc_hdr {
  24         struct smc_wr_rx_hdr common;
  25         u8 length;      /* 44 */
  26 #if defined(__BIG_ENDIAN_BITFIELD)
  27         u8 reserved:4,
  28            add_link_rej_rsn:4;
  29 #elif defined(__LITTLE_ENDIAN_BITFIELD)
  30         u8 add_link_rej_rsn:4,
  31            reserved:4;
  32 #endif
  33         u8 flags;
  34 };
  35
  36 #define SMC_LLC_FLAG_NO_RMBE_EYEC       0x03
  37
  38 struct smc_llc_msg_confirm_link {       /* type 0x01 */
  39         struct smc_llc_hdr hd;
  40         u8 sender_mac[ETH_ALEN];
  41         u8 sender_gid[SMC_GID_SIZE];
  42         u8 sender_qp_num[3];
  43         u8 link_num;
  44         u8 link_uid[SMC_LGR_ID_SIZE];
  45         u8 max_links;
  46         u8 reserved[9];
  47 };
  48
  49 #define SMC_LLC_FLAG_ADD_LNK_REJ        0x40
  50 #define SMC_LLC_REJ_RSN_NO_ALT_PATH     1
  51
  52 #define SMC_LLC_ADD_LNK_MAX_LINKS       2
  53
  54 struct smc_llc_msg_add_link {           /* type 0x02 */
  55         struct smc_llc_hdr hd;
  56         u8 sender_mac[ETH_ALEN];
  57         u8 reserved2[2];
  58         u8 sender_gid[SMC_GID_SIZE];
  59         u8 sender_qp_num[3];
  60         u8 link_num;
  61         u8 flags2;      /* QP mtu */
  62         u8 initial_psn[3];
  63         u8 reserved[8];
  64 };
  65
  66 #define SMC_LLC_FLAG_DEL_LINK_ALL       0x40
  67 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY   0x20
  68
  69 struct smc_llc_msg_del_link {           /* type 0x04 */
  70         struct smc_llc_hdr hd;
  71         u8 link_num;
  72         __be32 reason;
  73         u8 reserved[35];
  74 } __packed;                     /* format defined in RFC7609 */
  75
  76 struct smc_llc_msg_test_link {          /* type 0x07 */
  77         struct smc_llc_hdr hd;
  78         u8 user_data[16];
  79         u8 reserved[24];
  80 };
  81
  82 struct smc_rmb_rtoken {
  83         union {
  84                 u8 num_rkeys;   /* first rtoken byte of CONFIRM LINK msg */
  85                                 /* is actually the num of rtokens, first */
  86                                 /* rtoken is always for the current link */
  87                 u8 link_id;     /* link id of the rtoken */
  88         };
  89         __be32 rmb_key;
  90         __be64 rmb_vaddr;
  91 } __packed;                     /* format defined in RFC7609 */
  92
  93 #define SMC_LLC_RKEYS_PER_MSG   3
  94
  95 struct smc_llc_msg_confirm_rkey {       /* type 0x06 */
  96         struct smc_llc_hdr hd;
  97         struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
  98         u8 reserved;
  99 };
 100
 101 struct smc_llc_msg_confirm_rkey_cont {  /* type 0x08 */
 102         struct smc_llc_hdr hd;
 103         u8 num_rkeys;
 104         struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
 105 };
 106
 107 #define SMC_LLC_DEL_RKEY_MAX    8
 108 #define SMC_LLC_FLAG_RKEY_NEG   0x20
 109
 110 struct smc_llc_msg_delete_rkey {        /* type 0x09 */
 111         struct smc_llc_hdr hd;
 112         u8 num_rkeys;
 113         u8 err_mask;
 114         u8 reserved[2];
 115         __be32 rkey[8];
 116         u8 reserved2[4];
 117 };
 118
 119 union smc_llc_msg {
 120         struct smc_llc_msg_confirm_link confirm_link;
 121         struct smc_llc_msg_add_link add_link;
 122         struct smc_llc_msg_del_link delete_link;
 123
 124         struct smc_llc_msg_confirm_rkey confirm_rkey;
 125         struct smc_llc_msg_confirm_rkey_cont confirm_rkey_cont;
 126         struct smc_llc_msg_delete_rkey delete_rkey;
 127
 128         struct smc_llc_msg_test_link test_link;
 129         struct {
 130                 struct smc_llc_hdr hdr;
 131                 u8 data[SMC_LLC_DATA_LEN];
 132         } raw;
 133 };
 134
 135 #define SMC_LLC_FLAG_RESP               0x80
 136
 137 struct smc_llc_qentry {
 138         struct list_head list;
 139         struct smc_link *link;
 140         union smc_llc_msg msg;
 141 };
 142
 143 struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow)
 144 {
 145         struct smc_llc_qentry *qentry = flow->qentry;
 146
 147         flow->qentry = NULL;
 148         return qentry;
 149 }
 150
 151 void smc_llc_flow_qentry_del(struct smc_llc_flow *flow)
 152 {
 153         struct smc_llc_qentry *qentry;
 154
 155         if (flow->qentry) {
 156                 qentry = flow->qentry;
 157                 flow->qentry = NULL;
 158                 kfree(qentry);
 159         }
 160 }
 161
 162 static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow,
 163                                            struct smc_llc_qentry *qentry)
 164 {
 165         flow->qentry = qentry;
 166 }
 167
 168 /* try to start a new llc flow, initiated by an incoming llc msg */
 169 static bool smc_llc_flow_start(struct smc_llc_flow *flow,
 170                                struct smc_llc_qentry *qentry)
 171 {
 172         struct smc_link_group *lgr = qentry->link->lgr;
 173
 174         spin_lock_bh(&lgr->llc_flow_lock);
 175         if (flow->type) {
 176                 /* a flow is already active */
 177                 if ((qentry->msg.raw.hdr.common.type == SMC_LLC_ADD_LINK ||
 178                      qentry->msg.raw.hdr.common.type == SMC_LLC_DELETE_LINK) &&
 179                     !lgr->delayed_event) {
 180                         lgr->delayed_event = qentry;
 181                 } else {
 182                         /* forget this llc request */
 183                         kfree(qentry);
 184                 }
 185                 spin_unlock_bh(&lgr->llc_flow_lock);
 186                 return false;
 187         }
 188         switch (qentry->msg.raw.hdr.common.type) {
 189         case SMC_LLC_ADD_LINK:
 190                 flow->type = SMC_LLC_FLOW_ADD_LINK;
 191                 break;
 192         case SMC_LLC_DELETE_LINK:
 193                 flow->type = SMC_LLC_FLOW_DEL_LINK;
 194                 break;
 195         case SMC_LLC_CONFIRM_RKEY:
 196         case SMC_LLC_DELETE_RKEY:
 197                 flow->type = SMC_LLC_FLOW_RKEY;
 198                 break;
 199         default:
 200                 flow->type = SMC_LLC_FLOW_NONE;
 201         }
 202         if (qentry == lgr->delayed_event)
 203                 lgr->delayed_event = NULL;
 204         spin_unlock_bh(&lgr->llc_flow_lock);
 205         smc_llc_flow_qentry_set(flow, qentry);
 206         return true;
 207 }
 208
 209 /* start a new local llc flow, wait till current flow finished */
 210 int smc_llc_flow_initiate(struct smc_link_group *lgr,
 211                           enum smc_llc_flowtype type)
 212 {
 213         enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE;
 214         int rc;
 215
 216         /* all flows except confirm_rkey and delete_rkey are exclusive,
 217          * confirm/delete rkey flows can run concurrently (local and remote)
 218          */
 219         if (type == SMC_LLC_FLOW_RKEY)
 220                 allowed_remote = SMC_LLC_FLOW_RKEY;
 221 again:
 222         if (list_empty(&lgr->list))
 223                 return -ENODEV;
 224         spin_lock_bh(&lgr->llc_flow_lock);
 225         if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
 226             (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
 227              lgr->llc_flow_rmt.type == allowed_remote)) {
 228                 lgr->llc_flow_lcl.type = type;
 229                 spin_unlock_bh(&lgr->llc_flow_lock);
 230                 return 0;
 231         }
 232         spin_unlock_bh(&lgr->llc_flow_lock);
 233         rc = wait_event_interruptible_timeout(lgr->llc_waiter,
 234                         (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
 235                          (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
 236                           lgr->llc_flow_rmt.type == allowed_remote)),
 237                         SMC_LLC_WAIT_TIME);
 238         if (!rc)
 239                 return -ETIMEDOUT;
 240         goto again;
 241 }
 242
 243 /* finish the current llc flow */
 244 void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)
 245 {
 246         spin_lock_bh(&lgr->llc_flow_lock);
 247         memset(flow, 0, sizeof(*flow));
 248         flow->type = SMC_LLC_FLOW_NONE;
 249         spin_unlock_bh(&lgr->llc_flow_lock);
 250         if (!list_empty(&lgr->list) && lgr->delayed_event &&
 251             flow == &lgr->llc_flow_lcl)
 252                 schedule_work(&lgr->llc_event_work);
 253         else
 254                 wake_up_interruptible(&lgr->llc_waiter);
 255 }
 256
 257 /* lnk is optional and used for early wakeup when link goes down, useful in
 258  * cases where we wait for a response on the link after we sent a request
 259  */
 260 struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
 261                                     struct smc_link *lnk,
 262                                     int time_out, u8 exp_msg)
 263 {
 264         struct smc_llc_flow *flow = &lgr->llc_flow_lcl;
 265
 266         wait_event_interruptible_timeout(lgr->llc_waiter,
 267                                          (flow->qentry ||
 268                                           (lnk && !smc_link_usable(lnk)) ||
 269                                           list_empty(&lgr->list)),
 270                                          time_out);
 271         if (!flow->qentry ||
 272             (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) {
 273                 smc_llc_flow_qentry_del(flow);
 274                 goto out;
 275         }
 276         if (exp_msg && flow->qentry->msg.raw.hdr.common.type != exp_msg) {
 277                 if (exp_msg == SMC_LLC_ADD_LINK &&
 278                     flow->qentry->msg.raw.hdr.common.type ==
 279                     SMC_LLC_DELETE_LINK) {
 280                         /* flow_start will delay the unexpected msg */
 281                         smc_llc_flow_start(&lgr->llc_flow_lcl,
 282                                            smc_llc_flow_qentry_clr(flow));
 283                         return NULL;
 284                 }
 285                 smc_llc_flow_qentry_del(flow);
 286         }
 287 out:
 288         return flow->qentry;
 289 }
 290
 291 /********************************** send *************************************/
 292
 293 struct smc_llc_tx_pend {
 294 };
 295
 296 /* handler for send/transmission completion of an LLC msg */
 297 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
 298                                struct smc_link *link,
 299                                enum ib_wc_status wc_status)
 300 {
 301         /* future work: handle wc_status error for recovery and failover */
 302 }
 303
 304 /**
 305  * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
 306  * @link: Pointer to SMC link used for sending LLC control message.
 307  * @wr_buf: Out variable returning pointer to work request payload buffer.
 308  * @pend: Out variable returning pointer to private pending WR tracking.
 309  *        It's the context the transmit complete handler will get.
 310  *
 311  * Reserves and pre-fills an entry for a pending work request send/tx.
 312  * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
 313  * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
 314  *
 315  * Return: 0 on success, otherwise an error value.
 316  */
 317 static int smc_llc_add_pending_send(struct smc_link *link,
 318                                     struct smc_wr_buf **wr_buf,
 319                                     struct smc_wr_tx_pend_priv **pend)
 320 {
 321         int rc;
 322
 323         rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
 324                                      pend);
 325         if (rc < 0)
 326                 return rc;
 327         BUILD_BUG_ON_MSG(
 328                 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
 329                 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
 330         BUILD_BUG_ON_MSG(
 331                 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
 332                 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
 333         BUILD_BUG_ON_MSG(
 334                 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
 335                 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
 336         return 0;
 337 }
 338
 339 /* high-level API to send LLC confirm link */
 340 int smc_llc_send_confirm_link(struct smc_link *link,
 341                               enum smc_llc_reqresp reqresp)
 342 {
 343         struct smc_link_group *lgr = smc_get_lgr(link);
 344         struct smc_llc_msg_confirm_link *confllc;
 345         struct smc_wr_tx_pend_priv *pend;
 346         struct smc_wr_buf *wr_buf;
 347         int rc;
 348
 349         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 350         if (rc)
 351                 return rc;
 352         confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
 353         memset(confllc, 0, sizeof(*confllc));
 354         confllc->hd.common.type = SMC_LLC_CONFIRM_LINK;
 355         confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link);
 356         confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
 357         if (reqresp == SMC_LLC_RESP)
 358                 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
 359         memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
 360                ETH_ALEN);
 361         memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
 362         hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
 363         confllc->link_num = link->link_id;
 364         memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE);
 365         confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */
 366         /* send llc message */
 367         rc = smc_wr_tx_send(link, pend);
 368         return rc;
 369 }
 370
 371 /* send LLC confirm rkey request */
 372 static int smc_llc_send_confirm_rkey(struct smc_link *link,
 373                                      struct smc_buf_desc *rmb_desc)
 374 {
 375         struct smc_llc_msg_confirm_rkey *rkeyllc;
 376         struct smc_wr_tx_pend_priv *pend;
 377         struct smc_wr_buf *wr_buf;
 378         int rc;
 379
 380         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 381         if (rc)
 382                 return rc;
 383         rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
 384         memset(rkeyllc, 0, sizeof(*rkeyllc));
 385         rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY;
 386         rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey);
 387         rkeyllc->rtoken[0].rmb_key =
 388                 htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
 389         rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64(
 390                 (u64)sg_dma_address(rmb_desc->sgt[link->link_idx].sgl));
 391         /* send llc message */
 392         rc = smc_wr_tx_send(link, pend);
 393         return rc;
 394 }
 395
 396 /* send LLC delete rkey request */
 397 static int smc_llc_send_delete_rkey(struct smc_link *link,
 398                                     struct smc_buf_desc *rmb_desc)
 399 {
 400         struct smc_llc_msg_delete_rkey *rkeyllc;
 401         struct smc_wr_tx_pend_priv *pend;
 402         struct smc_wr_buf *wr_buf;
 403         int rc;
 404
 405         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 406         if (rc)
 407                 return rc;
 408         rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
 409         memset(rkeyllc, 0, sizeof(*rkeyllc));
 410         rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY;
 411         rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey);
 412         rkeyllc->num_rkeys = 1;
 413         rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
 414         /* send llc message */
 415         rc = smc_wr_tx_send(link, pend);
 416         return rc;
 417 }
 418
 419 /* prepare an add link message */
 420 static void smc_llc_prep_add_link(struct smc_llc_msg_add_link *addllc,
 421                                   struct smc_link *link, u8 mac[], u8 gid[],
 422                                   enum smc_llc_reqresp reqresp)
 423 {
 424         memset(addllc, 0, sizeof(*addllc));
 425         addllc->hd.common.type = SMC_LLC_ADD_LINK;
 426         addllc->hd.length = sizeof(struct smc_llc_msg_add_link);
 427         if (reqresp == SMC_LLC_RESP) {
 428                 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
 429                 /* always reject more links for now */
 430                 addllc->hd.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
 431                 addllc->hd.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
 432         }
 433         memcpy(addllc->sender_mac, mac, ETH_ALEN);
 434         memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
 435 }
 436
 437 /* send ADD LINK request or response */
 438 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
 439                           enum smc_llc_reqresp reqresp)
 440 {
 441         struct smc_llc_msg_add_link *addllc;
 442         struct smc_wr_tx_pend_priv *pend;
 443         struct smc_wr_buf *wr_buf;
 444         int rc;
 445
 446         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 447         if (rc)
 448                 return rc;
 449         addllc = (struct smc_llc_msg_add_link *)wr_buf;
 450         smc_llc_prep_add_link(addllc, link, mac, gid, reqresp);
 451         /* send llc message */
 452         rc = smc_wr_tx_send(link, pend);
 453         return rc;
 454 }
 455
 456 /* prepare a delete link message */
 457 static void smc_llc_prep_delete_link(struct smc_llc_msg_del_link *delllc,
 458                                      struct smc_link *link,
 459                                      enum smc_llc_reqresp reqresp, bool orderly)
 460 {
 461         memset(delllc, 0, sizeof(*delllc));
 462         delllc->hd.common.type = SMC_LLC_DELETE_LINK;
 463         delllc->hd.length = sizeof(struct smc_llc_msg_add_link);
 464         if (reqresp == SMC_LLC_RESP)
 465                 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
 466         /* DEL_LINK_ALL because only 1 link supported */
 467         delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
 468         if (orderly)
 469                 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
 470         delllc->link_num = link->link_id;
 471 }
 472
 473 /* send DELETE LINK request or response */
 474 int smc_llc_send_delete_link(struct smc_link *link,
 475                              enum smc_llc_reqresp reqresp, bool orderly)
 476 {
 477         struct smc_llc_msg_del_link *delllc;
 478         struct smc_wr_tx_pend_priv *pend;
 479         struct smc_wr_buf *wr_buf;
 480         int rc;
 481
 482         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 483         if (rc)
 484                 return rc;
 485         delllc = (struct smc_llc_msg_del_link *)wr_buf;
 486         smc_llc_prep_delete_link(delllc, link, reqresp, orderly);
 487         /* send llc message */
 488         rc = smc_wr_tx_send(link, pend);
 489         return rc;
 490 }
 491
 492 /* send LLC test link request */
 493 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
 494 {
 495         struct smc_llc_msg_test_link *testllc;
 496         struct smc_wr_tx_pend_priv *pend;
 497         struct smc_wr_buf *wr_buf;
 498         int rc;
 499
 500         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 501         if (rc)
 502                 return rc;
 503         testllc = (struct smc_llc_msg_test_link *)wr_buf;
 504         memset(testllc, 0, sizeof(*testllc));
 505         testllc->hd.common.type = SMC_LLC_TEST_LINK;
 506         testllc->hd.length = sizeof(struct smc_llc_msg_test_link);
 507         memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
 508         /* send llc message */
 509         rc = smc_wr_tx_send(link, pend);
 510         return rc;
 511 }
 512
 513 /* schedule an llc send on link, may wait for buffers */
 514 static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
 515 {
 516         struct smc_wr_tx_pend_priv *pend;
 517         struct smc_wr_buf *wr_buf;
 518         int rc;
 519
 520         if (!smc_link_usable(link))
 521                 return -ENOLINK;
 522         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 523         if (rc)
 524                 return rc;
 525         memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
 526         return smc_wr_tx_send(link, pend);
 527 }
 528
 529 /********************************* receive ***********************************/
 530
 531 static void smc_llc_rx_confirm_link(struct smc_link *link,
 532                                     struct smc_llc_msg_confirm_link *llc)
 533 {
 534         struct smc_link_group *lgr = smc_get_lgr(link);
 535         int conf_rc = 0;
 536
 537         /* RMBE eyecatchers are not supported */
 538         if (!(llc->hd.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
 539                 conf_rc = ENOTSUPP;
 540
 541         if (lgr->role == SMC_CLNT &&
 542             link->state == SMC_LNK_ACTIVATING) {
 543                 link->llc_confirm_rc = conf_rc;
 544                 link->link_id = llc->link_num;
 545                 complete(&link->llc_confirm);
 546         }
 547 }
 548
 549 static void smc_llc_rx_add_link(struct smc_link *link,
 550                                 struct smc_llc_msg_add_link *llc)
 551 {
 552         struct smc_link_group *lgr = smc_get_lgr(link);
 553
 554         if (link->state == SMC_LNK_ACTIVATING) {
 555                 complete(&link->llc_add);
 556                 return;
 557         }
 558
 559         if (lgr->role == SMC_SERV) {
 560                 smc_llc_prep_add_link(llc, link,
 561                                 link->smcibdev->mac[link->ibport - 1],
 562                                 link->gid, SMC_LLC_REQ);
 563
 564         } else {
 565                 smc_llc_prep_add_link(llc, link,
 566                                 link->smcibdev->mac[link->ibport - 1],
 567                                 link->gid, SMC_LLC_RESP);
 568         }
 569         smc_llc_send_message(link, llc);
 570 }
 571
 572 static void smc_llc_rx_delete_link(struct smc_link *link,
 573                                    struct smc_llc_msg_del_link *llc)
 574 {
 575         struct smc_link_group *lgr = smc_get_lgr(link);
 576
 577         smc_lgr_forget(lgr);
 578         smc_llc_link_deleting(link);
 579         if (lgr->role == SMC_SERV) {
 580                 /* client asks to delete this link, send request */
 581                 smc_llc_prep_delete_link(llc, link, SMC_LLC_REQ, true);
 582         } else {
 583                 /* server requests to delete this link, send response */
 584                 smc_llc_prep_delete_link(llc, link, SMC_LLC_RESP, true);
 585         }
 586         smc_llc_send_message(link, llc);
 587         smc_lgr_terminate_sched(lgr);
 588 }
 589
 590 static void smc_llc_rx_test_link(struct smc_link *link,
 591                                  struct smc_llc_msg_test_link *llc)
 592 {
 593         llc->hd.flags |= SMC_LLC_FLAG_RESP;
 594         smc_llc_send_message(link, llc);
 595 }
 596
 597 static void smc_llc_rx_confirm_rkey(struct smc_link *link,
 598                                     struct smc_llc_msg_confirm_rkey *llc)
 599 {
 600         int rc;
 601
 602         rc = smc_rtoken_add(link,
 603                             llc->rtoken[0].rmb_vaddr,
 604                             llc->rtoken[0].rmb_key);
 605
 606         /* ignore rtokens for other links, we have only one link */
 607
 608         llc->hd.flags |= SMC_LLC_FLAG_RESP;
 609         if (rc < 0)
 610                 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
 611         smc_llc_send_message(link, llc);
 612 }
 613
 614 static void smc_llc_rx_confirm_rkey_cont(struct smc_link *link,
 615                                       struct smc_llc_msg_confirm_rkey_cont *llc)
 616 {
 617         /* ignore rtokens for other links, we have only one link */
 618         llc->hd.flags |= SMC_LLC_FLAG_RESP;
 619         smc_llc_send_message(link, llc);
 620 }
 621
 622 static void smc_llc_rx_delete_rkey(struct smc_link *link,
 623                                    struct smc_llc_msg_delete_rkey *llc)
 624 {
 625         u8 err_mask = 0;
 626         int i, max;
 627
 628         max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
 629         for (i = 0; i < max; i++) {
 630                 if (smc_rtoken_delete(link, llc->rkey[i]))
 631                         err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
 632         }
 633
 634         if (err_mask) {
 635                 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
 636                 llc->err_mask = err_mask;
 637         }
 638
 639         llc->hd.flags |= SMC_LLC_FLAG_RESP;
 640         smc_llc_send_message(link, llc);
 641 }
 642
 643 /* flush the llc event queue */
 644 static void smc_llc_event_flush(struct smc_link_group *lgr)
 645 {
 646         struct smc_llc_qentry *qentry, *q;
 647
 648         spin_lock_bh(&lgr->llc_event_q_lock);
 649         list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
 650                 list_del_init(&qentry->list);
 651                 kfree(qentry);
 652         }
 653         spin_unlock_bh(&lgr->llc_event_q_lock);
 654 }
 655
 656 static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
 657 {
 658         union smc_llc_msg *llc = &qentry->msg;
 659         struct smc_link *link = qentry->link;
 660
 661         if (!smc_link_usable(link))
 662                 goto out;
 663
 664         switch (llc->raw.hdr.common.type) {
 665         case SMC_LLC_TEST_LINK:
 666                 smc_llc_rx_test_link(link, &llc->test_link);
 667                 break;
 668         case SMC_LLC_CONFIRM_LINK:
 669                 smc_llc_rx_confirm_link(link, &llc->confirm_link);
 670                 break;
 671         case SMC_LLC_ADD_LINK:
 672                 smc_llc_rx_add_link(link, &llc->add_link);
 673                 break;
 674         case SMC_LLC_DELETE_LINK:
 675                 smc_llc_rx_delete_link(link, &llc->delete_link);
 676                 break;
 677         case SMC_LLC_CONFIRM_RKEY:
 678                 smc_llc_rx_confirm_rkey(link, &llc->confirm_rkey);
 679                 break;
 680         case SMC_LLC_CONFIRM_RKEY_CONT:
 681                 smc_llc_rx_confirm_rkey_cont(link, &llc->confirm_rkey_cont);
 682                 break;
 683         case SMC_LLC_DELETE_RKEY:
 684                 smc_llc_rx_delete_rkey(link, &llc->delete_rkey);
 685                 break;
 686         }
 687 out:
 688         kfree(qentry);
 689 }
 690
 691 /* worker to process llc messages on the event queue */
 692 static void smc_llc_event_work(struct work_struct *work)
 693 {
 694         struct smc_link_group *lgr = container_of(work, struct smc_link_group,
 695                                                   llc_event_work);
 696         struct smc_llc_qentry *qentry;
 697
 698         if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {
 699                 if (smc_link_usable(lgr->delayed_event->link)) {
 700                         smc_llc_event_handler(lgr->delayed_event);
 701                 } else {
 702                         qentry = lgr->delayed_event;
 703                         lgr->delayed_event = NULL;
 704                         kfree(qentry);
 705                 }
 706         }
 707
 708 again:
 709         spin_lock_bh(&lgr->llc_event_q_lock);
 710         if (!list_empty(&lgr->llc_event_q)) {
 711                 qentry = list_first_entry(&lgr->llc_event_q,
 712                                           struct smc_llc_qentry, list);
 713                 list_del_init(&qentry->list);
 714                 spin_unlock_bh(&lgr->llc_event_q_lock);
 715                 smc_llc_event_handler(qentry);
 716                 goto again;
 717         }
 718         spin_unlock_bh(&lgr->llc_event_q_lock);
 719 }
 720
 721 /* process llc responses in tasklet context */
 722 static void smc_llc_rx_response(struct smc_link *link,
 723                                 struct smc_llc_qentry *qentry)
 724 {
 725         u8 llc_type = qentry->msg.raw.hdr.common.type;
 726         union smc_llc_msg *llc = &qentry->msg;
 727         int rc = 0;
 728
 729         switch (llc_type) {
 730         case SMC_LLC_TEST_LINK:
 731                 if (link->state == SMC_LNK_ACTIVE)
 732                         complete(&link->llc_testlink_resp);
 733                 break;
 734         case SMC_LLC_CONFIRM_LINK:
 735                 if (!(llc->raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
 736                         rc = ENOTSUPP;
 737                 if (link->lgr->role == SMC_SERV &&
 738                     link->state == SMC_LNK_ACTIVATING) {
 739                         link->llc_confirm_resp_rc = rc;
 740                         complete(&link->llc_confirm_resp);
 741                 }
 742                 break;
 743         case SMC_LLC_ADD_LINK:
 744                 if (link->state == SMC_LNK_ACTIVATING)
 745                         complete(&link->llc_add_resp);
 746                 break;
 747         case SMC_LLC_DELETE_LINK:
 748                 if (link->lgr->role == SMC_SERV)
 749                         smc_lgr_schedule_free_work_fast(link->lgr);
 750                 break;
 751         case SMC_LLC_CONFIRM_RKEY:
 752                 link->llc_confirm_rkey_resp_rc = llc->raw.hdr.flags &
 753                                                  SMC_LLC_FLAG_RKEY_NEG;
 754                 complete(&link->llc_confirm_rkey_resp);
 755                 break;
 756         case SMC_LLC_CONFIRM_RKEY_CONT:
 757                 /* unused as long as we don't send this type of msg */
 758                 break;
 759         case SMC_LLC_DELETE_RKEY:
 760                 link->llc_delete_rkey_resp_rc = llc->raw.hdr.flags &
 761                                                 SMC_LLC_FLAG_RKEY_NEG;
 762                 complete(&link->llc_delete_rkey_resp);
 763                 break;
 764         }
 765         kfree(qentry);
 766 }
 767
 768 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc)
 769 {
 770         struct smc_link_group *lgr = link->lgr;
 771         struct smc_llc_qentry *qentry;
 772         unsigned long flags;
 773
 774         qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC);
 775         if (!qentry)
 776                 return;
 777         qentry->link = link;
 778         INIT_LIST_HEAD(&qentry->list);
 779         memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
 780
 781         /* process responses immediately */
 782         if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) {
 783                 smc_llc_rx_response(link, qentry);
 784                 return;
 785         }
 786
 787         /* add requests to event queue */
 788         spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
 789         list_add_tail(&qentry->list, &lgr->llc_event_q);
 790         spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);
 791         schedule_work(&link->lgr->llc_event_work);
 792 }
 793
 794 /* copy received msg and add it to the event queue */
 795 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
 796 {
 797         struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
 798         union smc_llc_msg *llc = buf;
 799
 800         if (wc->byte_len < sizeof(*llc))
 801                 return; /* short message */
 802         if (llc->raw.hdr.length != sizeof(*llc))
 803                 return; /* invalid message */
 804
 805         smc_llc_enqueue(link, llc);
 806 }
 807
 808 /***************************** worker, utils *********************************/
 809
 810 static void smc_llc_testlink_work(struct work_struct *work)
 811 {
 812         struct smc_link *link = container_of(to_delayed_work(work),
 813                                              struct smc_link, llc_testlink_wrk);
 814         unsigned long next_interval;
 815         unsigned long expire_time;
 816         u8 user_data[16] = { 0 };
 817         int rc;
 818
 819         if (link->state != SMC_LNK_ACTIVE)
 820                 return;         /* don't reschedule worker */
 821         expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
 822         if (time_is_after_jiffies(expire_time)) {
 823                 next_interval = expire_time - jiffies;
 824                 goto out;
 825         }
 826         reinit_completion(&link->llc_testlink_resp);
 827         smc_llc_send_test_link(link, user_data);
 828         /* receive TEST LINK response over RoCE fabric */
 829         rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
 830                                                        SMC_LLC_WAIT_TIME);
 831         if (link->state != SMC_LNK_ACTIVE)
 832                 return;         /* link state changed */
 833         if (rc <= 0) {
 834                 smc_lgr_terminate_sched(smc_get_lgr(link));
 835                 return;
 836         }
 837         next_interval = link->llc_testlink_time;
 838 out:
 839         schedule_delayed_work(&link->llc_testlink_wrk, next_interval);
 840 }
 841
 842 void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
 843 {
 844         struct net *net = sock_net(smc->clcsock->sk);
 845
 846         INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);
 847         INIT_LIST_HEAD(&lgr->llc_event_q);
 848         spin_lock_init(&lgr->llc_event_q_lock);
 849         spin_lock_init(&lgr->llc_flow_lock);
 850         init_waitqueue_head(&lgr->llc_waiter);
 851         lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time;
 852 }
 853
 854 /* called after lgr was removed from lgr_list */
 855 void smc_llc_lgr_clear(struct smc_link_group *lgr)
 856 {
 857         smc_llc_event_flush(lgr);
 858         wake_up_interruptible_all(&lgr->llc_waiter);
 859         cancel_work_sync(&lgr->llc_event_work);
 860         if (lgr->delayed_event) {
 861                 kfree(lgr->delayed_event);
 862                 lgr->delayed_event = NULL;
 863         }
 864 }
 865
 866 int smc_llc_link_init(struct smc_link *link)
 867 {
 868         init_completion(&link->llc_confirm);
 869         init_completion(&link->llc_confirm_resp);
 870         init_completion(&link->llc_add);
 871         init_completion(&link->llc_add_resp);
 872         init_completion(&link->llc_confirm_rkey_resp);
 873         init_completion(&link->llc_delete_rkey_resp);
 874         mutex_init(&link->llc_delete_rkey_mutex);
 875         init_completion(&link->llc_testlink_resp);
 876         INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
 877         return 0;
 878 }
 879
 880 void smc_llc_link_active(struct smc_link *link)
 881 {
 882         link->state = SMC_LNK_ACTIVE;
 883         if (link->lgr->llc_testlink_time) {
 884                 link->llc_testlink_time = link->lgr->llc_testlink_time * HZ;
 885                 schedule_delayed_work(&link->llc_testlink_wrk,
 886                                       link->llc_testlink_time);
 887         }
 888 }
 889
 890 void smc_llc_link_deleting(struct smc_link *link)
 891 {
 892         link->state = SMC_LNK_DELETING;
 893         smc_wr_wakeup_tx_wait(link);
 894 }
 895
 896 /* called in worker context */
 897 void smc_llc_link_clear(struct smc_link *link)
 898 {
 899         complete(&link->llc_testlink_resp);
 900         cancel_delayed_work_sync(&link->llc_testlink_wrk);
 901         smc_wr_wakeup_reg_wait(link);
 902         smc_wr_wakeup_tx_wait(link);
 903 }
 904
 905 /* register a new rtoken at the remote peer */
 906 int smc_llc_do_confirm_rkey(struct smc_link *link,
 907                             struct smc_buf_desc *rmb_desc)
 908 {
 909         int rc;
 910
 911         /* protected by mutex smc_create_lgr_pending */
 912         reinit_completion(&link->llc_confirm_rkey_resp);
 913         rc = smc_llc_send_confirm_rkey(link, rmb_desc);
 914         if (rc)
 915                 return rc;
 916         /* receive CONFIRM RKEY response from server over RoCE fabric */
 917         rc = wait_for_completion_interruptible_timeout(
 918                         &link->llc_confirm_rkey_resp, SMC_LLC_WAIT_TIME);
 919         if (rc <= 0 || link->llc_confirm_rkey_resp_rc)
 920                 return -EFAULT;
 921         return 0;
 922 }
 923
 924 /* unregister an rtoken at the remote peer */
 925 int smc_llc_do_delete_rkey(struct smc_link *link,
 926                            struct smc_buf_desc *rmb_desc)
 927 {
 928         int rc = 0;
 929
 930         mutex_lock(&link->llc_delete_rkey_mutex);
 931         if (link->state != SMC_LNK_ACTIVE)
 932                 goto out;
 933         reinit_completion(&link->llc_delete_rkey_resp);
 934         rc = smc_llc_send_delete_rkey(link, rmb_desc);
 935         if (rc)
 936                 goto out;
 937         /* receive DELETE RKEY response from server over RoCE fabric */
 938         rc = wait_for_completion_interruptible_timeout(
 939                         &link->llc_delete_rkey_resp, SMC_LLC_WAIT_TIME);
 940         if (rc <= 0 || link->llc_delete_rkey_resp_rc)
 941                 rc = -EFAULT;
 942         else
 943                 rc = 0;
 944 out:
 945         mutex_unlock(&link->llc_delete_rkey_mutex);
 946         return rc;
 947 }
 948
 949 /***************************** init, exit, misc ******************************/
 950
 951 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
 952         {
 953                 .handler        = smc_llc_rx_handler,
 954                 .type           = SMC_LLC_CONFIRM_LINK
 955         },
 956         {
 957                 .handler        = smc_llc_rx_handler,
 958                 .type           = SMC_LLC_TEST_LINK
 959         },
 960         {
 961                 .handler        = smc_llc_rx_handler,
 962                 .type           = SMC_LLC_ADD_LINK
 963         },
 964         {
 965                 .handler        = smc_llc_rx_handler,
 966                 .type           = SMC_LLC_DELETE_LINK
 967         },
 968         {
 969                 .handler        = smc_llc_rx_handler,
 970                 .type           = SMC_LLC_CONFIRM_RKEY
 971         },
 972         {
 973                 .handler        = smc_llc_rx_handler,
 974                 .type           = SMC_LLC_CONFIRM_RKEY_CONT
 975         },
 976         {
 977                 .handler        = smc_llc_rx_handler,
 978                 .type           = SMC_LLC_DELETE_RKEY
 979         },
 980         {
 981                 .handler        = NULL,
 982         }
 983 };
 984
 985 int __init smc_llc_init(void)
 986 {
 987         struct smc_wr_rx_handler *handler;
 988         int rc = 0;
 989
 990         for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
 991                 INIT_HLIST_NODE(&handler->list);
 992                 rc = smc_wr_rx_register_handler(handler);
 993                 if (rc)
 994                         break;
 995         }
 996         return rc;
 997 }