net/smc/af_smc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
   6  *  applies to SOCK_STREAM sockets only
   7  *  offers an alternative communication option for TCP-protocol sockets
   8  *  applicable with RoCE-cards only
   9  *
  10  *  Initial restrictions:
  11  *    - support for alternate links postponed
  12  *
  13  *  Copyright IBM Corp. 2016, 2018
  14  *
  15  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
  16  *              based on prototype from Frank Blaschka
  17  */
  18
  19 #define KMSG_COMPONENT "smc"
  20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  21
  22 #include <linux/module.h>
  23 #include <linux/socket.h>
  24 #include <linux/workqueue.h>
  25 #include <linux/in.h>
  26 #include <linux/sched/signal.h>
  27 #include <linux/if_vlan.h>
  28 #include <linux/rcupdate_wait.h>
  29 #include <linux/ctype.h>
  30
  31 #include <net/sock.h>
  32 #include <net/tcp.h>
  33 #include <net/smc.h>
  34 #include <asm/ioctls.h>
  35
  36 #include <net/net_namespace.h>
  37 #include <net/netns/generic.h>
  38 #include "smc_netns.h"
  39
  40 #include "smc.h"
  41 #include "smc_clc.h"
  42 #include "smc_llc.h"
  43 #include "smc_cdc.h"
  44 #include "smc_core.h"
  45 #include "smc_ib.h"
  46 #include "smc_ism.h"
  47 #include "smc_pnet.h"
  48 #include "smc_netlink.h"
  49 #include "smc_tx.h"
  50 #include "smc_rx.h"
  51 #include "smc_close.h"
  52 #include "smc_stats.h"
  53 #include "smc_tracepoint.h"
  54
  55 static DEFINE_MUTEX(smc_server_lgr_pending);    /* serialize link group
  56                                                  * creation on server
  57                                                  */
  58 static DEFINE_MUTEX(smc_client_lgr_pending);    /* serialize link group
  59                                                  * creation on client
  60                                                  */
  61
  62 struct workqueue_struct *smc_hs_wq;     /* wq for handshake work */
  63 struct workqueue_struct *smc_close_wq;  /* wq for close work */
  64
  65 static void smc_tcp_listen_work(struct work_struct *);
  66 static void smc_connect_work(struct work_struct *);
  67
  68 static void smc_set_keepalive(struct sock *sk, int val)
  69 {
  70         struct smc_sock *smc = smc_sk(sk);
  71
  72         smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
  73 }
  74
  75 static struct smc_hashinfo smc_v4_hashinfo = {
  76         .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
  77 };
  78
  79 static struct smc_hashinfo smc_v6_hashinfo = {
  80         .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
  81 };
  82
  83 int smc_hash_sk(struct sock *sk)
  84 {
  85         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  86         struct hlist_head *head;
  87
  88         head = &h->ht;
  89
  90         write_lock_bh(&h->lock);
  91         sk_add_node(sk, head);
  92         write_unlock_bh(&h->lock);
  93         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  94
  95         return 0;
  96 }
  97 EXPORT_SYMBOL_GPL(smc_hash_sk);
  98
  99 void smc_unhash_sk(struct sock *sk)
 100 {
 101         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
 102
 103         write_lock_bh(&h->lock);
 104         if (sk_del_node_init(sk))
 105                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
 106         write_unlock_bh(&h->lock);
 107 }
 108 EXPORT_SYMBOL_GPL(smc_unhash_sk);
 109
 110 struct proto smc_proto = {
 111         .name           = "SMC",
 112         .owner          = THIS_MODULE,
 113         .keepalive      = smc_set_keepalive,
 114         .hash           = smc_hash_sk,
 115         .unhash         = smc_unhash_sk,
 116         .obj_size       = sizeof(struct smc_sock),
 117         .h.smc_hash     = &smc_v4_hashinfo,
 118         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 119 };
 120 EXPORT_SYMBOL_GPL(smc_proto);
 121
 122 struct proto smc_proto6 = {
 123         .name           = "SMC6",
 124         .owner          = THIS_MODULE,
 125         .keepalive      = smc_set_keepalive,
 126         .hash           = smc_hash_sk,
 127         .unhash         = smc_unhash_sk,
 128         .obj_size       = sizeof(struct smc_sock),
 129         .h.smc_hash     = &smc_v6_hashinfo,
 130         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 131 };
 132 EXPORT_SYMBOL_GPL(smc_proto6);
 133
 134 static void smc_restore_fallback_changes(struct smc_sock *smc)
 135 {
 136         if (smc->clcsock->file) { /* non-accepted sockets have no file yet */
 137                 smc->clcsock->file->private_data = smc->sk.sk_socket;
 138                 smc->clcsock->file = NULL;
 139         }
 140 }
 141
 142 static int __smc_release(struct smc_sock *smc)
 143 {
 144         struct sock *sk = &smc->sk;
 145         int rc = 0;
 146
 147         if (!smc->use_fallback) {
 148                 rc = smc_close_active(smc);
 149                 sock_set_flag(sk, SOCK_DEAD);
 150                 sk->sk_shutdown |= SHUTDOWN_MASK;
 151         } else {
 152                 if (sk->sk_state != SMC_CLOSED) {
 153                         if (sk->sk_state != SMC_LISTEN &&
 154                             sk->sk_state != SMC_INIT)
 155                                 sock_put(sk); /* passive closing */
 156                         if (sk->sk_state == SMC_LISTEN) {
 157                                 /* wake up clcsock accept */
 158                                 rc = kernel_sock_shutdown(smc->clcsock,
 159                                                           SHUT_RDWR);
 160                         }
 161                         sk->sk_state = SMC_CLOSED;
 162                         sk->sk_state_change(sk);
 163                 }
 164                 smc_restore_fallback_changes(smc);
 165         }
 166
 167         sk->sk_prot->unhash(sk);
 168
 169         if (sk->sk_state == SMC_CLOSED) {
 170                 if (smc->clcsock) {
 171                         release_sock(sk);
 172                         smc_clcsock_release(smc);
 173                         lock_sock(sk);
 174                 }
 175                 if (!smc->use_fallback)
 176                         smc_conn_free(&smc->conn);
 177         }
 178
 179         return rc;
 180 }
 181
 182 static int smc_release(struct socket *sock)
 183 {
 184         struct sock *sk = sock->sk;
 185         struct smc_sock *smc;
 186         int rc = 0;
 187
 188         if (!sk)
 189                 goto out;
 190
 191         sock_hold(sk); /* sock_put below */
 192         smc = smc_sk(sk);
 193
 194         /* cleanup for a dangling non-blocking connect */
 195         if (smc->connect_nonblock && sk->sk_state == SMC_INIT)
 196                 tcp_abort(smc->clcsock->sk, ECONNABORTED);
 197
 198         if (cancel_work_sync(&smc->connect_work))
 199                 sock_put(&smc->sk); /* sock_hold in smc_connect for passive closing */
 200
 201         if (sk->sk_state == SMC_LISTEN)
 202                 /* smc_close_non_accepted() is called and acquires
 203                  * sock lock for child sockets again
 204                  */
 205                 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 206         else
 207                 lock_sock(sk);
 208
 209         rc = __smc_release(smc);
 210
 211         /* detach socket */
 212         sock_orphan(sk);
 213         sock->sk = NULL;
 214         release_sock(sk);
 215
 216         sock_put(sk); /* sock_hold above */
 217         sock_put(sk); /* final sock_put */
 218 out:
 219         return rc;
 220 }
 221
 222 static void smc_destruct(struct sock *sk)
 223 {
 224         if (sk->sk_state != SMC_CLOSED)
 225                 return;
 226         if (!sock_flag(sk, SOCK_DEAD))
 227                 return;
 228
 229         sk_refcnt_debug_dec(sk);
 230 }
 231
 232 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 233                                    int protocol)
 234 {
 235         struct smc_sock *smc;
 236         struct proto *prot;
 237         struct sock *sk;
 238
 239         prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
 240         sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
 241         if (!sk)
 242                 return NULL;
 243
 244         sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
 245         sk->sk_state = SMC_INIT;
 246         sk->sk_destruct = smc_destruct;
 247         sk->sk_protocol = protocol;
 248         smc = smc_sk(sk);
 249         INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 250         INIT_WORK(&smc->connect_work, smc_connect_work);
 251         INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
 252         INIT_LIST_HEAD(&smc->accept_q);
 253         spin_lock_init(&smc->accept_q_lock);
 254         spin_lock_init(&smc->conn.send_lock);
 255         sk->sk_prot->hash(sk);
 256         sk_refcnt_debug_inc(sk);
 257         mutex_init(&smc->clcsock_release_lock);
 258
 259         return sk;
 260 }
 261
 262 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 263                     int addr_len)
 264 {
 265         struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
 266         struct sock *sk = sock->sk;
 267         struct smc_sock *smc;
 268         int rc;
 269
 270         smc = smc_sk(sk);
 271
 272         /* replicate tests from inet_bind(), to be safe wrt. future changes */
 273         rc = -EINVAL;
 274         if (addr_len < sizeof(struct sockaddr_in))
 275                 goto out;
 276
 277         rc = -EAFNOSUPPORT;
 278         if (addr->sin_family != AF_INET &&
 279             addr->sin_family != AF_INET6 &&
 280             addr->sin_family != AF_UNSPEC)
 281                 goto out;
 282         /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
 283         if (addr->sin_family == AF_UNSPEC &&
 284             addr->sin_addr.s_addr != htonl(INADDR_ANY))
 285                 goto out;
 286
 287         lock_sock(sk);
 288
 289         /* Check if socket is already active */
 290         rc = -EINVAL;
 291         if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
 292                 goto out_rel;
 293
 294         smc->clcsock->sk->sk_reuse = sk->sk_reuse;
 295         rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 296
 297 out_rel:
 298         release_sock(sk);
 299 out:
 300         return rc;
 301 }
 302
 303 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
 304                                    unsigned long mask)
 305 {
 306         /* options we don't get control via setsockopt for */
 307         nsk->sk_type = osk->sk_type;
 308         nsk->sk_sndbuf = osk->sk_sndbuf;
 309         nsk->sk_rcvbuf = osk->sk_rcvbuf;
 310         nsk->sk_sndtimeo = osk->sk_sndtimeo;
 311         nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
 312         nsk->sk_mark = osk->sk_mark;
 313         nsk->sk_priority = osk->sk_priority;
 314         nsk->sk_rcvlowat = osk->sk_rcvlowat;
 315         nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
 316         nsk->sk_err = osk->sk_err;
 317
 318         nsk->sk_flags &= ~mask;
 319         nsk->sk_flags |= osk->sk_flags & mask;
 320 }
 321
 322 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
 323                              (1UL << SOCK_KEEPOPEN) | \
 324                              (1UL << SOCK_LINGER) | \
 325                              (1UL << SOCK_BROADCAST) | \
 326                              (1UL << SOCK_TIMESTAMP) | \
 327                              (1UL << SOCK_DBG) | \
 328                              (1UL << SOCK_RCVTSTAMP) | \
 329                              (1UL << SOCK_RCVTSTAMPNS) | \
 330                              (1UL << SOCK_LOCALROUTE) | \
 331                              (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
 332                              (1UL << SOCK_RXQ_OVFL) | \
 333                              (1UL << SOCK_WIFI_STATUS) | \
 334                              (1UL << SOCK_NOFCS) | \
 335                              (1UL << SOCK_FILTER_LOCKED) | \
 336                              (1UL << SOCK_TSTAMP_NEW))
 337 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
 338  * clc socket (since smc is not called for these options from net/core)
 339  */
 340 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
 341 {
 342         smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
 343 }
 344
 345 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
 346                              (1UL << SOCK_KEEPOPEN) | \
 347                              (1UL << SOCK_LINGER) | \
 348                              (1UL << SOCK_DBG))
 349 /* copy only settings and flags relevant for smc from clc to smc socket */
 350 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
 351 {
 352         smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 353 }
 354
 355 /* register the new rmb on all links */
 356 static int smcr_lgr_reg_rmbs(struct smc_link *link,
 357                              struct smc_buf_desc *rmb_desc)
 358 {
 359         struct smc_link_group *lgr = link->lgr;
 360         int i, rc = 0;
 361
 362         rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
 363         if (rc)
 364                 return rc;
 365         /* protect against parallel smc_llc_cli_rkey_exchange() and
 366          * parallel smcr_link_reg_rmb()
 367          */
 368         mutex_lock(&lgr->llc_conf_mutex);
 369         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 370                 if (!smc_link_active(&lgr->lnk[i]))
 371                         continue;
 372                 rc = smcr_link_reg_rmb(&lgr->lnk[i], rmb_desc);
 373                 if (rc)
 374                         goto out;
 375         }
 376
 377         /* exchange confirm_rkey msg with peer */
 378         rc = smc_llc_do_confirm_rkey(link, rmb_desc);
 379         if (rc) {
 380                 rc = -EFAULT;
 381                 goto out;
 382         }
 383         rmb_desc->is_conf_rkey = true;
 384 out:
 385         mutex_unlock(&lgr->llc_conf_mutex);
 386         smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
 387         return rc;
 388 }
 389
 390 static int smcr_clnt_conf_first_link(struct smc_sock *smc)
 391 {
 392         struct smc_link *link = smc->conn.lnk;
 393         struct smc_llc_qentry *qentry;
 394         int rc;
 395
 396         /* receive CONFIRM LINK request from server over RoCE fabric */
 397         qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
 398                               SMC_LLC_CONFIRM_LINK);
 399         if (!qentry) {
 400                 struct smc_clc_msg_decline dclc;
 401
 402                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 403                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 404                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
 405         }
 406         smc_llc_save_peer_uid(qentry);
 407         rc = smc_llc_eval_conf_link(qentry, SMC_LLC_REQ);
 408         smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
 409         if (rc)
 410                 return SMC_CLC_DECL_RMBE_EC;
 411
 412         rc = smc_ib_modify_qp_rts(link);
 413         if (rc)
 414                 return SMC_CLC_DECL_ERR_RDYLNK;
 415
 416         smc_wr_remember_qp_attr(link);
 417
 418         if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
 419                 return SMC_CLC_DECL_ERR_REGRMB;
 420
 421         /* confirm_rkey is implicit on 1st contact */
 422         smc->conn.rmb_desc->is_conf_rkey = true;
 423
 424         /* send CONFIRM LINK response over RoCE fabric */
 425         rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
 426         if (rc < 0)
 427                 return SMC_CLC_DECL_TIMEOUT_CL;
 428
 429         smc_llc_link_active(link);
 430         smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
 431
 432         /* optional 2nd link, receive ADD LINK request from server */
 433         qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
 434                               SMC_LLC_ADD_LINK);
 435         if (!qentry) {
 436                 struct smc_clc_msg_decline dclc;
 437
 438                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 439                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 440                 if (rc == -EAGAIN)
 441                         rc = 0; /* no DECLINE received, go with one link */
 442                 return rc;
 443         }
 444         smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl);
 445         smc_llc_cli_add_link(link, qentry);
 446         return 0;
 447 }
 448
 449 static bool smc_isascii(char *hostname)
 450 {
 451         int i;
 452
 453         for (i = 0; i < SMC_MAX_HOSTNAME_LEN; i++)
 454                 if (!isascii(hostname[i]))
 455                         return false;
 456         return true;
 457 }
 458
 459 static void smc_conn_save_peer_info_fce(struct smc_sock *smc,
 460                                         struct smc_clc_msg_accept_confirm *clc)
 461 {
 462         struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
 463                 (struct smc_clc_msg_accept_confirm_v2 *)clc;
 464         struct smc_clc_first_contact_ext *fce;
 465         int clc_v2_len;
 466
 467         if (clc->hdr.version == SMC_V1 ||
 468             !(clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK))
 469                 return;
 470
 471         if (smc->conn.lgr->is_smcd) {
 472                 memcpy(smc->conn.lgr->negotiated_eid, clc_v2->d1.eid,
 473                        SMC_MAX_EID_LEN);
 474                 clc_v2_len = offsetofend(struct smc_clc_msg_accept_confirm_v2,
 475                                          d1);
 476         } else {
 477                 memcpy(smc->conn.lgr->negotiated_eid, clc_v2->r1.eid,
 478                        SMC_MAX_EID_LEN);
 479                 clc_v2_len = offsetofend(struct smc_clc_msg_accept_confirm_v2,
 480                                          r1);
 481         }
 482         fce = (struct smc_clc_first_contact_ext *)(((u8 *)clc_v2) + clc_v2_len);
 483         smc->conn.lgr->peer_os = fce->os_type;
 484         smc->conn.lgr->peer_smc_release = fce->release;
 485         if (smc_isascii(fce->hostname))
 486                 memcpy(smc->conn.lgr->peer_hostname, fce->hostname,
 487                        SMC_MAX_HOSTNAME_LEN);
 488 }
 489
 490 static void smcr_conn_save_peer_info(struct smc_sock *smc,
 491                                      struct smc_clc_msg_accept_confirm *clc)
 492 {
 493         int bufsize = smc_uncompress_bufsize(clc->r0.rmbe_size);
 494
 495         smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx;
 496         smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token);
 497         smc->conn.peer_rmbe_size = bufsize;
 498         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 499         smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
 500 }
 501
 502 static void smcd_conn_save_peer_info(struct smc_sock *smc,
 503                                      struct smc_clc_msg_accept_confirm *clc)
 504 {
 505         int bufsize = smc_uncompress_bufsize(clc->d0.dmbe_size);
 506
 507         smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx;
 508         smc->conn.peer_token = clc->d0.token;
 509         /* msg header takes up space in the buffer */
 510         smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
 511         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 512         smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
 513 }
 514
 515 static void smc_conn_save_peer_info(struct smc_sock *smc,
 516                                     struct smc_clc_msg_accept_confirm *clc)
 517 {
 518         if (smc->conn.lgr->is_smcd)
 519                 smcd_conn_save_peer_info(smc, clc);
 520         else
 521                 smcr_conn_save_peer_info(smc, clc);
 522         smc_conn_save_peer_info_fce(smc, clc);
 523 }
 524
 525 static void smc_link_save_peer_info(struct smc_link *link,
 526                                     struct smc_clc_msg_accept_confirm *clc,
 527                                     struct smc_init_info *ini)
 528 {
 529         link->peer_qpn = ntoh24(clc->r0.qpn);
 530         memcpy(link->peer_gid, ini->peer_gid, SMC_GID_SIZE);
 531         memcpy(link->peer_mac, ini->peer_mac, sizeof(link->peer_mac));
 532         link->peer_psn = ntoh24(clc->r0.psn);
 533         link->peer_mtu = clc->r0.qp_mtu;
 534 }
 535
 536 static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc,
 537                                        struct smc_stats_fback *fback_arr)
 538 {
 539         int cnt;
 540
 541         for (cnt = 0; cnt < SMC_MAX_FBACK_RSN_CNT; cnt++) {
 542                 if (fback_arr[cnt].fback_code == smc->fallback_rsn) {
 543                         fback_arr[cnt].count++;
 544                         break;
 545                 }
 546                 if (!fback_arr[cnt].fback_code) {
 547                         fback_arr[cnt].fback_code = smc->fallback_rsn;
 548                         fback_arr[cnt].count++;
 549                         break;
 550                 }
 551         }
 552 }
 553
 554 static void smc_stat_fallback(struct smc_sock *smc)
 555 {
 556         struct net *net = sock_net(&smc->sk);
 557
 558         mutex_lock(&net->smc.mutex_fback_rsn);
 559         if (smc->listen_smc) {
 560                 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->srv);
 561                 net->smc.fback_rsn->srv_fback_cnt++;
 562         } else {
 563                 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->clnt);
 564                 net->smc.fback_rsn->clnt_fback_cnt++;
 565         }
 566         mutex_unlock(&net->smc.mutex_fback_rsn);
 567 }
 568
 569 /* must be called under rcu read lock */
 570 static void smc_fback_wakeup_waitqueue(struct smc_sock *smc, void *key)
 571 {
 572         struct socket_wq *wq;
 573         __poll_t flags;
 574
 575         wq = rcu_dereference(smc->sk.sk_wq);
 576         if (!skwq_has_sleeper(wq))
 577                 return;
 578
 579         /* wake up smc sk->sk_wq */
 580         if (!key) {
 581                 /* sk_state_change */
 582                 wake_up_interruptible_all(&wq->wait);
 583         } else {
 584                 flags = key_to_poll(key);
 585                 if (flags & (EPOLLIN | EPOLLOUT))
 586                         /* sk_data_ready or sk_write_space */
 587                         wake_up_interruptible_sync_poll(&wq->wait, flags);
 588                 else if (flags & EPOLLERR)
 589                         /* sk_error_report */
 590                         wake_up_interruptible_poll(&wq->wait, flags);
 591         }
 592 }
 593
 594 static int smc_fback_mark_woken(wait_queue_entry_t *wait,
 595                                 unsigned int mode, int sync, void *key)
 596 {
 597         struct smc_mark_woken *mark =
 598                 container_of(wait, struct smc_mark_woken, wait_entry);
 599
 600         mark->woken = true;
 601         mark->key = key;
 602         return 0;
 603 }
 604
 605 static void smc_fback_forward_wakeup(struct smc_sock *smc, struct sock *clcsk,
 606                                      void (*clcsock_callback)(struct sock *sk))
 607 {
 608         struct smc_mark_woken mark = { .woken = false };
 609         struct socket_wq *wq;
 610
 611         init_waitqueue_func_entry(&mark.wait_entry,
 612                                   smc_fback_mark_woken);
 613         rcu_read_lock();
 614         wq = rcu_dereference(clcsk->sk_wq);
 615         if (!wq)
 616                 goto out;
 617         add_wait_queue(sk_sleep(clcsk), &mark.wait_entry);
 618         clcsock_callback(clcsk);
 619         remove_wait_queue(sk_sleep(clcsk), &mark.wait_entry);
 620
 621         if (mark.woken)
 622                 smc_fback_wakeup_waitqueue(smc, mark.key);
 623 out:
 624         rcu_read_unlock();
 625 }
 626
 627 static void smc_fback_state_change(struct sock *clcsk)
 628 {
 629         struct smc_sock *smc =
 630                 smc_clcsock_user_data(clcsk);
 631
 632         if (!smc)
 633                 return;
 634         smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_state_change);
 635 }
 636
 637 static void smc_fback_data_ready(struct sock *clcsk)
 638 {
 639         struct smc_sock *smc =
 640                 smc_clcsock_user_data(clcsk);
 641
 642         if (!smc)
 643                 return;
 644         smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_data_ready);
 645 }
 646
 647 static void smc_fback_write_space(struct sock *clcsk)
 648 {
 649         struct smc_sock *smc =
 650                 smc_clcsock_user_data(clcsk);
 651
 652         if (!smc)
 653                 return;
 654         smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_write_space);
 655 }
 656
 657 static void smc_fback_error_report(struct sock *clcsk)
 658 {
 659         struct smc_sock *smc =
 660                 smc_clcsock_user_data(clcsk);
 661
 662         if (!smc)
 663                 return;
 664         smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_error_report);
 665 }
 666
 667 static int smc_switch_to_fallback(struct smc_sock *smc, int reason_code)
 668 {
 669         struct sock *clcsk;
 670         int rc = 0;
 671
 672         mutex_lock(&smc->clcsock_release_lock);
 673         if (!smc->clcsock) {
 674                 rc = -EBADF;
 675                 goto out;
 676         }
 677         clcsk = smc->clcsock->sk;
 678
 679         if (smc->use_fallback)
 680                 goto out;
 681         smc->use_fallback = true;
 682         smc->fallback_rsn = reason_code;
 683         smc_stat_fallback(smc);
 684         trace_smc_switch_to_fallback(smc, reason_code);
 685         if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
 686                 smc->clcsock->file = smc->sk.sk_socket->file;
 687                 smc->clcsock->file->private_data = smc->clcsock;
 688                 smc->clcsock->wq.fasync_list =
 689                         smc->sk.sk_socket->wq.fasync_list;
 690
 691                 /* There might be some wait entries remaining
 692                  * in smc sk->sk_wq and they should be woken up
 693                  * as clcsock's wait queue is woken up.
 694                  */
 695                 smc->clcsk_state_change = clcsk->sk_state_change;
 696                 smc->clcsk_data_ready = clcsk->sk_data_ready;
 697                 smc->clcsk_write_space = clcsk->sk_write_space;
 698                 smc->clcsk_error_report = clcsk->sk_error_report;
 699
 700                 clcsk->sk_state_change = smc_fback_state_change;
 701                 clcsk->sk_data_ready = smc_fback_data_ready;
 702                 clcsk->sk_write_space = smc_fback_write_space;
 703                 clcsk->sk_error_report = smc_fback_error_report;
 704
 705                 smc->clcsock->sk->sk_user_data =
 706                         (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
 707         }
 708 out:
 709         mutex_unlock(&smc->clcsock_release_lock);
 710         return rc;
 711 }
 712
 713 /* fall back during connect */
 714 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
 715 {
 716         struct net *net = sock_net(&smc->sk);
 717         int rc = 0;
 718
 719         rc = smc_switch_to_fallback(smc, reason_code);
 720         if (rc) { /* fallback fails */
 721                 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 722                 if (smc->sk.sk_state == SMC_INIT)
 723                         sock_put(&smc->sk); /* passive closing */
 724                 return rc;
 725         }
 726         smc_copy_sock_settings_to_clc(smc);
 727         smc->connect_nonblock = 0;
 728         if (smc->sk.sk_state == SMC_INIT)
 729                 smc->sk.sk_state = SMC_ACTIVE;
 730         return 0;
 731 }
 732
 733 /* decline and fall back during connect */
 734 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code,
 735                                         u8 version)
 736 {
 737         struct net *net = sock_net(&smc->sk);
 738         int rc;
 739
 740         if (reason_code < 0) { /* error, fallback is not possible */
 741                 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 742                 if (smc->sk.sk_state == SMC_INIT)
 743                         sock_put(&smc->sk); /* passive closing */
 744                 return reason_code;
 745         }
 746         if (reason_code != SMC_CLC_DECL_PEERDECL) {
 747                 rc = smc_clc_send_decline(smc, reason_code, version);
 748                 if (rc < 0) {
 749                         this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 750                         if (smc->sk.sk_state == SMC_INIT)
 751                                 sock_put(&smc->sk); /* passive closing */
 752                         return rc;
 753                 }
 754         }
 755         return smc_connect_fallback(smc, reason_code);
 756 }
 757
 758 static void smc_conn_abort(struct smc_sock *smc, int local_first)
 759 {
 760         struct smc_connection *conn = &smc->conn;
 761         struct smc_link_group *lgr = conn->lgr;
 762         bool lgr_valid = false;
 763
 764         if (smc_conn_lgr_valid(conn))
 765                 lgr_valid = true;
 766
 767         smc_conn_free(conn);
 768         if (local_first && lgr_valid)
 769                 smc_lgr_cleanup_early(lgr);
 770 }
 771
 772 /* check if there is a rdma device available for this connection. */
 773 /* called for connect and listen */
 774 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
 775 {
 776         /* PNET table look up: search active ib_device and port
 777          * within same PNETID that also contains the ethernet device
 778          * used for the internal TCP socket
 779          */
 780         smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
 781         if (!ini->check_smcrv2 && !ini->ib_dev)
 782                 return SMC_CLC_DECL_NOSMCRDEV;
 783         if (ini->check_smcrv2 && !ini->smcrv2.ib_dev_v2)
 784                 return SMC_CLC_DECL_NOSMCRDEV;
 785         return 0;
 786 }
 787
 788 /* check if there is an ISM device available for this connection. */
 789 /* called for connect and listen */
 790 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
 791 {
 792         /* Find ISM device with same PNETID as connecting interface  */
 793         smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
 794         if (!ini->ism_dev[0])
 795                 return SMC_CLC_DECL_NOSMCDDEV;
 796         else
 797                 ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]);
 798         return 0;
 799 }
 800
 801 /* is chid unique for the ism devices that are already determined? */
 802 static bool smc_find_ism_v2_is_unique_chid(u16 chid, struct smc_init_info *ini,
 803                                            int cnt)
 804 {
 805         int i = (!ini->ism_dev[0]) ? 1 : 0;
 806
 807         for (; i < cnt; i++)
 808                 if (ini->ism_chid[i] == chid)
 809                         return false;
 810         return true;
 811 }
 812
 813 /* determine possible V2 ISM devices (either without PNETID or with PNETID plus
 814  * PNETID matching net_device)
 815  */
 816 static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
 817                                        struct smc_init_info *ini)
 818 {
 819         int rc = SMC_CLC_DECL_NOSMCDDEV;
 820         struct smcd_dev *smcd;
 821         int i = 1;
 822         u16 chid;
 823
 824         if (smcd_indicated(ini->smc_type_v1))
 825                 rc = 0;         /* already initialized for V1 */
 826         mutex_lock(&smcd_dev_list.mutex);
 827         list_for_each_entry(smcd, &smcd_dev_list.list, list) {
 828                 if (smcd->going_away || smcd == ini->ism_dev[0])
 829                         continue;
 830                 chid = smc_ism_get_chid(smcd);
 831                 if (!smc_find_ism_v2_is_unique_chid(chid, ini, i))
 832                         continue;
 833                 if (!smc_pnet_is_pnetid_set(smcd->pnetid) ||
 834                     smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) {
 835                         ini->ism_dev[i] = smcd;
 836                         ini->ism_chid[i] = chid;
 837                         ini->is_smcd = true;
 838                         rc = 0;
 839                         i++;
 840                         if (i > SMC_MAX_ISM_DEVS)
 841                                 break;
 842                 }
 843         }
 844         mutex_unlock(&smcd_dev_list.mutex);
 845         ini->ism_offered_cnt = i - 1;
 846         if (!ini->ism_dev[0] && !ini->ism_dev[1])
 847                 ini->smcd_version = 0;
 848
 849         return rc;
 850 }
 851
 852 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
 853 static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
 854                                       struct smc_init_info *ini)
 855 {
 856         if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id))
 857                 return SMC_CLC_DECL_ISMVLANERR;
 858         return 0;
 859 }
 860
 861 static int smc_find_proposal_devices(struct smc_sock *smc,
 862                                      struct smc_init_info *ini)
 863 {
 864         int rc = 0;
 865
 866         /* check if there is an ism device available */
 867         if (!(ini->smcd_version & SMC_V1) ||
 868             smc_find_ism_device(smc, ini) ||
 869             smc_connect_ism_vlan_setup(smc, ini))
 870                 ini->smcd_version &= ~SMC_V1;
 871         /* else ISM V1 is supported for this connection */
 872
 873         /* check if there is an rdma device available */
 874         if (!(ini->smcr_version & SMC_V1) ||
 875             smc_find_rdma_device(smc, ini))
 876                 ini->smcr_version &= ~SMC_V1;
 877         /* else RDMA is supported for this connection */
 878
 879         ini->smc_type_v1 = smc_indicated_type(ini->smcd_version & SMC_V1,
 880                                               ini->smcr_version & SMC_V1);
 881
 882         /* check if there is an ism v2 device available */
 883         if (!(ini->smcd_version & SMC_V2) ||
 884             !smc_ism_is_v2_capable() ||
 885             smc_find_ism_v2_device_clnt(smc, ini))
 886                 ini->smcd_version &= ~SMC_V2;
 887
 888         /* check if there is an rdma v2 device available */
 889         ini->check_smcrv2 = true;
 890         ini->smcrv2.saddr = smc->clcsock->sk->sk_rcv_saddr;
 891         if (!(ini->smcr_version & SMC_V2) ||
 892             smc->clcsock->sk->sk_family != AF_INET ||
 893             !smc_clc_ueid_count() ||
 894             smc_find_rdma_device(smc, ini))
 895                 ini->smcr_version &= ~SMC_V2;
 896         ini->check_smcrv2 = false;
 897
 898         ini->smc_type_v2 = smc_indicated_type(ini->smcd_version & SMC_V2,
 899                                               ini->smcr_version & SMC_V2);
 900
 901         /* if neither ISM nor RDMA are supported, fallback */
 902         if (ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N)
 903                 rc = SMC_CLC_DECL_NOSMCDEV;
 904
 905         return rc;
 906 }
 907
 908 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
 909  * used, the VLAN ID will be registered again during the connection setup.
 910  */
 911 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc,
 912                                         struct smc_init_info *ini)
 913 {
 914         if (!smcd_indicated(ini->smc_type_v1))
 915                 return 0;
 916         if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id))
 917                 return SMC_CLC_DECL_CNFERR;
 918         return 0;
 919 }
 920
 921 #define SMC_CLC_MAX_ACCEPT_LEN \
 922         (sizeof(struct smc_clc_msg_accept_confirm_v2) + \
 923          sizeof(struct smc_clc_first_contact_ext) + \
 924          sizeof(struct smc_clc_msg_trail))
 925
 926 /* CLC handshake during connect */
 927 static int smc_connect_clc(struct smc_sock *smc,
 928                            struct smc_clc_msg_accept_confirm_v2 *aclc2,
 929                            struct smc_init_info *ini)
 930 {
 931         int rc = 0;
 932
 933         /* do inband token exchange */
 934         rc = smc_clc_send_proposal(smc, ini);
 935         if (rc)
 936                 return rc;
 937         /* receive SMC Accept CLC message */
 938         return smc_clc_wait_msg(smc, aclc2, SMC_CLC_MAX_ACCEPT_LEN,
 939                                 SMC_CLC_ACCEPT, CLC_WAIT_TIME);
 940 }
 941
 942 void smc_fill_gid_list(struct smc_link_group *lgr,
 943                        struct smc_gidlist *gidlist,
 944                        struct smc_ib_device *known_dev, u8 *known_gid)
 945 {
 946         struct smc_init_info *alt_ini = NULL;
 947
 948         memset(gidlist, 0, sizeof(*gidlist));
 949         memcpy(gidlist->list[gidlist->len++], known_gid, SMC_GID_SIZE);
 950
 951         alt_ini = kzalloc(sizeof(*alt_ini), GFP_KERNEL);
 952         if (!alt_ini)
 953                 goto out;
 954
 955         alt_ini->vlan_id = lgr->vlan_id;
 956         alt_ini->check_smcrv2 = true;
 957         alt_ini->smcrv2.saddr = lgr->saddr;
 958         smc_pnet_find_alt_roce(lgr, alt_ini, known_dev);
 959
 960         if (!alt_ini->smcrv2.ib_dev_v2)
 961                 goto out;
 962
 963         memcpy(gidlist->list[gidlist->len++], alt_ini->smcrv2.ib_gid_v2,
 964                SMC_GID_SIZE);
 965
 966 out:
 967         kfree(alt_ini);
 968 }
 969
 970 static int smc_connect_rdma_v2_prepare(struct smc_sock *smc,
 971                                        struct smc_clc_msg_accept_confirm *aclc,
 972                                        struct smc_init_info *ini)
 973 {
 974         struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
 975                 (struct smc_clc_msg_accept_confirm_v2 *)aclc;
 976         struct smc_clc_first_contact_ext *fce =
 977                 (struct smc_clc_first_contact_ext *)
 978                         (((u8 *)clc_v2) + sizeof(*clc_v2));
 979
 980         if (!ini->first_contact_peer || aclc->hdr.version == SMC_V1)
 981                 return 0;
 982
 983         if (fce->v2_direct) {
 984                 memcpy(ini->smcrv2.nexthop_mac, &aclc->r0.lcl.mac, ETH_ALEN);
 985                 ini->smcrv2.uses_gateway = false;
 986         } else {
 987                 if (smc_ib_find_route(smc->clcsock->sk->sk_rcv_saddr,
 988                                       smc_ib_gid_to_ipv4(aclc->r0.lcl.gid),
 989                                       ini->smcrv2.nexthop_mac,
 990                                       &ini->smcrv2.uses_gateway))
 991                         return SMC_CLC_DECL_NOROUTE;
 992                 if (!ini->smcrv2.uses_gateway) {
 993                         /* mismatch: peer claims indirect, but its direct */
 994                         return SMC_CLC_DECL_NOINDIRECT;
 995                 }
 996         }
 997         return 0;
 998 }
 999
1000 /* setup for RDMA connection of client */
1001 static int smc_connect_rdma(struct smc_sock *smc,
1002                             struct smc_clc_msg_accept_confirm *aclc,
1003                             struct smc_init_info *ini)
1004 {
1005         int i, reason_code = 0;
1006         struct smc_link *link;
1007         u8 *eid = NULL;
1008
1009         ini->is_smcd = false;
1010         ini->ib_clcqpn = ntoh24(aclc->r0.qpn);
1011         ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
1012         memcpy(ini->peer_systemid, aclc->r0.lcl.id_for_peer, SMC_SYSTEMID_LEN);
1013         memcpy(ini->peer_gid, aclc->r0.lcl.gid, SMC_GID_SIZE);
1014         memcpy(ini->peer_mac, aclc->r0.lcl.mac, ETH_ALEN);
1015
1016         reason_code = smc_connect_rdma_v2_prepare(smc, aclc, ini);
1017         if (reason_code)
1018                 return reason_code;
1019
1020         mutex_lock(&smc_client_lgr_pending);
1021         reason_code = smc_conn_create(smc, ini);
1022         if (reason_code) {
1023                 mutex_unlock(&smc_client_lgr_pending);
1024                 return reason_code;
1025         }
1026
1027         smc_conn_save_peer_info(smc, aclc);
1028
1029         if (ini->first_contact_local) {
1030                 link = smc->conn.lnk;
1031         } else {
1032                 /* set link that was assigned by server */
1033                 link = NULL;
1034                 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1035                         struct smc_link *l = &smc->conn.lgr->lnk[i];
1036
1037                         if (l->peer_qpn == ntoh24(aclc->r0.qpn) &&
1038                             !memcmp(l->peer_gid, &aclc->r0.lcl.gid,
1039                                     SMC_GID_SIZE) &&
1040                             (aclc->hdr.version > SMC_V1 ||
1041                              !memcmp(l->peer_mac, &aclc->r0.lcl.mac,
1042                                      sizeof(l->peer_mac)))) {
1043                                 link = l;
1044                                 break;
1045                         }
1046                 }
1047                 if (!link) {
1048                         reason_code = SMC_CLC_DECL_NOSRVLINK;
1049                         goto connect_abort;
1050                 }
1051                 smc_switch_link_and_count(&smc->conn, link);
1052         }
1053
1054         /* create send buffer and rmb */
1055         if (smc_buf_create(smc, false)) {
1056                 reason_code = SMC_CLC_DECL_MEM;
1057                 goto connect_abort;
1058         }
1059
1060         if (ini->first_contact_local)
1061                 smc_link_save_peer_info(link, aclc, ini);
1062
1063         if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) {
1064                 reason_code = SMC_CLC_DECL_ERR_RTOK;
1065                 goto connect_abort;
1066         }
1067
1068         smc_close_init(smc);
1069         smc_rx_init(smc);
1070
1071         if (ini->first_contact_local) {
1072                 if (smc_ib_ready_link(link)) {
1073                         reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1074                         goto connect_abort;
1075                 }
1076         } else {
1077                 if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) {
1078                         reason_code = SMC_CLC_DECL_ERR_REGRMB;
1079                         goto connect_abort;
1080                 }
1081         }
1082         smc_rmb_sync_sg_for_device(&smc->conn);
1083
1084         if (aclc->hdr.version > SMC_V1) {
1085                 struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
1086                         (struct smc_clc_msg_accept_confirm_v2 *)aclc;
1087
1088                 eid = clc_v2->r1.eid;
1089                 if (ini->first_contact_local)
1090                         smc_fill_gid_list(link->lgr, &ini->smcrv2.gidlist,
1091                                           link->smcibdev, link->gid);
1092         }
1093
1094         reason_code = smc_clc_send_confirm(smc, ini->first_contact_local,
1095                                            aclc->hdr.version, eid, ini);
1096         if (reason_code)
1097                 goto connect_abort;
1098
1099         smc_tx_init(smc);
1100
1101         if (ini->first_contact_local) {
1102                 /* QP confirmation over RoCE fabric */
1103                 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
1104                 reason_code = smcr_clnt_conf_first_link(smc);
1105                 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
1106                 if (reason_code)
1107                         goto connect_abort;
1108         }
1109         mutex_unlock(&smc_client_lgr_pending);
1110
1111         smc_copy_sock_settings_to_clc(smc);
1112         smc->connect_nonblock = 0;
1113         if (smc->sk.sk_state == SMC_INIT)
1114                 smc->sk.sk_state = SMC_ACTIVE;
1115
1116         return 0;
1117 connect_abort:
1118         smc_conn_abort(smc, ini->first_contact_local);
1119         mutex_unlock(&smc_client_lgr_pending);
1120         smc->connect_nonblock = 0;
1121
1122         return reason_code;
1123 }
1124
1125 /* The server has chosen one of the proposed ISM devices for the communication.
1126  * Determine from the CHID of the received CLC ACCEPT the ISM device chosen.
1127  */
1128 static int
1129 smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm_v2 *aclc,
1130                                struct smc_init_info *ini)
1131 {
1132         int i;
1133
1134         for (i = 0; i < ini->ism_offered_cnt + 1; i++) {
1135                 if (ini->ism_chid[i] == ntohs(aclc->d1.chid)) {
1136                         ini->ism_selected = i;
1137                         return 0;
1138                 }
1139         }
1140
1141         return -EPROTO;
1142 }
1143
1144 /* setup for ISM connection of client */
1145 static int smc_connect_ism(struct smc_sock *smc,
1146                            struct smc_clc_msg_accept_confirm *aclc,
1147                            struct smc_init_info *ini)
1148 {
1149         u8 *eid = NULL;
1150         int rc = 0;
1151
1152         ini->is_smcd = true;
1153         ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
1154
1155         if (aclc->hdr.version == SMC_V2) {
1156                 struct smc_clc_msg_accept_confirm_v2 *aclc_v2 =
1157                         (struct smc_clc_msg_accept_confirm_v2 *)aclc;
1158
1159                 rc = smc_v2_determine_accepted_chid(aclc_v2, ini);
1160                 if (rc)
1161                         return rc;
1162         }
1163         ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid;
1164
1165         /* there is only one lgr role for SMC-D; use server lock */
1166         mutex_lock(&smc_server_lgr_pending);
1167         rc = smc_conn_create(smc, ini);
1168         if (rc) {
1169                 mutex_unlock(&smc_server_lgr_pending);
1170                 return rc;
1171         }
1172
1173         /* Create send and receive buffers */
1174         rc = smc_buf_create(smc, true);
1175         if (rc) {
1176                 rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM;
1177                 goto connect_abort;
1178         }
1179
1180         smc_conn_save_peer_info(smc, aclc);
1181         smc_close_init(smc);
1182         smc_rx_init(smc);
1183         smc_tx_init(smc);
1184
1185         if (aclc->hdr.version > SMC_V1) {
1186                 struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
1187                         (struct smc_clc_msg_accept_confirm_v2 *)aclc;
1188
1189                 eid = clc_v2->d1.eid;
1190         }
1191
1192         rc = smc_clc_send_confirm(smc, ini->first_contact_local,
1193                                   aclc->hdr.version, eid, NULL);
1194         if (rc)
1195                 goto connect_abort;
1196         mutex_unlock(&smc_server_lgr_pending);
1197
1198         smc_copy_sock_settings_to_clc(smc);
1199         smc->connect_nonblock = 0;
1200         if (smc->sk.sk_state == SMC_INIT)
1201                 smc->sk.sk_state = SMC_ACTIVE;
1202
1203         return 0;
1204 connect_abort:
1205         smc_conn_abort(smc, ini->first_contact_local);
1206         mutex_unlock(&smc_server_lgr_pending);
1207         smc->connect_nonblock = 0;
1208
1209         return rc;
1210 }
1211
1212 /* check if received accept type and version matches a proposed one */
1213 static int smc_connect_check_aclc(struct smc_init_info *ini,
1214                                   struct smc_clc_msg_accept_confirm *aclc)
1215 {
1216         if (aclc->hdr.typev1 != SMC_TYPE_R &&
1217             aclc->hdr.typev1 != SMC_TYPE_D)
1218                 return SMC_CLC_DECL_MODEUNSUPP;
1219
1220         if (aclc->hdr.version >= SMC_V2) {
1221                 if ((aclc->hdr.typev1 == SMC_TYPE_R &&
1222                      !smcr_indicated(ini->smc_type_v2)) ||
1223                     (aclc->hdr.typev1 == SMC_TYPE_D &&
1224                      !smcd_indicated(ini->smc_type_v2)))
1225                         return SMC_CLC_DECL_MODEUNSUPP;
1226         } else {
1227                 if ((aclc->hdr.typev1 == SMC_TYPE_R &&
1228                      !smcr_indicated(ini->smc_type_v1)) ||
1229                     (aclc->hdr.typev1 == SMC_TYPE_D &&
1230                      !smcd_indicated(ini->smc_type_v1)))
1231                         return SMC_CLC_DECL_MODEUNSUPP;
1232         }
1233
1234         return 0;
1235 }
1236
1237 /* perform steps before actually connecting */
1238 static int __smc_connect(struct smc_sock *smc)
1239 {
1240         u8 version = smc_ism_is_v2_capable() ? SMC_V2 : SMC_V1;
1241         struct smc_clc_msg_accept_confirm_v2 *aclc2;
1242         struct smc_clc_msg_accept_confirm *aclc;
1243         struct smc_init_info *ini = NULL;
1244         u8 *buf = NULL;
1245         int rc = 0;
1246
1247         if (smc->use_fallback)
1248                 return smc_connect_fallback(smc, smc->fallback_rsn);
1249
1250         /* if peer has not signalled SMC-capability, fall back */
1251         if (!tcp_sk(smc->clcsock->sk)->syn_smc)
1252                 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
1253
1254         /* IPSec connections opt out of SMC optimizations */
1255         if (using_ipsec(smc))
1256                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC,
1257                                                     version);
1258
1259         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1260         if (!ini)
1261                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM,
1262                                                     version);
1263
1264         ini->smcd_version = SMC_V1 | SMC_V2;
1265         ini->smcr_version = SMC_V1 | SMC_V2;
1266         ini->smc_type_v1 = SMC_TYPE_B;
1267         ini->smc_type_v2 = SMC_TYPE_B;
1268
1269         /* get vlan id from IP device */
1270         if (smc_vlan_by_tcpsk(smc->clcsock, ini)) {
1271                 ini->smcd_version &= ~SMC_V1;
1272                 ini->smcr_version = 0;
1273                 ini->smc_type_v1 = SMC_TYPE_N;
1274                 if (!ini->smcd_version) {
1275                         rc = SMC_CLC_DECL_GETVLANERR;
1276                         goto fallback;
1277                 }
1278         }
1279
1280         rc = smc_find_proposal_devices(smc, ini);
1281         if (rc)
1282                 goto fallback;
1283
1284         buf = kzalloc(SMC_CLC_MAX_ACCEPT_LEN, GFP_KERNEL);
1285         if (!buf) {
1286                 rc = SMC_CLC_DECL_MEM;
1287                 goto fallback;
1288         }
1289         aclc2 = (struct smc_clc_msg_accept_confirm_v2 *)buf;
1290         aclc = (struct smc_clc_msg_accept_confirm *)aclc2;
1291
1292         /* perform CLC handshake */
1293         rc = smc_connect_clc(smc, aclc2, ini);
1294         if (rc)
1295                 goto vlan_cleanup;
1296
1297         /* check if smc modes and versions of CLC proposal and accept match */
1298         rc = smc_connect_check_aclc(ini, aclc);
1299         version = aclc->hdr.version == SMC_V1 ? SMC_V1 : SMC_V2;
1300         if (rc)
1301                 goto vlan_cleanup;
1302
1303         /* depending on previous steps, connect using rdma or ism */
1304         if (aclc->hdr.typev1 == SMC_TYPE_R) {
1305                 ini->smcr_version = version;
1306                 rc = smc_connect_rdma(smc, aclc, ini);
1307         } else if (aclc->hdr.typev1 == SMC_TYPE_D) {
1308                 ini->smcd_version = version;
1309                 rc = smc_connect_ism(smc, aclc, ini);
1310         }
1311         if (rc)
1312                 goto vlan_cleanup;
1313
1314         SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc);
1315         smc_connect_ism_vlan_cleanup(smc, ini);
1316         kfree(buf);
1317         kfree(ini);
1318         return 0;
1319
1320 vlan_cleanup:
1321         smc_connect_ism_vlan_cleanup(smc, ini);
1322         kfree(buf);
1323 fallback:
1324         kfree(ini);
1325         return smc_connect_decline_fallback(smc, rc, version);
1326 }
1327
1328 static void smc_connect_work(struct work_struct *work)
1329 {
1330         struct smc_sock *smc = container_of(work, struct smc_sock,
1331                                             connect_work);
1332         long timeo = smc->sk.sk_sndtimeo;
1333         int rc = 0;
1334
1335         if (!timeo)
1336                 timeo = MAX_SCHEDULE_TIMEOUT;
1337         lock_sock(smc->clcsock->sk);
1338         if (smc->clcsock->sk->sk_err) {
1339                 smc->sk.sk_err = smc->clcsock->sk->sk_err;
1340         } else if ((1 << smc->clcsock->sk->sk_state) &
1341                                         (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
1342                 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
1343                 if ((rc == -EPIPE) &&
1344                     ((1 << smc->clcsock->sk->sk_state) &
1345                                         (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
1346                         rc = 0;
1347         }
1348         release_sock(smc->clcsock->sk);
1349         lock_sock(&smc->sk);
1350         if (rc != 0 || smc->sk.sk_err) {
1351                 smc->sk.sk_state = SMC_CLOSED;
1352                 if (rc == -EPIPE || rc == -EAGAIN)
1353                         smc->sk.sk_err = EPIPE;
1354                 else if (signal_pending(current))
1355                         smc->sk.sk_err = -sock_intr_errno(timeo);
1356                 sock_put(&smc->sk); /* passive closing */
1357                 goto out;
1358         }
1359
1360         rc = __smc_connect(smc);
1361         if (rc < 0)
1362                 smc->sk.sk_err = -rc;
1363
1364 out:
1365         if (!sock_flag(&smc->sk, SOCK_DEAD)) {
1366                 if (smc->sk.sk_err) {
1367                         smc->sk.sk_state_change(&smc->sk);
1368                 } else { /* allow polling before and after fallback decision */
1369                         smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
1370                         smc->sk.sk_write_space(&smc->sk);
1371                 }
1372         }
1373         release_sock(&smc->sk);
1374 }
1375
1376 static int smc_connect(struct socket *sock, struct sockaddr *addr,
1377                        int alen, int flags)
1378 {
1379         struct sock *sk = sock->sk;
1380         struct smc_sock *smc;
1381         int rc = -EINVAL;
1382
1383         smc = smc_sk(sk);
1384
1385         /* separate smc parameter checking to be safe */
1386         if (alen < sizeof(addr->sa_family))
1387                 goto out_err;
1388         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
1389                 goto out_err;
1390
1391         lock_sock(sk);
1392         switch (sk->sk_state) {
1393         default:
1394                 goto out;
1395         case SMC_ACTIVE:
1396                 rc = -EISCONN;
1397                 goto out;
1398         case SMC_INIT:
1399                 break;
1400         }
1401
1402         smc_copy_sock_settings_to_clc(smc);
1403         tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1404         if (smc->connect_nonblock) {
1405                 rc = -EALREADY;
1406                 goto out;
1407         }
1408         rc = kernel_connect(smc->clcsock, addr, alen, flags);
1409         if (rc && rc != -EINPROGRESS)
1410                 goto out;
1411
1412         sock_hold(&smc->sk); /* sock put in passive closing */
1413         if (smc->use_fallback)
1414                 goto out;
1415         if (flags & O_NONBLOCK) {
1416                 if (queue_work(smc_hs_wq, &smc->connect_work))
1417                         smc->connect_nonblock = 1;
1418                 rc = -EINPROGRESS;
1419         } else {
1420                 rc = __smc_connect(smc);
1421                 if (rc < 0)
1422                         goto out;
1423                 else
1424                         rc = 0; /* success cases including fallback */
1425         }
1426
1427 out:
1428         release_sock(sk);
1429 out_err:
1430         return rc;
1431 }
1432
1433 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
1434 {
1435         struct socket *new_clcsock = NULL;
1436         struct sock *lsk = &lsmc->sk;
1437         struct sock *new_sk;
1438         int rc = -EINVAL;
1439
1440         release_sock(lsk);
1441         new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
1442         if (!new_sk) {
1443                 rc = -ENOMEM;
1444                 lsk->sk_err = ENOMEM;
1445                 *new_smc = NULL;
1446                 lock_sock(lsk);
1447                 goto out;
1448         }
1449         *new_smc = smc_sk(new_sk);
1450
1451         mutex_lock(&lsmc->clcsock_release_lock);
1452         if (lsmc->clcsock)
1453                 rc = kernel_accept(lsmc->clcsock, &new_clcsock, SOCK_NONBLOCK);
1454         mutex_unlock(&lsmc->clcsock_release_lock);
1455         lock_sock(lsk);
1456         if  (rc < 0 && rc != -EAGAIN)
1457                 lsk->sk_err = -rc;
1458         if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
1459                 new_sk->sk_prot->unhash(new_sk);
1460                 if (new_clcsock)
1461                         sock_release(new_clcsock);
1462                 new_sk->sk_state = SMC_CLOSED;
1463                 sock_set_flag(new_sk, SOCK_DEAD);
1464                 sock_put(new_sk); /* final */
1465                 *new_smc = NULL;
1466                 goto out;
1467         }
1468
1469         /* new clcsock has inherited the smc listen-specific sk_data_ready
1470          * function; switch it back to the original sk_data_ready function
1471          */
1472         new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready;
1473         (*new_smc)->clcsock = new_clcsock;
1474 out:
1475         return rc;
1476 }
1477
1478 /* add a just created sock to the accept queue of the listen sock as
1479  * candidate for a following socket accept call from user space
1480  */
1481 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
1482 {
1483         struct smc_sock *par = smc_sk(parent);
1484
1485         sock_hold(sk); /* sock_put in smc_accept_unlink () */
1486         spin_lock(&par->accept_q_lock);
1487         list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
1488         spin_unlock(&par->accept_q_lock);
1489         sk_acceptq_added(parent);
1490 }
1491
1492 /* remove a socket from the accept queue of its parental listening socket */
1493 static void smc_accept_unlink(struct sock *sk)
1494 {
1495         struct smc_sock *par = smc_sk(sk)->listen_smc;
1496
1497         spin_lock(&par->accept_q_lock);
1498         list_del_init(&smc_sk(sk)->accept_q);
1499         spin_unlock(&par->accept_q_lock);
1500         sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
1501         sock_put(sk); /* sock_hold in smc_accept_enqueue */
1502 }
1503
1504 /* remove a sock from the accept queue to bind it to a new socket created
1505  * for a socket accept call from user space
1506  */
1507 struct sock *smc_accept_dequeue(struct sock *parent,
1508                                 struct socket *new_sock)
1509 {
1510         struct smc_sock *isk, *n;
1511         struct sock *new_sk;
1512
1513         list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
1514                 new_sk = (struct sock *)isk;
1515
1516                 smc_accept_unlink(new_sk);
1517                 if (new_sk->sk_state == SMC_CLOSED) {
1518                         new_sk->sk_prot->unhash(new_sk);
1519                         if (isk->clcsock) {
1520                                 sock_release(isk->clcsock);
1521                                 isk->clcsock = NULL;
1522                         }
1523                         sock_put(new_sk); /* final */
1524                         continue;
1525                 }
1526                 if (new_sock) {
1527                         sock_graft(new_sk, new_sock);
1528                         if (isk->use_fallback) {
1529                                 smc_sk(new_sk)->clcsock->file = new_sock->file;
1530                                 isk->clcsock->file->private_data = isk->clcsock;
1531                         }
1532                 }
1533                 return new_sk;
1534         }
1535         return NULL;
1536 }
1537
1538 /* clean up for a created but never accepted sock */
1539 void smc_close_non_accepted(struct sock *sk)
1540 {
1541         struct smc_sock *smc = smc_sk(sk);
1542
1543         sock_hold(sk); /* sock_put below */
1544         lock_sock(sk);
1545         if (!sk->sk_lingertime)
1546                 /* wait for peer closing */
1547                 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
1548         __smc_release(smc);
1549         release_sock(sk);
1550         sock_put(sk); /* sock_hold above */
1551         sock_put(sk); /* final sock_put */
1552 }
1553
1554 static int smcr_serv_conf_first_link(struct smc_sock *smc)
1555 {
1556         struct smc_link *link = smc->conn.lnk;
1557         struct smc_llc_qentry *qentry;
1558         int rc;
1559
1560         if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
1561                 return SMC_CLC_DECL_ERR_REGRMB;
1562
1563         /* send CONFIRM LINK request to client over the RoCE fabric */
1564         rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
1565         if (rc < 0)
1566                 return SMC_CLC_DECL_TIMEOUT_CL;
1567
1568         /* receive CONFIRM LINK response from client over the RoCE fabric */
1569         qentry = smc_llc_wait(link->lgr, link, SMC_LLC_WAIT_TIME,
1570                               SMC_LLC_CONFIRM_LINK);
1571         if (!qentry) {
1572                 struct smc_clc_msg_decline dclc;
1573
1574                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1575                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1576                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1577         }
1578         smc_llc_save_peer_uid(qentry);
1579         rc = smc_llc_eval_conf_link(qentry, SMC_LLC_RESP);
1580         smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
1581         if (rc)
1582                 return SMC_CLC_DECL_RMBE_EC;
1583
1584         /* confirm_rkey is implicit on 1st contact */
1585         smc->conn.rmb_desc->is_conf_rkey = true;
1586
1587         smc_llc_link_active(link);
1588         smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
1589
1590         /* initial contact - try to establish second link */
1591         smc_llc_srv_add_link(link, NULL);
1592         return 0;
1593 }
1594
1595 /* listen worker: finish */
1596 static void smc_listen_out(struct smc_sock *new_smc)
1597 {
1598         struct smc_sock *lsmc = new_smc->listen_smc;
1599         struct sock *newsmcsk = &new_smc->sk;
1600
1601         if (lsmc->sk.sk_state == SMC_LISTEN) {
1602                 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1603                 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1604                 release_sock(&lsmc->sk);
1605         } else { /* no longer listening */
1606                 smc_close_non_accepted(newsmcsk);
1607         }
1608
1609         /* Wake up accept */
1610         lsmc->sk.sk_data_ready(&lsmc->sk);
1611         sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1612 }
1613
1614 /* listen worker: finish in state connected */
1615 static void smc_listen_out_connected(struct smc_sock *new_smc)
1616 {
1617         struct sock *newsmcsk = &new_smc->sk;
1618
1619         sk_refcnt_debug_inc(newsmcsk);
1620         if (newsmcsk->sk_state == SMC_INIT)
1621                 newsmcsk->sk_state = SMC_ACTIVE;
1622
1623         smc_listen_out(new_smc);
1624 }
1625
1626 /* listen worker: finish in error state */
1627 static void smc_listen_out_err(struct smc_sock *new_smc)
1628 {
1629         struct sock *newsmcsk = &new_smc->sk;
1630         struct net *net = sock_net(newsmcsk);
1631
1632         this_cpu_inc(net->smc.smc_stats->srv_hshake_err_cnt);
1633         if (newsmcsk->sk_state == SMC_INIT)
1634                 sock_put(&new_smc->sk); /* passive closing */
1635         newsmcsk->sk_state = SMC_CLOSED;
1636
1637         smc_listen_out(new_smc);
1638 }
1639
1640 /* listen worker: decline and fall back if possible */
1641 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1642                                int local_first, u8 version)
1643 {
1644         /* RDMA setup failed, switch back to TCP */
1645         smc_conn_abort(new_smc, local_first);
1646         if (reason_code < 0 ||
1647             smc_switch_to_fallback(new_smc, reason_code)) {
1648                 /* error, no fallback possible */
1649                 smc_listen_out_err(new_smc);
1650                 return;
1651         }
1652         if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1653                 if (smc_clc_send_decline(new_smc, reason_code, version) < 0) {
1654                         smc_listen_out_err(new_smc);
1655                         return;
1656                 }
1657         }
1658         smc_listen_out_connected(new_smc);
1659 }
1660
1661 /* listen worker: version checking */
1662 static int smc_listen_v2_check(struct smc_sock *new_smc,
1663                                struct smc_clc_msg_proposal *pclc,
1664                                struct smc_init_info *ini)
1665 {
1666         struct smc_clc_smcd_v2_extension *pclc_smcd_v2_ext;
1667         struct smc_clc_v2_extension *pclc_v2_ext;
1668         int rc = SMC_CLC_DECL_PEERNOSMC;
1669
1670         ini->smc_type_v1 = pclc->hdr.typev1;
1671         ini->smc_type_v2 = pclc->hdr.typev2;
1672         ini->smcd_version = smcd_indicated(ini->smc_type_v1) ? SMC_V1 : 0;
1673         ini->smcr_version = smcr_indicated(ini->smc_type_v1) ? SMC_V1 : 0;
1674         if (pclc->hdr.version > SMC_V1) {
1675                 if (smcd_indicated(ini->smc_type_v2))
1676                         ini->smcd_version |= SMC_V2;
1677                 if (smcr_indicated(ini->smc_type_v2))
1678                         ini->smcr_version |= SMC_V2;
1679         }
1680         if (!(ini->smcd_version & SMC_V2) && !(ini->smcr_version & SMC_V2)) {
1681                 rc = SMC_CLC_DECL_PEERNOSMC;
1682                 goto out;
1683         }
1684         pclc_v2_ext = smc_get_clc_v2_ext(pclc);
1685         if (!pclc_v2_ext) {
1686                 ini->smcd_version &= ~SMC_V2;
1687                 ini->smcr_version &= ~SMC_V2;
1688                 rc = SMC_CLC_DECL_NOV2EXT;
1689                 goto out;
1690         }
1691         pclc_smcd_v2_ext = smc_get_clc_smcd_v2_ext(pclc_v2_ext);
1692         if (ini->smcd_version & SMC_V2) {
1693                 if (!smc_ism_is_v2_capable()) {
1694                         ini->smcd_version &= ~SMC_V2;
1695                         rc = SMC_CLC_DECL_NOISM2SUPP;
1696                 } else if (!pclc_smcd_v2_ext) {
1697                         ini->smcd_version &= ~SMC_V2;
1698                         rc = SMC_CLC_DECL_NOV2DEXT;
1699                 } else if (!pclc_v2_ext->hdr.eid_cnt &&
1700                            !pclc_v2_ext->hdr.flag.seid) {
1701                         ini->smcd_version &= ~SMC_V2;
1702                         rc = SMC_CLC_DECL_NOUEID;
1703                 }
1704         }
1705         if (ini->smcr_version & SMC_V2) {
1706                 if (!pclc_v2_ext->hdr.eid_cnt) {
1707                         ini->smcr_version &= ~SMC_V2;
1708                         rc = SMC_CLC_DECL_NOUEID;
1709                 }
1710         }
1711
1712 out:
1713         if (!ini->smcd_version && !ini->smcr_version)
1714                 return rc;
1715
1716         return 0;
1717 }
1718
1719 /* listen worker: check prefixes */
1720 static int smc_listen_prfx_check(struct smc_sock *new_smc,
1721                                  struct smc_clc_msg_proposal *pclc)
1722 {
1723         struct smc_clc_msg_proposal_prefix *pclc_prfx;
1724         struct socket *newclcsock = new_smc->clcsock;
1725
1726         if (pclc->hdr.typev1 == SMC_TYPE_N)
1727                 return 0;
1728         pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1729         if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1730                 return SMC_CLC_DECL_DIFFPREFIX;
1731
1732         return 0;
1733 }
1734
1735 /* listen worker: initialize connection and buffers */
1736 static int smc_listen_rdma_init(struct smc_sock *new_smc,
1737                                 struct smc_init_info *ini)
1738 {
1739         int rc;
1740
1741         /* allocate connection / link group */
1742         rc = smc_conn_create(new_smc, ini);
1743         if (rc)
1744                 return rc;
1745
1746         /* create send buffer and rmb */
1747         if (smc_buf_create(new_smc, false))
1748                 return SMC_CLC_DECL_MEM;
1749
1750         return 0;
1751 }
1752
1753 /* listen worker: initialize connection and buffers for SMC-D */
1754 static int smc_listen_ism_init(struct smc_sock *new_smc,
1755                                struct smc_init_info *ini)
1756 {
1757         int rc;
1758
1759         rc = smc_conn_create(new_smc, ini);
1760         if (rc)
1761                 return rc;
1762
1763         /* Create send and receive buffers */
1764         rc = smc_buf_create(new_smc, true);
1765         if (rc) {
1766                 smc_conn_abort(new_smc, ini->first_contact_local);
1767                 return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB :
1768                                          SMC_CLC_DECL_MEM;
1769         }
1770
1771         return 0;
1772 }
1773
1774 static bool smc_is_already_selected(struct smcd_dev *smcd,
1775                                     struct smc_init_info *ini,
1776                                     int matches)
1777 {
1778         int i;
1779
1780         for (i = 0; i < matches; i++)
1781                 if (smcd == ini->ism_dev[i])
1782                         return true;
1783
1784         return false;
1785 }
1786
1787 /* check for ISM devices matching proposed ISM devices */
1788 static void smc_check_ism_v2_match(struct smc_init_info *ini,
1789                                    u16 proposed_chid, u64 proposed_gid,
1790                                    unsigned int *matches)
1791 {
1792         struct smcd_dev *smcd;
1793
1794         list_for_each_entry(smcd, &smcd_dev_list.list, list) {
1795                 if (smcd->going_away)
1796                         continue;
1797                 if (smc_is_already_selected(smcd, ini, *matches))
1798                         continue;
1799                 if (smc_ism_get_chid(smcd) == proposed_chid &&
1800                     !smc_ism_cantalk(proposed_gid, ISM_RESERVED_VLANID, smcd)) {
1801                         ini->ism_peer_gid[*matches] = proposed_gid;
1802                         ini->ism_dev[*matches] = smcd;
1803                         (*matches)++;
1804                         break;
1805                 }
1806         }
1807 }
1808
1809 static void smc_find_ism_store_rc(u32 rc, struct smc_init_info *ini)
1810 {
1811         if (!ini->rc)
1812                 ini->rc = rc;
1813 }
1814
1815 static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
1816                                         struct smc_clc_msg_proposal *pclc,
1817                                         struct smc_init_info *ini)
1818 {
1819         struct smc_clc_smcd_v2_extension *smcd_v2_ext;
1820         struct smc_clc_v2_extension *smc_v2_ext;
1821         struct smc_clc_msg_smcd *pclc_smcd;
1822         unsigned int matches = 0;
1823         u8 smcd_version;
1824         u8 *eid = NULL;
1825         int i, rc;
1826
1827         if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2))
1828                 goto not_found;
1829
1830         pclc_smcd = smc_get_clc_msg_smcd(pclc);
1831         smc_v2_ext = smc_get_clc_v2_ext(pclc);
1832         smcd_v2_ext = smc_get_clc_smcd_v2_ext(smc_v2_ext);
1833
1834         mutex_lock(&smcd_dev_list.mutex);
1835         if (pclc_smcd->ism.chid)
1836                 /* check for ISM device matching proposed native ISM device */
1837                 smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid),
1838                                        ntohll(pclc_smcd->ism.gid), &matches);
1839         for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) {
1840                 /* check for ISM devices matching proposed non-native ISM
1841                  * devices
1842                  */
1843                 smc_check_ism_v2_match(ini,
1844                                        ntohs(smcd_v2_ext->gidchid[i - 1].chid),
1845                                        ntohll(smcd_v2_ext->gidchid[i - 1].gid),
1846                                        &matches);
1847         }
1848         mutex_unlock(&smcd_dev_list.mutex);
1849
1850         if (!ini->ism_dev[0]) {
1851                 smc_find_ism_store_rc(SMC_CLC_DECL_NOSMCD2DEV, ini);
1852                 goto not_found;
1853         }
1854
1855         smc_ism_get_system_eid(&eid);
1856         if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext,
1857                                smcd_v2_ext->system_eid, eid))
1858                 goto not_found;
1859
1860         /* separate - outside the smcd_dev_list.lock */
1861         smcd_version = ini->smcd_version;
1862         for (i = 0; i < matches; i++) {
1863                 ini->smcd_version = SMC_V2;
1864                 ini->is_smcd = true;
1865                 ini->ism_selected = i;
1866                 rc = smc_listen_ism_init(new_smc, ini);
1867                 if (rc) {
1868                         smc_find_ism_store_rc(rc, ini);
1869                         /* try next active ISM device */
1870                         continue;
1871                 }
1872                 return; /* matching and usable V2 ISM device found */
1873         }
1874         /* no V2 ISM device could be initialized */
1875         ini->smcd_version = smcd_version;       /* restore original value */
1876         ini->negotiated_eid[0] = 0;
1877
1878 not_found:
1879         ini->smcd_version &= ~SMC_V2;
1880         ini->ism_dev[0] = NULL;
1881         ini->is_smcd = false;
1882 }
1883
1884 static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc,
1885                                         struct smc_clc_msg_proposal *pclc,
1886                                         struct smc_init_info *ini)
1887 {
1888         struct smc_clc_msg_smcd *pclc_smcd = smc_get_clc_msg_smcd(pclc);
1889         int rc = 0;
1890
1891         /* check if ISM V1 is available */
1892         if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1))
1893                 goto not_found;
1894         ini->is_smcd = true; /* prepare ISM check */
1895         ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid);
1896         rc = smc_find_ism_device(new_smc, ini);
1897         if (rc)
1898                 goto not_found;
1899         ini->ism_selected = 0;
1900         rc = smc_listen_ism_init(new_smc, ini);
1901         if (!rc)
1902                 return;         /* V1 ISM device found */
1903
1904 not_found:
1905         smc_find_ism_store_rc(rc, ini);
1906         ini->smcd_version &= ~SMC_V1;
1907         ini->ism_dev[0] = NULL;
1908         ini->is_smcd = false;
1909 }
1910
1911 /* listen worker: register buffers */
1912 static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first)
1913 {
1914         struct smc_connection *conn = &new_smc->conn;
1915
1916         if (!local_first) {
1917                 if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc))
1918                         return SMC_CLC_DECL_ERR_REGRMB;
1919         }
1920         smc_rmb_sync_sg_for_device(&new_smc->conn);
1921
1922         return 0;
1923 }
1924
1925 static void smc_find_rdma_v2_device_serv(struct smc_sock *new_smc,
1926                                          struct smc_clc_msg_proposal *pclc,
1927                                          struct smc_init_info *ini)
1928 {
1929         struct smc_clc_v2_extension *smc_v2_ext;
1930         u8 smcr_version;
1931         int rc;
1932
1933         if (!(ini->smcr_version & SMC_V2) || !smcr_indicated(ini->smc_type_v2))
1934                 goto not_found;
1935
1936         smc_v2_ext = smc_get_clc_v2_ext(pclc);
1937         if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext, NULL, NULL))
1938                 goto not_found;
1939
1940         /* prepare RDMA check */
1941         memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN);
1942         memcpy(ini->peer_gid, smc_v2_ext->roce, SMC_GID_SIZE);
1943         memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN);
1944         ini->check_smcrv2 = true;
1945         ini->smcrv2.clc_sk = new_smc->clcsock->sk;
1946         ini->smcrv2.saddr = new_smc->clcsock->sk->sk_rcv_saddr;
1947         ini->smcrv2.daddr = smc_ib_gid_to_ipv4(smc_v2_ext->roce);
1948         rc = smc_find_rdma_device(new_smc, ini);
1949         if (rc) {
1950                 smc_find_ism_store_rc(rc, ini);
1951                 goto not_found;
1952         }
1953         if (!ini->smcrv2.uses_gateway)
1954                 memcpy(ini->smcrv2.nexthop_mac, pclc->lcl.mac, ETH_ALEN);
1955
1956         smcr_version = ini->smcr_version;
1957         ini->smcr_version = SMC_V2;
1958         rc = smc_listen_rdma_init(new_smc, ini);
1959         if (!rc)
1960                 rc = smc_listen_rdma_reg(new_smc, ini->first_contact_local);
1961         if (!rc)
1962                 return;
1963         ini->smcr_version = smcr_version;
1964         smc_find_ism_store_rc(rc, ini);
1965
1966 not_found:
1967         ini->smcr_version &= ~SMC_V2;
1968         ini->check_smcrv2 = false;
1969 }
1970
1971 static int smc_find_rdma_v1_device_serv(struct smc_sock *new_smc,
1972                                         struct smc_clc_msg_proposal *pclc,
1973                                         struct smc_init_info *ini)
1974 {
1975         int rc;
1976
1977         if (!(ini->smcr_version & SMC_V1) || !smcr_indicated(ini->smc_type_v1))
1978                 return SMC_CLC_DECL_NOSMCDEV;
1979
1980         /* prepare RDMA check */
1981         memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN);
1982         memcpy(ini->peer_gid, pclc->lcl.gid, SMC_GID_SIZE);
1983         memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN);
1984         rc = smc_find_rdma_device(new_smc, ini);
1985         if (rc) {
1986                 /* no RDMA device found */
1987                 return SMC_CLC_DECL_NOSMCDEV;
1988         }
1989         rc = smc_listen_rdma_init(new_smc, ini);
1990         if (rc)
1991                 return rc;
1992         return smc_listen_rdma_reg(new_smc, ini->first_contact_local);
1993 }
1994
1995 /* determine the local device matching to proposal */
1996 static int smc_listen_find_device(struct smc_sock *new_smc,
1997                                   struct smc_clc_msg_proposal *pclc,
1998                                   struct smc_init_info *ini)
1999 {
2000         int prfx_rc;
2001
2002         /* check for ISM device matching V2 proposed device */
2003         smc_find_ism_v2_device_serv(new_smc, pclc, ini);
2004         if (ini->ism_dev[0])
2005                 return 0;
2006
2007         /* check for matching IP prefix and subnet length (V1) */
2008         prfx_rc = smc_listen_prfx_check(new_smc, pclc);
2009         if (prfx_rc)
2010                 smc_find_ism_store_rc(prfx_rc, ini);
2011
2012         /* get vlan id from IP device */
2013         if (smc_vlan_by_tcpsk(new_smc->clcsock, ini))
2014                 return ini->rc ?: SMC_CLC_DECL_GETVLANERR;
2015
2016         /* check for ISM device matching V1 proposed device */
2017         if (!prfx_rc)
2018                 smc_find_ism_v1_device_serv(new_smc, pclc, ini);
2019         if (ini->ism_dev[0])
2020                 return 0;
2021
2022         if (!smcr_indicated(pclc->hdr.typev1) &&
2023             !smcr_indicated(pclc->hdr.typev2))
2024                 /* skip RDMA and decline */
2025                 return ini->rc ?: SMC_CLC_DECL_NOSMCDDEV;
2026
2027         /* check if RDMA V2 is available */
2028         smc_find_rdma_v2_device_serv(new_smc, pclc, ini);
2029         if (ini->smcrv2.ib_dev_v2)
2030                 return 0;
2031
2032         /* check if RDMA V1 is available */
2033         if (!prfx_rc) {
2034                 int rc;
2035
2036                 rc = smc_find_rdma_v1_device_serv(new_smc, pclc, ini);
2037                 smc_find_ism_store_rc(rc, ini);
2038                 return (!rc) ? 0 : ini->rc;
2039         }
2040         return SMC_CLC_DECL_NOSMCDEV;
2041 }
2042
2043 /* listen worker: finish RDMA setup */
2044 static int smc_listen_rdma_finish(struct smc_sock *new_smc,
2045                                   struct smc_clc_msg_accept_confirm *cclc,
2046                                   bool local_first,
2047                                   struct smc_init_info *ini)
2048 {
2049         struct smc_link *link = new_smc->conn.lnk;
2050         int reason_code = 0;
2051
2052         if (local_first)
2053                 smc_link_save_peer_info(link, cclc, ini);
2054
2055         if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc))
2056                 return SMC_CLC_DECL_ERR_RTOK;
2057
2058         if (local_first) {
2059                 if (smc_ib_ready_link(link))
2060                         return SMC_CLC_DECL_ERR_RDYLNK;
2061                 /* QP confirmation over RoCE fabric */
2062                 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
2063                 reason_code = smcr_serv_conf_first_link(new_smc);
2064                 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
2065         }
2066         return reason_code;
2067 }
2068
2069 /* setup for connection of server */
2070 static void smc_listen_work(struct work_struct *work)
2071 {
2072         struct smc_sock *new_smc = container_of(work, struct smc_sock,
2073                                                 smc_listen_work);
2074         struct socket *newclcsock = new_smc->clcsock;
2075         struct smc_clc_msg_accept_confirm *cclc;
2076         struct smc_clc_msg_proposal_area *buf;
2077         struct smc_clc_msg_proposal *pclc;
2078         struct smc_init_info *ini = NULL;
2079         u8 proposal_version = SMC_V1;
2080         u8 accept_version;
2081         int rc = 0;
2082
2083         if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
2084                 return smc_listen_out_err(new_smc);
2085
2086         if (new_smc->use_fallback) {
2087                 smc_listen_out_connected(new_smc);
2088                 return;
2089         }
2090
2091         /* check if peer is smc capable */
2092         if (!tcp_sk(newclcsock->sk)->syn_smc) {
2093                 rc = smc_switch_to_fallback(new_smc, SMC_CLC_DECL_PEERNOSMC);
2094                 if (rc)
2095                         smc_listen_out_err(new_smc);
2096                 else
2097                         smc_listen_out_connected(new_smc);
2098                 return;
2099         }
2100
2101         /* do inband token exchange -
2102          * wait for and receive SMC Proposal CLC message
2103          */
2104         buf = kzalloc(sizeof(*buf), GFP_KERNEL);
2105         if (!buf) {
2106                 rc = SMC_CLC_DECL_MEM;
2107                 goto out_decl;
2108         }
2109         pclc = (struct smc_clc_msg_proposal *)buf;
2110         rc = smc_clc_wait_msg(new_smc, pclc, sizeof(*buf),
2111                               SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
2112         if (rc)
2113                 goto out_decl;
2114
2115         if (pclc->hdr.version > SMC_V1)
2116                 proposal_version = SMC_V2;
2117
2118         /* IPSec connections opt out of SMC optimizations */
2119         if (using_ipsec(new_smc)) {
2120                 rc = SMC_CLC_DECL_IPSEC;
2121                 goto out_decl;
2122         }
2123
2124         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
2125         if (!ini) {
2126                 rc = SMC_CLC_DECL_MEM;
2127                 goto out_decl;
2128         }
2129
2130         /* initial version checking */
2131         rc = smc_listen_v2_check(new_smc, pclc, ini);
2132         if (rc)
2133                 goto out_decl;
2134
2135         mutex_lock(&smc_server_lgr_pending);
2136         smc_close_init(new_smc);
2137         smc_rx_init(new_smc);
2138         smc_tx_init(new_smc);
2139
2140         /* determine ISM or RoCE device used for connection */
2141         rc = smc_listen_find_device(new_smc, pclc, ini);
2142         if (rc)
2143                 goto out_unlock;
2144
2145         /* send SMC Accept CLC message */
2146         accept_version = ini->is_smcd ? ini->smcd_version : ini->smcr_version;
2147         rc = smc_clc_send_accept(new_smc, ini->first_contact_local,
2148                                  accept_version, ini->negotiated_eid);
2149         if (rc)
2150                 goto out_unlock;
2151
2152         /* SMC-D does not need this lock any more */
2153         if (ini->is_smcd)
2154                 mutex_unlock(&smc_server_lgr_pending);
2155
2156         /* receive SMC Confirm CLC message */
2157         memset(buf, 0, sizeof(*buf));
2158         cclc = (struct smc_clc_msg_accept_confirm *)buf;
2159         rc = smc_clc_wait_msg(new_smc, cclc, sizeof(*buf),
2160                               SMC_CLC_CONFIRM, CLC_WAIT_TIME);
2161         if (rc) {
2162                 if (!ini->is_smcd)
2163                         goto out_unlock;
2164                 goto out_decl;
2165         }
2166
2167         /* finish worker */
2168         if (!ini->is_smcd) {
2169                 rc = smc_listen_rdma_finish(new_smc, cclc,
2170                                             ini->first_contact_local, ini);
2171                 if (rc)
2172                         goto out_unlock;
2173                 mutex_unlock(&smc_server_lgr_pending);
2174         }
2175         smc_conn_save_peer_info(new_smc, cclc);
2176         smc_listen_out_connected(new_smc);
2177         SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini);
2178         goto out_free;
2179
2180 out_unlock:
2181         mutex_unlock(&smc_server_lgr_pending);
2182 out_decl:
2183         smc_listen_decline(new_smc, rc, ini ? ini->first_contact_local : 0,
2184                            proposal_version);
2185 out_free:
2186         kfree(ini);
2187         kfree(buf);
2188 }
2189
2190 static void smc_tcp_listen_work(struct work_struct *work)
2191 {
2192         struct smc_sock *lsmc = container_of(work, struct smc_sock,
2193                                              tcp_listen_work);
2194         struct sock *lsk = &lsmc->sk;
2195         struct smc_sock *new_smc;
2196         int rc = 0;
2197
2198         lock_sock(lsk);
2199         while (lsk->sk_state == SMC_LISTEN) {
2200                 rc = smc_clcsock_accept(lsmc, &new_smc);
2201                 if (rc) /* clcsock accept queue empty or error */
2202                         goto out;
2203                 if (!new_smc)
2204                         continue;
2205
2206                 new_smc->listen_smc = lsmc;
2207                 new_smc->use_fallback = lsmc->use_fallback;
2208                 new_smc->fallback_rsn = lsmc->fallback_rsn;
2209                 sock_hold(lsk); /* sock_put in smc_listen_work */
2210                 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
2211                 smc_copy_sock_settings_to_smc(new_smc);
2212                 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
2213                 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
2214                 sock_hold(&new_smc->sk); /* sock_put in passive closing */
2215                 if (!queue_work(smc_hs_wq, &new_smc->smc_listen_work))
2216                         sock_put(&new_smc->sk);
2217         }
2218
2219 out:
2220         release_sock(lsk);
2221         sock_put(&lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */
2222 }
2223
2224 static void smc_clcsock_data_ready(struct sock *listen_clcsock)
2225 {
2226         struct smc_sock *lsmc =
2227                 smc_clcsock_user_data(listen_clcsock);
2228
2229         if (!lsmc)
2230                 return;
2231         lsmc->clcsk_data_ready(listen_clcsock);
2232         if (lsmc->sk.sk_state == SMC_LISTEN) {
2233                 sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */
2234                 if (!queue_work(smc_hs_wq, &lsmc->tcp_listen_work))
2235                         sock_put(&lsmc->sk);
2236         }
2237 }
2238
2239 static int smc_listen(struct socket *sock, int backlog)
2240 {
2241         struct sock *sk = sock->sk;
2242         struct smc_sock *smc;
2243         int rc;
2244
2245         smc = smc_sk(sk);
2246         lock_sock(sk);
2247
2248         rc = -EINVAL;
2249         if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
2250             smc->connect_nonblock)
2251                 goto out;
2252
2253         rc = 0;
2254         if (sk->sk_state == SMC_LISTEN) {
2255                 sk->sk_max_ack_backlog = backlog;
2256                 goto out;
2257         }
2258         /* some socket options are handled in core, so we could not apply
2259          * them to the clc socket -- copy smc socket options to clc socket
2260          */
2261         smc_copy_sock_settings_to_clc(smc);
2262         if (!smc->use_fallback)
2263                 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
2264
2265         /* save original sk_data_ready function and establish
2266          * smc-specific sk_data_ready function
2267          */
2268         smc->clcsk_data_ready = smc->clcsock->sk->sk_data_ready;
2269         smc->clcsock->sk->sk_data_ready = smc_clcsock_data_ready;
2270         smc->clcsock->sk->sk_user_data =
2271                 (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
2272         rc = kernel_listen(smc->clcsock, backlog);
2273         if (rc) {
2274                 smc->clcsock->sk->sk_data_ready = smc->clcsk_data_ready;
2275                 goto out;
2276         }
2277         sk->sk_max_ack_backlog = backlog;
2278         sk->sk_ack_backlog = 0;
2279         sk->sk_state = SMC_LISTEN;
2280
2281 out:
2282         release_sock(sk);
2283         return rc;
2284 }
2285
2286 static int smc_accept(struct socket *sock, struct socket *new_sock,
2287                       int flags, bool kern)
2288 {
2289         struct sock *sk = sock->sk, *nsk;
2290         DECLARE_WAITQUEUE(wait, current);
2291         struct smc_sock *lsmc;
2292         long timeo;
2293         int rc = 0;
2294
2295         lsmc = smc_sk(sk);
2296         sock_hold(sk); /* sock_put below */
2297         lock_sock(sk);
2298
2299         if (lsmc->sk.sk_state != SMC_LISTEN) {
2300                 rc = -EINVAL;
2301                 release_sock(sk);
2302                 goto out;
2303         }
2304
2305         /* Wait for an incoming connection */
2306         timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
2307         add_wait_queue_exclusive(sk_sleep(sk), &wait);
2308         while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
2309                 set_current_state(TASK_INTERRUPTIBLE);
2310                 if (!timeo) {
2311                         rc = -EAGAIN;
2312                         break;
2313                 }
2314                 release_sock(sk);
2315                 timeo = schedule_timeout(timeo);
2316                 /* wakeup by sk_data_ready in smc_listen_work() */
2317                 sched_annotate_sleep();
2318                 lock_sock(sk);
2319                 if (signal_pending(current)) {
2320                         rc = sock_intr_errno(timeo);
2321                         break;
2322                 }
2323         }
2324         set_current_state(TASK_RUNNING);
2325         remove_wait_queue(sk_sleep(sk), &wait);
2326
2327         if (!rc)
2328                 rc = sock_error(nsk);
2329         release_sock(sk);
2330         if (rc)
2331                 goto out;
2332
2333         if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
2334                 /* wait till data arrives on the socket */
2335                 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
2336                                                                 MSEC_PER_SEC);
2337                 if (smc_sk(nsk)->use_fallback) {
2338                         struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
2339
2340                         lock_sock(clcsk);
2341                         if (skb_queue_empty(&clcsk->sk_receive_queue))
2342                                 sk_wait_data(clcsk, &timeo, NULL);
2343                         release_sock(clcsk);
2344                 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
2345                         lock_sock(nsk);
2346                         smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
2347                         release_sock(nsk);
2348                 }
2349         }
2350
2351 out:
2352         sock_put(sk); /* sock_hold above */
2353         return rc;
2354 }
2355
2356 static int smc_getname(struct socket *sock, struct sockaddr *addr,
2357                        int peer)
2358 {
2359         struct smc_sock *smc;
2360
2361         if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
2362             (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
2363                 return -ENOTCONN;
2364
2365         smc = smc_sk(sock->sk);
2366
2367         return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
2368 }
2369
2370 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
2371 {
2372         struct sock *sk = sock->sk;
2373         struct smc_sock *smc;
2374         int rc = -EPIPE;
2375
2376         smc = smc_sk(sk);
2377         lock_sock(sk);
2378         if ((sk->sk_state != SMC_ACTIVE) &&
2379             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2380             (sk->sk_state != SMC_INIT))
2381                 goto out;
2382
2383         if (msg->msg_flags & MSG_FASTOPEN) {
2384                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2385                         rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2386                         if (rc)
2387                                 goto out;
2388                 } else {
2389                         rc = -EINVAL;
2390                         goto out;
2391                 }
2392         }
2393
2394         if (smc->use_fallback) {
2395                 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
2396         } else {
2397                 rc = smc_tx_sendmsg(smc, msg, len);
2398                 SMC_STAT_TX_PAYLOAD(smc, len, rc);
2399         }
2400 out:
2401         release_sock(sk);
2402         return rc;
2403 }
2404
2405 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2406                        int flags)
2407 {
2408         struct sock *sk = sock->sk;
2409         struct smc_sock *smc;
2410         int rc = -ENOTCONN;
2411
2412         smc = smc_sk(sk);
2413         lock_sock(sk);
2414         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2415                 /* socket was connected before, no more data to read */
2416                 rc = 0;
2417                 goto out;
2418         }
2419         if ((sk->sk_state == SMC_INIT) ||
2420             (sk->sk_state == SMC_LISTEN) ||
2421             (sk->sk_state == SMC_CLOSED))
2422                 goto out;
2423
2424         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2425                 rc = 0;
2426                 goto out;
2427         }
2428
2429         if (smc->use_fallback) {
2430                 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
2431         } else {
2432                 msg->msg_namelen = 0;
2433                 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
2434                 SMC_STAT_RX_PAYLOAD(smc, rc, rc);
2435         }
2436
2437 out:
2438         release_sock(sk);
2439         return rc;
2440 }
2441
2442 static __poll_t smc_accept_poll(struct sock *parent)
2443 {
2444         struct smc_sock *isk = smc_sk(parent);
2445         __poll_t mask = 0;
2446
2447         spin_lock(&isk->accept_q_lock);
2448         if (!list_empty(&isk->accept_q))
2449                 mask = EPOLLIN | EPOLLRDNORM;
2450         spin_unlock(&isk->accept_q_lock);
2451
2452         return mask;
2453 }
2454
2455 static __poll_t smc_poll(struct file *file, struct socket *sock,
2456                              poll_table *wait)
2457 {
2458         struct sock *sk = sock->sk;
2459         struct smc_sock *smc;
2460         __poll_t mask = 0;
2461
2462         if (!sk)
2463                 return EPOLLNVAL;
2464
2465         smc = smc_sk(sock->sk);
2466         if (smc->use_fallback) {
2467                 /* delegate to CLC child sock */
2468                 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
2469                 sk->sk_err = smc->clcsock->sk->sk_err;
2470         } else {
2471                 if (sk->sk_state != SMC_CLOSED)
2472                         sock_poll_wait(file, sock, wait);
2473                 if (sk->sk_err)
2474                         mask |= EPOLLERR;
2475                 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
2476                     (sk->sk_state == SMC_CLOSED))
2477                         mask |= EPOLLHUP;
2478                 if (sk->sk_state == SMC_LISTEN) {
2479                         /* woken up by sk_data_ready in smc_listen_work() */
2480                         mask |= smc_accept_poll(sk);
2481                 } else if (smc->use_fallback) { /* as result of connect_work()*/
2482                         mask |= smc->clcsock->ops->poll(file, smc->clcsock,
2483                                                            wait);
2484                         sk->sk_err = smc->clcsock->sk->sk_err;
2485                 } else {
2486                         if ((sk->sk_state != SMC_INIT &&
2487                              atomic_read(&smc->conn.sndbuf_space)) ||
2488                             sk->sk_shutdown & SEND_SHUTDOWN) {
2489                                 mask |= EPOLLOUT | EPOLLWRNORM;
2490                         } else {
2491                                 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2492                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2493                         }
2494                         if (atomic_read(&smc->conn.bytes_to_rcv))
2495                                 mask |= EPOLLIN | EPOLLRDNORM;
2496                         if (sk->sk_shutdown & RCV_SHUTDOWN)
2497                                 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2498                         if (sk->sk_state == SMC_APPCLOSEWAIT1)
2499                                 mask |= EPOLLIN;
2500                         if (smc->conn.urg_state == SMC_URG_VALID)
2501                                 mask |= EPOLLPRI;
2502                 }
2503         }
2504
2505         return mask;
2506 }
2507
2508 static int smc_shutdown(struct socket *sock, int how)
2509 {
2510         struct sock *sk = sock->sk;
2511         bool do_shutdown = true;
2512         struct smc_sock *smc;
2513         int rc = -EINVAL;
2514         int old_state;
2515         int rc1 = 0;
2516
2517         smc = smc_sk(sk);
2518
2519         if ((how < SHUT_RD) || (how > SHUT_RDWR))
2520                 return rc;
2521
2522         lock_sock(sk);
2523
2524         rc = -ENOTCONN;
2525         if ((sk->sk_state != SMC_ACTIVE) &&
2526             (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
2527             (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
2528             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2529             (sk->sk_state != SMC_APPCLOSEWAIT2) &&
2530             (sk->sk_state != SMC_APPFINCLOSEWAIT))
2531                 goto out;
2532         if (smc->use_fallback) {
2533                 rc = kernel_sock_shutdown(smc->clcsock, how);
2534                 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
2535                 if (sk->sk_shutdown == SHUTDOWN_MASK)
2536                         sk->sk_state = SMC_CLOSED;
2537                 goto out;
2538         }
2539         switch (how) {
2540         case SHUT_RDWR:         /* shutdown in both directions */
2541                 old_state = sk->sk_state;
2542                 rc = smc_close_active(smc);
2543                 if (old_state == SMC_ACTIVE &&
2544                     sk->sk_state == SMC_PEERCLOSEWAIT1)
2545                         do_shutdown = false;
2546                 break;
2547         case SHUT_WR:
2548                 rc = smc_close_shutdown_write(smc);
2549                 break;
2550         case SHUT_RD:
2551                 rc = 0;
2552                 /* nothing more to do because peer is not involved */
2553                 break;
2554         }
2555         if (do_shutdown && smc->clcsock)
2556                 rc1 = kernel_sock_shutdown(smc->clcsock, how);
2557         /* map sock_shutdown_cmd constants to sk_shutdown value range */
2558         sk->sk_shutdown |= how + 1;
2559
2560 out:
2561         release_sock(sk);
2562         return rc ? rc : rc1;
2563 }
2564
2565 static int smc_setsockopt(struct socket *sock, int level, int optname,
2566                           sockptr_t optval, unsigned int optlen)
2567 {
2568         struct sock *sk = sock->sk;
2569         struct smc_sock *smc;
2570         int val, rc;
2571
2572         if (level == SOL_TCP && optname == TCP_ULP)
2573                 return -EOPNOTSUPP;
2574
2575         smc = smc_sk(sk);
2576
2577         /* generic setsockopts reaching us here always apply to the
2578          * CLC socket
2579          */
2580         mutex_lock(&smc->clcsock_release_lock);
2581         if (!smc->clcsock) {
2582                 mutex_unlock(&smc->clcsock_release_lock);
2583                 return -EBADF;
2584         }
2585         if (unlikely(!smc->clcsock->ops->setsockopt))
2586                 rc = -EOPNOTSUPP;
2587         else
2588                 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
2589                                                    optval, optlen);
2590         if (smc->clcsock->sk->sk_err) {
2591                 sk->sk_err = smc->clcsock->sk->sk_err;
2592                 sk_error_report(sk);
2593         }
2594         mutex_unlock(&smc->clcsock_release_lock);
2595
2596         if (optlen < sizeof(int))
2597                 return -EINVAL;
2598         if (copy_from_sockptr(&val, optval, sizeof(int)))
2599                 return -EFAULT;
2600
2601         lock_sock(sk);
2602         if (rc || smc->use_fallback)
2603                 goto out;
2604         switch (optname) {
2605         case TCP_FASTOPEN:
2606         case TCP_FASTOPEN_CONNECT:
2607         case TCP_FASTOPEN_KEY:
2608         case TCP_FASTOPEN_NO_COOKIE:
2609                 /* option not supported by SMC */
2610                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2611                         rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2612                 } else {
2613                         rc = -EINVAL;
2614                 }
2615                 break;
2616         case TCP_NODELAY:
2617                 if (sk->sk_state != SMC_INIT &&
2618                     sk->sk_state != SMC_LISTEN &&
2619                     sk->sk_state != SMC_CLOSED) {
2620                         if (val) {
2621                                 SMC_STAT_INC(smc, ndly_cnt);
2622                                 mod_delayed_work(smc->conn.lgr->tx_wq,
2623                                                  &smc->conn.tx_work, 0);
2624                         }
2625                 }
2626                 break;
2627         case TCP_CORK:
2628                 if (sk->sk_state != SMC_INIT &&
2629                     sk->sk_state != SMC_LISTEN &&
2630                     sk->sk_state != SMC_CLOSED) {
2631                         if (!val) {
2632                                 SMC_STAT_INC(smc, cork_cnt);
2633                                 mod_delayed_work(smc->conn.lgr->tx_wq,
2634                                                  &smc->conn.tx_work, 0);
2635                         }
2636                 }
2637                 break;
2638         case TCP_DEFER_ACCEPT:
2639                 smc->sockopt_defer_accept = val;
2640                 break;
2641         default:
2642                 break;
2643         }
2644 out:
2645         release_sock(sk);
2646
2647         return rc;
2648 }
2649
2650 static int smc_getsockopt(struct socket *sock, int level, int optname,
2651                           char __user *optval, int __user *optlen)
2652 {
2653         struct smc_sock *smc;
2654         int rc;
2655
2656         smc = smc_sk(sock->sk);
2657         mutex_lock(&smc->clcsock_release_lock);
2658         if (!smc->clcsock) {
2659                 mutex_unlock(&smc->clcsock_release_lock);
2660                 return -EBADF;
2661         }
2662         /* socket options apply to the CLC socket */
2663         if (unlikely(!smc->clcsock->ops->getsockopt)) {
2664                 mutex_unlock(&smc->clcsock_release_lock);
2665                 return -EOPNOTSUPP;
2666         }
2667         rc = smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
2668                                            optval, optlen);
2669         mutex_unlock(&smc->clcsock_release_lock);
2670         return rc;
2671 }
2672
2673 static int smc_ioctl(struct socket *sock, unsigned int cmd,
2674                      unsigned long arg)
2675 {
2676         union smc_host_cursor cons, urg;
2677         struct smc_connection *conn;
2678         struct smc_sock *smc;
2679         int answ;
2680
2681         smc = smc_sk(sock->sk);
2682         conn = &smc->conn;
2683         lock_sock(&smc->sk);
2684         if (smc->use_fallback) {
2685                 if (!smc->clcsock) {
2686                         release_sock(&smc->sk);
2687                         return -EBADF;
2688                 }
2689                 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
2690                 release_sock(&smc->sk);
2691                 return answ;
2692         }
2693         switch (cmd) {
2694         case SIOCINQ: /* same as FIONREAD */
2695                 if (smc->sk.sk_state == SMC_LISTEN) {
2696                         release_sock(&smc->sk);
2697                         return -EINVAL;
2698                 }
2699                 if (smc->sk.sk_state == SMC_INIT ||
2700                     smc->sk.sk_state == SMC_CLOSED)
2701                         answ = 0;
2702                 else
2703                         answ = atomic_read(&smc->conn.bytes_to_rcv);
2704                 break;
2705         case SIOCOUTQ:
2706                 /* output queue size (not send + not acked) */
2707                 if (smc->sk.sk_state == SMC_LISTEN) {
2708                         release_sock(&smc->sk);
2709                         return -EINVAL;
2710                 }
2711                 if (smc->sk.sk_state == SMC_INIT ||
2712                     smc->sk.sk_state == SMC_CLOSED)
2713                         answ = 0;
2714                 else
2715                         answ = smc->conn.sndbuf_desc->len -
2716                                         atomic_read(&smc->conn.sndbuf_space);
2717                 break;
2718         case SIOCOUTQNSD:
2719                 /* output queue size (not send only) */
2720                 if (smc->sk.sk_state == SMC_LISTEN) {
2721                         release_sock(&smc->sk);
2722                         return -EINVAL;
2723                 }
2724                 if (smc->sk.sk_state == SMC_INIT ||
2725                     smc->sk.sk_state == SMC_CLOSED)
2726                         answ = 0;
2727                 else
2728                         answ = smc_tx_prepared_sends(&smc->conn);
2729                 break;
2730         case SIOCATMARK:
2731                 if (smc->sk.sk_state == SMC_LISTEN) {
2732                         release_sock(&smc->sk);
2733                         return -EINVAL;
2734                 }
2735                 if (smc->sk.sk_state == SMC_INIT ||
2736                     smc->sk.sk_state == SMC_CLOSED) {
2737                         answ = 0;
2738                 } else {
2739                         smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
2740                         smc_curs_copy(&urg, &conn->urg_curs, conn);
2741                         answ = smc_curs_diff(conn->rmb_desc->len,
2742                                              &cons, &urg) == 1;
2743                 }
2744                 break;
2745         default:
2746                 release_sock(&smc->sk);
2747                 return -ENOIOCTLCMD;
2748         }
2749         release_sock(&smc->sk);
2750
2751         return put_user(answ, (int __user *)arg);
2752 }
2753
2754 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
2755                             int offset, size_t size, int flags)
2756 {
2757         struct sock *sk = sock->sk;
2758         struct smc_sock *smc;
2759         int rc = -EPIPE;
2760
2761         smc = smc_sk(sk);
2762         lock_sock(sk);
2763         if (sk->sk_state != SMC_ACTIVE) {
2764                 release_sock(sk);
2765                 goto out;
2766         }
2767         release_sock(sk);
2768         if (smc->use_fallback) {
2769                 rc = kernel_sendpage(smc->clcsock, page, offset,
2770                                      size, flags);
2771         } else {
2772                 SMC_STAT_INC(smc, sendpage_cnt);
2773                 rc = sock_no_sendpage(sock, page, offset, size, flags);
2774         }
2775
2776 out:
2777         return rc;
2778 }
2779
2780 /* Map the affected portions of the rmbe into an spd, note the number of bytes
2781  * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
2782  * updates till whenever a respective page has been fully processed.
2783  * Note that subsequent recv() calls have to wait till all splice() processing
2784  * completed.
2785  */
2786 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
2787                                struct pipe_inode_info *pipe, size_t len,
2788                                unsigned int flags)
2789 {
2790         struct sock *sk = sock->sk;
2791         struct smc_sock *smc;
2792         int rc = -ENOTCONN;
2793
2794         smc = smc_sk(sk);
2795         lock_sock(sk);
2796         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2797                 /* socket was connected before, no more data to read */
2798                 rc = 0;
2799                 goto out;
2800         }
2801         if (sk->sk_state == SMC_INIT ||
2802             sk->sk_state == SMC_LISTEN ||
2803             sk->sk_state == SMC_CLOSED)
2804                 goto out;
2805
2806         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2807                 rc = 0;
2808                 goto out;
2809         }
2810
2811         if (smc->use_fallback) {
2812                 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
2813                                                     pipe, len, flags);
2814         } else {
2815                 if (*ppos) {
2816                         rc = -ESPIPE;
2817                         goto out;
2818                 }
2819                 if (flags & SPLICE_F_NONBLOCK)
2820                         flags = MSG_DONTWAIT;
2821                 else
2822                         flags = 0;
2823                 SMC_STAT_INC(smc, splice_cnt);
2824                 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
2825         }
2826 out:
2827         release_sock(sk);
2828
2829         return rc;
2830 }
2831
2832 /* must look like tcp */
2833 static const struct proto_ops smc_sock_ops = {
2834         .family         = PF_SMC,
2835         .owner          = THIS_MODULE,
2836         .release        = smc_release,
2837         .bind           = smc_bind,
2838         .connect        = smc_connect,
2839         .socketpair     = sock_no_socketpair,
2840         .accept         = smc_accept,
2841         .getname        = smc_getname,
2842         .poll           = smc_poll,
2843         .ioctl          = smc_ioctl,
2844         .listen         = smc_listen,
2845         .shutdown       = smc_shutdown,
2846         .setsockopt     = smc_setsockopt,
2847         .getsockopt     = smc_getsockopt,
2848         .sendmsg        = smc_sendmsg,
2849         .recvmsg        = smc_recvmsg,
2850         .mmap           = sock_no_mmap,
2851         .sendpage       = smc_sendpage,
2852         .splice_read    = smc_splice_read,
2853 };
2854
2855 static int __smc_create(struct net *net, struct socket *sock, int protocol,
2856                         int kern, struct socket *clcsock)
2857 {
2858         int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
2859         struct smc_sock *smc;
2860         struct sock *sk;
2861         int rc;
2862
2863         rc = -ESOCKTNOSUPPORT;
2864         if (sock->type != SOCK_STREAM)
2865                 goto out;
2866
2867         rc = -EPROTONOSUPPORT;
2868         if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
2869                 goto out;
2870
2871         rc = -ENOBUFS;
2872         sock->ops = &smc_sock_ops;
2873         sk = smc_sock_alloc(net, sock, protocol);
2874         if (!sk)
2875                 goto out;
2876
2877         /* create internal TCP socket for CLC handshake and fallback */
2878         smc = smc_sk(sk);
2879         smc->use_fallback = false; /* assume rdma capability first */
2880         smc->fallback_rsn = 0;
2881
2882         rc = 0;
2883         if (!clcsock) {
2884                 rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
2885                                       &smc->clcsock);
2886                 if (rc) {
2887                         sk_common_release(sk);
2888                         goto out;
2889                 }
2890         } else {
2891                 smc->clcsock = clcsock;
2892         }
2893
2894         smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
2895         smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
2896
2897 out:
2898         return rc;
2899 }
2900
2901 static int smc_create(struct net *net, struct socket *sock, int protocol,
2902                       int kern)
2903 {
2904         return __smc_create(net, sock, protocol, kern, NULL);
2905 }
2906
2907 static const struct net_proto_family smc_sock_family_ops = {
2908         .family = PF_SMC,
2909         .owner  = THIS_MODULE,
2910         .create = smc_create,
2911 };
2912
2913 static int smc_ulp_init(struct sock *sk)
2914 {
2915         struct socket *tcp = sk->sk_socket;
2916         struct net *net = sock_net(sk);
2917         struct socket *smcsock;
2918         int protocol, ret;
2919
2920         /* only TCP can be replaced */
2921         if (tcp->type != SOCK_STREAM || sk->sk_protocol != IPPROTO_TCP ||
2922             (sk->sk_family != AF_INET && sk->sk_family != AF_INET6))
2923                 return -ESOCKTNOSUPPORT;
2924         /* don't handle wq now */
2925         if (tcp->state != SS_UNCONNECTED || !tcp->file || tcp->wq.fasync_list)
2926                 return -ENOTCONN;
2927
2928         if (sk->sk_family == AF_INET)
2929                 protocol = SMCPROTO_SMC;
2930         else
2931                 protocol = SMCPROTO_SMC6;
2932
2933         smcsock = sock_alloc();
2934         if (!smcsock)
2935                 return -ENFILE;
2936
2937         smcsock->type = SOCK_STREAM;
2938         __module_get(THIS_MODULE); /* tried in __tcp_ulp_find_autoload */
2939         ret = __smc_create(net, smcsock, protocol, 1, tcp);
2940         if (ret) {
2941                 sock_release(smcsock); /* module_put() which ops won't be NULL */
2942                 return ret;
2943         }
2944
2945         /* replace tcp socket to smc */
2946         smcsock->file = tcp->file;
2947         smcsock->file->private_data = smcsock;
2948         smcsock->file->f_inode = SOCK_INODE(smcsock); /* replace inode when sock_close */
2949         smcsock->file->f_path.dentry->d_inode = SOCK_INODE(smcsock); /* dput() in __fput */
2950         tcp->file = NULL;
2951
2952         return ret;
2953 }
2954
2955 static void smc_ulp_clone(const struct request_sock *req, struct sock *newsk,
2956                           const gfp_t priority)
2957 {
2958         struct inet_connection_sock *icsk = inet_csk(newsk);
2959
2960         /* don't inherit ulp ops to child when listen */
2961         icsk->icsk_ulp_ops = NULL;
2962 }
2963
2964 static struct tcp_ulp_ops smc_ulp_ops __read_mostly = {
2965         .name           = "smc",
2966         .owner          = THIS_MODULE,
2967         .init           = smc_ulp_init,
2968         .clone          = smc_ulp_clone,
2969 };
2970
2971 unsigned int smc_net_id;
2972
2973 static __net_init int smc_net_init(struct net *net)
2974 {
2975         return smc_pnet_net_init(net);
2976 }
2977
2978 static void __net_exit smc_net_exit(struct net *net)
2979 {
2980         smc_pnet_net_exit(net);
2981 }
2982
2983 static __net_init int smc_net_stat_init(struct net *net)
2984 {
2985         return smc_stats_init(net);
2986 }
2987
2988 static void __net_exit smc_net_stat_exit(struct net *net)
2989 {
2990         smc_stats_exit(net);
2991 }
2992
2993 static struct pernet_operations smc_net_ops = {
2994         .init = smc_net_init,
2995         .exit = smc_net_exit,
2996         .id   = &smc_net_id,
2997         .size = sizeof(struct smc_net),
2998 };
2999
3000 static struct pernet_operations smc_net_stat_ops = {
3001         .init = smc_net_stat_init,
3002         .exit = smc_net_stat_exit,
3003 };
3004
3005 static int __init smc_init(void)
3006 {
3007         int rc;
3008
3009         rc = register_pernet_subsys(&smc_net_ops);
3010         if (rc)
3011                 return rc;
3012
3013         rc = register_pernet_subsys(&smc_net_stat_ops);
3014         if (rc)
3015                 return rc;
3016
3017         smc_ism_init();
3018         smc_clc_init();
3019
3020         rc = smc_nl_init();
3021         if (rc)
3022                 goto out_pernet_subsys;
3023
3024         rc = smc_pnet_init();
3025         if (rc)
3026                 goto out_nl;
3027
3028         rc = -ENOMEM;
3029         smc_hs_wq = alloc_workqueue("smc_hs_wq", 0, 0);
3030         if (!smc_hs_wq)
3031                 goto out_pnet;
3032
3033         smc_close_wq = alloc_workqueue("smc_close_wq", 0, 0);
3034         if (!smc_close_wq)
3035                 goto out_alloc_hs_wq;
3036
3037         rc = smc_core_init();
3038         if (rc) {
3039                 pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
3040                 goto out_alloc_wqs;
3041         }
3042
3043         rc = smc_llc_init();
3044         if (rc) {
3045                 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
3046                 goto out_core;
3047         }
3048
3049         rc = smc_cdc_init();
3050         if (rc) {
3051                 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
3052                 goto out_core;
3053         }
3054
3055         rc = proto_register(&smc_proto, 1);
3056         if (rc) {
3057                 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
3058                 goto out_core;
3059         }
3060
3061         rc = proto_register(&smc_proto6, 1);
3062         if (rc) {
3063                 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
3064                 goto out_proto;
3065         }
3066
3067         rc = sock_register(&smc_sock_family_ops);
3068         if (rc) {
3069                 pr_err("%s: sock_register fails with %d\n", __func__, rc);
3070                 goto out_proto6;
3071         }
3072         INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
3073         INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
3074
3075         rc = smc_ib_register_client();
3076         if (rc) {
3077                 pr_err("%s: ib_register fails with %d\n", __func__, rc);
3078                 goto out_sock;
3079         }
3080
3081         rc = tcp_register_ulp(&smc_ulp_ops);
3082         if (rc) {
3083                 pr_err("%s: tcp_ulp_register fails with %d\n", __func__, rc);
3084                 goto out_sock;
3085         }
3086
3087         static_branch_enable(&tcp_have_smc);
3088         return 0;
3089
3090 out_sock:
3091         sock_unregister(PF_SMC);
3092 out_proto6:
3093         proto_unregister(&smc_proto6);
3094 out_proto:
3095         proto_unregister(&smc_proto);
3096 out_core:
3097         smc_core_exit();
3098 out_alloc_wqs:
3099         destroy_workqueue(smc_close_wq);
3100 out_alloc_hs_wq:
3101         destroy_workqueue(smc_hs_wq);
3102 out_pnet:
3103         smc_pnet_exit();
3104 out_nl:
3105         smc_nl_exit();
3106 out_pernet_subsys:
3107         unregister_pernet_subsys(&smc_net_ops);
3108
3109         return rc;
3110 }
3111
3112 static void __exit smc_exit(void)
3113 {
3114         static_branch_disable(&tcp_have_smc);
3115         tcp_unregister_ulp(&smc_ulp_ops);
3116         sock_unregister(PF_SMC);
3117         smc_core_exit();
3118         smc_ib_unregister_client();
3119         destroy_workqueue(smc_close_wq);
3120         destroy_workqueue(smc_hs_wq);
3121         proto_unregister(&smc_proto6);
3122         proto_unregister(&smc_proto);
3123         smc_pnet_exit();
3124         smc_nl_exit();
3125         smc_clc_exit();
3126         unregister_pernet_subsys(&smc_net_stat_ops);
3127         unregister_pernet_subsys(&smc_net_ops);
3128         rcu_barrier();
3129 }
3130
3131 module_init(smc_init);
3132 module_exit(smc_exit);
3133
3134 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
3135 MODULE_DESCRIPTION("smc socket address family");
3136 MODULE_LICENSE("GPL");
3137 MODULE_ALIAS_NETPROTO(PF_SMC);
3138 MODULE_ALIAS_TCP_ULP("smc");