net/smc/af_smc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
   6  *  applies to SOCK_STREAM sockets only
   7  *  offers an alternative communication option for TCP-protocol sockets
   8  *  applicable with RoCE-cards only
   9  *
  10  *  Initial restrictions:
  11  *    - support for alternate links postponed
  12  *
  13  *  Copyright IBM Corp. 2016, 2018
  14  *
  15  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
  16  *              based on prototype from Frank Blaschka
  17  */
  18
  19 #define KMSG_COMPONENT "smc"
  20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  21
  22 #include <linux/module.h>
  23 #include <linux/socket.h>
  24 #include <linux/workqueue.h>
  25 #include <linux/in.h>
  26 #include <linux/sched/signal.h>
  27 #include <linux/if_vlan.h>
  28 #include <linux/rcupdate_wait.h>
  29 #include <linux/ctype.h>
  30
  31 #include <net/sock.h>
  32 #include <net/tcp.h>
  33 #include <net/smc.h>
  34 #include <asm/ioctls.h>
  35
  36 #include <net/net_namespace.h>
  37 #include <net/netns/generic.h>
  38 #include "smc_netns.h"
  39
  40 #include "smc.h"
  41 #include "smc_clc.h"
  42 #include "smc_llc.h"
  43 #include "smc_cdc.h"
  44 #include "smc_core.h"
  45 #include "smc_ib.h"
  46 #include "smc_ism.h"
  47 #include "smc_pnet.h"
  48 #include "smc_netlink.h"
  49 #include "smc_tx.h"
  50 #include "smc_rx.h"
  51 #include "smc_close.h"
  52 #include "smc_stats.h"
  53
  54 static DEFINE_MUTEX(smc_server_lgr_pending);    /* serialize link group
  55                                                  * creation on server
  56                                                  */
  57 static DEFINE_MUTEX(smc_client_lgr_pending);    /* serialize link group
  58                                                  * creation on client
  59                                                  */
  60
  61 struct workqueue_struct *smc_hs_wq;     /* wq for handshake work */
  62 struct workqueue_struct *smc_close_wq;  /* wq for close work */
  63
  64 static void smc_tcp_listen_work(struct work_struct *);
  65 static void smc_connect_work(struct work_struct *);
  66
  67 static void smc_set_keepalive(struct sock *sk, int val)
  68 {
  69         struct smc_sock *smc = smc_sk(sk);
  70
  71         smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
  72 }
  73
  74 static struct smc_hashinfo smc_v4_hashinfo = {
  75         .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
  76 };
  77
  78 static struct smc_hashinfo smc_v6_hashinfo = {
  79         .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
  80 };
  81
  82 int smc_hash_sk(struct sock *sk)
  83 {
  84         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  85         struct hlist_head *head;
  86
  87         head = &h->ht;
  88
  89         write_lock_bh(&h->lock);
  90         sk_add_node(sk, head);
  91         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  92         write_unlock_bh(&h->lock);
  93
  94         return 0;
  95 }
  96 EXPORT_SYMBOL_GPL(smc_hash_sk);
  97
  98 void smc_unhash_sk(struct sock *sk)
  99 {
 100         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
 101
 102         write_lock_bh(&h->lock);
 103         if (sk_del_node_init(sk))
 104                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
 105         write_unlock_bh(&h->lock);
 106 }
 107 EXPORT_SYMBOL_GPL(smc_unhash_sk);
 108
 109 struct proto smc_proto = {
 110         .name           = "SMC",
 111         .owner          = THIS_MODULE,
 112         .keepalive      = smc_set_keepalive,
 113         .hash           = smc_hash_sk,
 114         .unhash         = smc_unhash_sk,
 115         .obj_size       = sizeof(struct smc_sock),
 116         .h.smc_hash     = &smc_v4_hashinfo,
 117         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 118 };
 119 EXPORT_SYMBOL_GPL(smc_proto);
 120
 121 struct proto smc_proto6 = {
 122         .name           = "SMC6",
 123         .owner          = THIS_MODULE,
 124         .keepalive      = smc_set_keepalive,
 125         .hash           = smc_hash_sk,
 126         .unhash         = smc_unhash_sk,
 127         .obj_size       = sizeof(struct smc_sock),
 128         .h.smc_hash     = &smc_v6_hashinfo,
 129         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 130 };
 131 EXPORT_SYMBOL_GPL(smc_proto6);
 132
 133 static void smc_restore_fallback_changes(struct smc_sock *smc)
 134 {
 135         if (smc->clcsock->file) { /* non-accepted sockets have no file yet */
 136                 smc->clcsock->file->private_data = smc->sk.sk_socket;
 137                 smc->clcsock->file = NULL;
 138         }
 139 }
 140
 141 static int __smc_release(struct smc_sock *smc)
 142 {
 143         struct sock *sk = &smc->sk;
 144         int rc = 0;
 145
 146         if (!smc->use_fallback) {
 147                 rc = smc_close_active(smc);
 148                 sock_set_flag(sk, SOCK_DEAD);
 149                 sk->sk_shutdown |= SHUTDOWN_MASK;
 150         } else {
 151                 if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
 152                         sock_put(sk); /* passive closing */
 153                 if (sk->sk_state == SMC_LISTEN) {
 154                         /* wake up clcsock accept */
 155                         rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR);
 156                 }
 157                 sk->sk_state = SMC_CLOSED;
 158                 sk->sk_state_change(sk);
 159                 smc_restore_fallback_changes(smc);
 160         }
 161
 162         sk->sk_prot->unhash(sk);
 163
 164         if (sk->sk_state == SMC_CLOSED) {
 165                 if (smc->clcsock) {
 166                         release_sock(sk);
 167                         smc_clcsock_release(smc);
 168                         lock_sock(sk);
 169                 }
 170                 if (!smc->use_fallback)
 171                         smc_conn_free(&smc->conn);
 172         }
 173
 174         return rc;
 175 }
 176
 177 static int smc_release(struct socket *sock)
 178 {
 179         struct sock *sk = sock->sk;
 180         struct smc_sock *smc;
 181         int rc = 0;
 182
 183         if (!sk)
 184                 goto out;
 185
 186         sock_hold(sk); /* sock_put below */
 187         smc = smc_sk(sk);
 188
 189         /* cleanup for a dangling non-blocking connect */
 190         if (smc->connect_nonblock && sk->sk_state == SMC_INIT)
 191                 tcp_abort(smc->clcsock->sk, ECONNABORTED);
 192         flush_work(&smc->connect_work);
 193
 194         if (sk->sk_state == SMC_LISTEN)
 195                 /* smc_close_non_accepted() is called and acquires
 196                  * sock lock for child sockets again
 197                  */
 198                 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 199         else
 200                 lock_sock(sk);
 201
 202         rc = __smc_release(smc);
 203
 204         /* detach socket */
 205         sock_orphan(sk);
 206         sock->sk = NULL;
 207         release_sock(sk);
 208
 209         sock_put(sk); /* sock_hold above */
 210         sock_put(sk); /* final sock_put */
 211 out:
 212         return rc;
 213 }
 214
 215 static void smc_destruct(struct sock *sk)
 216 {
 217         if (sk->sk_state != SMC_CLOSED)
 218                 return;
 219         if (!sock_flag(sk, SOCK_DEAD))
 220                 return;
 221
 222         sk_refcnt_debug_dec(sk);
 223 }
 224
 225 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 226                                    int protocol)
 227 {
 228         struct smc_sock *smc;
 229         struct proto *prot;
 230         struct sock *sk;
 231
 232         prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
 233         sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
 234         if (!sk)
 235                 return NULL;
 236
 237         sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
 238         sk->sk_state = SMC_INIT;
 239         sk->sk_destruct = smc_destruct;
 240         sk->sk_protocol = protocol;
 241         smc = smc_sk(sk);
 242         INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 243         INIT_WORK(&smc->connect_work, smc_connect_work);
 244         INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
 245         INIT_LIST_HEAD(&smc->accept_q);
 246         spin_lock_init(&smc->accept_q_lock);
 247         spin_lock_init(&smc->conn.send_lock);
 248         sk->sk_prot->hash(sk);
 249         sk_refcnt_debug_inc(sk);
 250         mutex_init(&smc->clcsock_release_lock);
 251
 252         return sk;
 253 }
 254
 255 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 256                     int addr_len)
 257 {
 258         struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
 259         struct sock *sk = sock->sk;
 260         struct smc_sock *smc;
 261         int rc;
 262
 263         smc = smc_sk(sk);
 264
 265         /* replicate tests from inet_bind(), to be safe wrt. future changes */
 266         rc = -EINVAL;
 267         if (addr_len < sizeof(struct sockaddr_in))
 268                 goto out;
 269
 270         rc = -EAFNOSUPPORT;
 271         if (addr->sin_family != AF_INET &&
 272             addr->sin_family != AF_INET6 &&
 273             addr->sin_family != AF_UNSPEC)
 274                 goto out;
 275         /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
 276         if (addr->sin_family == AF_UNSPEC &&
 277             addr->sin_addr.s_addr != htonl(INADDR_ANY))
 278                 goto out;
 279
 280         lock_sock(sk);
 281
 282         /* Check if socket is already active */
 283         rc = -EINVAL;
 284         if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
 285                 goto out_rel;
 286
 287         smc->clcsock->sk->sk_reuse = sk->sk_reuse;
 288         rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 289
 290 out_rel:
 291         release_sock(sk);
 292 out:
 293         return rc;
 294 }
 295
 296 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
 297                                    unsigned long mask)
 298 {
 299         /* options we don't get control via setsockopt for */
 300         nsk->sk_type = osk->sk_type;
 301         nsk->sk_sndbuf = osk->sk_sndbuf;
 302         nsk->sk_rcvbuf = osk->sk_rcvbuf;
 303         nsk->sk_sndtimeo = osk->sk_sndtimeo;
 304         nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
 305         nsk->sk_mark = osk->sk_mark;
 306         nsk->sk_priority = osk->sk_priority;
 307         nsk->sk_rcvlowat = osk->sk_rcvlowat;
 308         nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
 309         nsk->sk_err = osk->sk_err;
 310
 311         nsk->sk_flags &= ~mask;
 312         nsk->sk_flags |= osk->sk_flags & mask;
 313 }
 314
 315 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
 316                              (1UL << SOCK_KEEPOPEN) | \
 317                              (1UL << SOCK_LINGER) | \
 318                              (1UL << SOCK_BROADCAST) | \
 319                              (1UL << SOCK_TIMESTAMP) | \
 320                              (1UL << SOCK_DBG) | \
 321                              (1UL << SOCK_RCVTSTAMP) | \
 322                              (1UL << SOCK_RCVTSTAMPNS) | \
 323                              (1UL << SOCK_LOCALROUTE) | \
 324                              (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
 325                              (1UL << SOCK_RXQ_OVFL) | \
 326                              (1UL << SOCK_WIFI_STATUS) | \
 327                              (1UL << SOCK_NOFCS) | \
 328                              (1UL << SOCK_FILTER_LOCKED) | \
 329                              (1UL << SOCK_TSTAMP_NEW))
 330 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
 331  * clc socket (since smc is not called for these options from net/core)
 332  */
 333 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
 334 {
 335         smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
 336 }
 337
 338 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
 339                              (1UL << SOCK_KEEPOPEN) | \
 340                              (1UL << SOCK_LINGER) | \
 341                              (1UL << SOCK_DBG))
 342 /* copy only settings and flags relevant for smc from clc to smc socket */
 343 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
 344 {
 345         smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 346 }
 347
 348 /* register the new rmb on all links */
 349 static int smcr_lgr_reg_rmbs(struct smc_link *link,
 350                              struct smc_buf_desc *rmb_desc)
 351 {
 352         struct smc_link_group *lgr = link->lgr;
 353         int i, rc = 0;
 354
 355         rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
 356         if (rc)
 357                 return rc;
 358         /* protect against parallel smc_llc_cli_rkey_exchange() and
 359          * parallel smcr_link_reg_rmb()
 360          */
 361         mutex_lock(&lgr->llc_conf_mutex);
 362         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 363                 if (!smc_link_active(&lgr->lnk[i]))
 364                         continue;
 365                 rc = smcr_link_reg_rmb(&lgr->lnk[i], rmb_desc);
 366                 if (rc)
 367                         goto out;
 368         }
 369
 370         /* exchange confirm_rkey msg with peer */
 371         rc = smc_llc_do_confirm_rkey(link, rmb_desc);
 372         if (rc) {
 373                 rc = -EFAULT;
 374                 goto out;
 375         }
 376         rmb_desc->is_conf_rkey = true;
 377 out:
 378         mutex_unlock(&lgr->llc_conf_mutex);
 379         smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
 380         return rc;
 381 }
 382
 383 static int smcr_clnt_conf_first_link(struct smc_sock *smc)
 384 {
 385         struct smc_link *link = smc->conn.lnk;
 386         struct smc_llc_qentry *qentry;
 387         int rc;
 388
 389         /* receive CONFIRM LINK request from server over RoCE fabric */
 390         qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
 391                               SMC_LLC_CONFIRM_LINK);
 392         if (!qentry) {
 393                 struct smc_clc_msg_decline dclc;
 394
 395                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 396                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 397                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
 398         }
 399         smc_llc_save_peer_uid(qentry);
 400         rc = smc_llc_eval_conf_link(qentry, SMC_LLC_REQ);
 401         smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
 402         if (rc)
 403                 return SMC_CLC_DECL_RMBE_EC;
 404
 405         rc = smc_ib_modify_qp_rts(link);
 406         if (rc)
 407                 return SMC_CLC_DECL_ERR_RDYLNK;
 408
 409         smc_wr_remember_qp_attr(link);
 410
 411         if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
 412                 return SMC_CLC_DECL_ERR_REGRMB;
 413
 414         /* confirm_rkey is implicit on 1st contact */
 415         smc->conn.rmb_desc->is_conf_rkey = true;
 416
 417         /* send CONFIRM LINK response over RoCE fabric */
 418         rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
 419         if (rc < 0)
 420                 return SMC_CLC_DECL_TIMEOUT_CL;
 421
 422         smc_llc_link_active(link);
 423         smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
 424
 425         /* optional 2nd link, receive ADD LINK request from server */
 426         qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
 427                               SMC_LLC_ADD_LINK);
 428         if (!qentry) {
 429                 struct smc_clc_msg_decline dclc;
 430
 431                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 432                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 433                 if (rc == -EAGAIN)
 434                         rc = 0; /* no DECLINE received, go with one link */
 435                 return rc;
 436         }
 437         smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl);
 438         smc_llc_cli_add_link(link, qentry);
 439         return 0;
 440 }
 441
 442 static void smcr_conn_save_peer_info(struct smc_sock *smc,
 443                                      struct smc_clc_msg_accept_confirm *clc)
 444 {
 445         int bufsize = smc_uncompress_bufsize(clc->r0.rmbe_size);
 446
 447         smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx;
 448         smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token);
 449         smc->conn.peer_rmbe_size = bufsize;
 450         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 451         smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
 452 }
 453
 454 static bool smc_isascii(char *hostname)
 455 {
 456         int i;
 457
 458         for (i = 0; i < SMC_MAX_HOSTNAME_LEN; i++)
 459                 if (!isascii(hostname[i]))
 460                         return false;
 461         return true;
 462 }
 463
 464 static void smcd_conn_save_peer_info(struct smc_sock *smc,
 465                                      struct smc_clc_msg_accept_confirm *clc)
 466 {
 467         int bufsize = smc_uncompress_bufsize(clc->d0.dmbe_size);
 468
 469         smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx;
 470         smc->conn.peer_token = clc->d0.token;
 471         /* msg header takes up space in the buffer */
 472         smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
 473         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 474         smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
 475         if (clc->hdr.version > SMC_V1 &&
 476             (clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK)) {
 477                 struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
 478                         (struct smc_clc_msg_accept_confirm_v2 *)clc;
 479                 struct smc_clc_first_contact_ext *fce =
 480                         (struct smc_clc_first_contact_ext *)
 481                                 (((u8 *)clc_v2) + sizeof(*clc_v2));
 482
 483                 memcpy(smc->conn.lgr->negotiated_eid, clc_v2->eid,
 484                        SMC_MAX_EID_LEN);
 485                 smc->conn.lgr->peer_os = fce->os_type;
 486                 smc->conn.lgr->peer_smc_release = fce->release;
 487                 if (smc_isascii(fce->hostname))
 488                         memcpy(smc->conn.lgr->peer_hostname, fce->hostname,
 489                                SMC_MAX_HOSTNAME_LEN);
 490         }
 491 }
 492
 493 static void smc_conn_save_peer_info(struct smc_sock *smc,
 494                                     struct smc_clc_msg_accept_confirm *clc)
 495 {
 496         if (smc->conn.lgr->is_smcd)
 497                 smcd_conn_save_peer_info(smc, clc);
 498         else
 499                 smcr_conn_save_peer_info(smc, clc);
 500 }
 501
 502 static void smc_link_save_peer_info(struct smc_link *link,
 503                                     struct smc_clc_msg_accept_confirm *clc)
 504 {
 505         link->peer_qpn = ntoh24(clc->r0.qpn);
 506         memcpy(link->peer_gid, clc->r0.lcl.gid, SMC_GID_SIZE);
 507         memcpy(link->peer_mac, clc->r0.lcl.mac, sizeof(link->peer_mac));
 508         link->peer_psn = ntoh24(clc->r0.psn);
 509         link->peer_mtu = clc->r0.qp_mtu;
 510 }
 511
 512 static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc,
 513                                        struct smc_stats_fback *fback_arr)
 514 {
 515         int cnt;
 516
 517         for (cnt = 0; cnt < SMC_MAX_FBACK_RSN_CNT; cnt++) {
 518                 if (fback_arr[cnt].fback_code == smc->fallback_rsn) {
 519                         fback_arr[cnt].count++;
 520                         break;
 521                 }
 522                 if (!fback_arr[cnt].fback_code) {
 523                         fback_arr[cnt].fback_code = smc->fallback_rsn;
 524                         fback_arr[cnt].count++;
 525                         break;
 526                 }
 527         }
 528 }
 529
 530 static void smc_stat_fallback(struct smc_sock *smc)
 531 {
 532         struct net *net = sock_net(&smc->sk);
 533
 534         mutex_lock(&net->smc.mutex_fback_rsn);
 535         if (smc->listen_smc) {
 536                 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->srv);
 537                 net->smc.fback_rsn->srv_fback_cnt++;
 538         } else {
 539                 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->clnt);
 540                 net->smc.fback_rsn->clnt_fback_cnt++;
 541         }
 542         mutex_unlock(&net->smc.mutex_fback_rsn);
 543 }
 544
 545 static void smc_switch_to_fallback(struct smc_sock *smc, int reason_code)
 546 {
 547         smc->use_fallback = true;
 548         smc->fallback_rsn = reason_code;
 549         smc_stat_fallback(smc);
 550         if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
 551                 smc->clcsock->file = smc->sk.sk_socket->file;
 552                 smc->clcsock->file->private_data = smc->clcsock;
 553                 smc->clcsock->wq.fasync_list =
 554                         smc->sk.sk_socket->wq.fasync_list;
 555         }
 556 }
 557
 558 /* fall back during connect */
 559 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
 560 {
 561         smc_switch_to_fallback(smc, reason_code);
 562         smc_copy_sock_settings_to_clc(smc);
 563         smc->connect_nonblock = 0;
 564         if (smc->sk.sk_state == SMC_INIT)
 565                 smc->sk.sk_state = SMC_ACTIVE;
 566         return 0;
 567 }
 568
 569 /* decline and fall back during connect */
 570 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code,
 571                                         u8 version)
 572 {
 573         struct net *net = sock_net(&smc->sk);
 574         int rc;
 575
 576         if (reason_code < 0) { /* error, fallback is not possible */
 577                 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 578                 if (smc->sk.sk_state == SMC_INIT)
 579                         sock_put(&smc->sk); /* passive closing */
 580                 return reason_code;
 581         }
 582         if (reason_code != SMC_CLC_DECL_PEERDECL) {
 583                 rc = smc_clc_send_decline(smc, reason_code, version);
 584                 if (rc < 0) {
 585                         this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 586                         if (smc->sk.sk_state == SMC_INIT)
 587                                 sock_put(&smc->sk); /* passive closing */
 588                         return rc;
 589                 }
 590         }
 591         return smc_connect_fallback(smc, reason_code);
 592 }
 593
 594 static void smc_conn_abort(struct smc_sock *smc, int local_first)
 595 {
 596         if (local_first)
 597                 smc_lgr_cleanup_early(&smc->conn);
 598         else
 599                 smc_conn_free(&smc->conn);
 600 }
 601
 602 /* check if there is a rdma device available for this connection. */
 603 /* called for connect and listen */
 604 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
 605 {
 606         /* PNET table look up: search active ib_device and port
 607          * within same PNETID that also contains the ethernet device
 608          * used for the internal TCP socket
 609          */
 610         smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
 611         if (!ini->ib_dev)
 612                 return SMC_CLC_DECL_NOSMCRDEV;
 613         return 0;
 614 }
 615
 616 /* check if there is an ISM device available for this connection. */
 617 /* called for connect and listen */
 618 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
 619 {
 620         /* Find ISM device with same PNETID as connecting interface  */
 621         smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
 622         if (!ini->ism_dev[0])
 623                 return SMC_CLC_DECL_NOSMCDDEV;
 624         else
 625                 ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]);
 626         return 0;
 627 }
 628
 629 /* is chid unique for the ism devices that are already determined? */
 630 static bool smc_find_ism_v2_is_unique_chid(u16 chid, struct smc_init_info *ini,
 631                                            int cnt)
 632 {
 633         int i = (!ini->ism_dev[0]) ? 1 : 0;
 634
 635         for (; i < cnt; i++)
 636                 if (ini->ism_chid[i] == chid)
 637                         return false;
 638         return true;
 639 }
 640
 641 /* determine possible V2 ISM devices (either without PNETID or with PNETID plus
 642  * PNETID matching net_device)
 643  */
 644 static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
 645                                        struct smc_init_info *ini)
 646 {
 647         int rc = SMC_CLC_DECL_NOSMCDDEV;
 648         struct smcd_dev *smcd;
 649         int i = 1;
 650         u16 chid;
 651
 652         if (smcd_indicated(ini->smc_type_v1))
 653                 rc = 0;         /* already initialized for V1 */
 654         mutex_lock(&smcd_dev_list.mutex);
 655         list_for_each_entry(smcd, &smcd_dev_list.list, list) {
 656                 if (smcd->going_away || smcd == ini->ism_dev[0])
 657                         continue;
 658                 chid = smc_ism_get_chid(smcd);
 659                 if (!smc_find_ism_v2_is_unique_chid(chid, ini, i))
 660                         continue;
 661                 if (!smc_pnet_is_pnetid_set(smcd->pnetid) ||
 662                     smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) {
 663                         ini->ism_dev[i] = smcd;
 664                         ini->ism_chid[i] = chid;
 665                         ini->is_smcd = true;
 666                         rc = 0;
 667                         i++;
 668                         if (i > SMC_MAX_ISM_DEVS)
 669                                 break;
 670                 }
 671         }
 672         mutex_unlock(&smcd_dev_list.mutex);
 673         ini->ism_offered_cnt = i - 1;
 674         if (!ini->ism_dev[0] && !ini->ism_dev[1])
 675                 ini->smcd_version = 0;
 676
 677         return rc;
 678 }
 679
 680 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
 681 static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
 682                                       struct smc_init_info *ini)
 683 {
 684         if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id))
 685                 return SMC_CLC_DECL_ISMVLANERR;
 686         return 0;
 687 }
 688
 689 static int smc_find_proposal_devices(struct smc_sock *smc,
 690                                      struct smc_init_info *ini)
 691 {
 692         int rc = 0;
 693
 694         /* check if there is an ism device available */
 695         if (ini->smcd_version & SMC_V1) {
 696                 if (smc_find_ism_device(smc, ini) ||
 697                     smc_connect_ism_vlan_setup(smc, ini)) {
 698                         if (ini->smc_type_v1 == SMC_TYPE_B)
 699                                 ini->smc_type_v1 = SMC_TYPE_R;
 700                         else
 701                                 ini->smc_type_v1 = SMC_TYPE_N;
 702                 } /* else ISM V1 is supported for this connection */
 703                 if (smc_find_rdma_device(smc, ini)) {
 704                         if (ini->smc_type_v1 == SMC_TYPE_B)
 705                                 ini->smc_type_v1 = SMC_TYPE_D;
 706                         else
 707                                 ini->smc_type_v1 = SMC_TYPE_N;
 708                 } /* else RDMA is supported for this connection */
 709         }
 710         if (smc_ism_is_v2_capable() && smc_find_ism_v2_device_clnt(smc, ini))
 711                 ini->smc_type_v2 = SMC_TYPE_N;
 712
 713         /* if neither ISM nor RDMA are supported, fallback */
 714         if (!smcr_indicated(ini->smc_type_v1) &&
 715             ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N)
 716                 rc = SMC_CLC_DECL_NOSMCDEV;
 717
 718         return rc;
 719 }
 720
 721 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
 722  * used, the VLAN ID will be registered again during the connection setup.
 723  */
 724 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc,
 725                                         struct smc_init_info *ini)
 726 {
 727         if (!smcd_indicated(ini->smc_type_v1))
 728                 return 0;
 729         if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id))
 730                 return SMC_CLC_DECL_CNFERR;
 731         return 0;
 732 }
 733
 734 #define SMC_CLC_MAX_ACCEPT_LEN \
 735         (sizeof(struct smc_clc_msg_accept_confirm_v2) + \
 736          sizeof(struct smc_clc_first_contact_ext) + \
 737          sizeof(struct smc_clc_msg_trail))
 738
 739 /* CLC handshake during connect */
 740 static int smc_connect_clc(struct smc_sock *smc,
 741                            struct smc_clc_msg_accept_confirm_v2 *aclc2,
 742                            struct smc_init_info *ini)
 743 {
 744         int rc = 0;
 745
 746         /* do inband token exchange */
 747         rc = smc_clc_send_proposal(smc, ini);
 748         if (rc)
 749                 return rc;
 750         /* receive SMC Accept CLC message */
 751         return smc_clc_wait_msg(smc, aclc2, SMC_CLC_MAX_ACCEPT_LEN,
 752                                 SMC_CLC_ACCEPT, CLC_WAIT_TIME);
 753 }
 754
 755 /* setup for RDMA connection of client */
 756 static int smc_connect_rdma(struct smc_sock *smc,
 757                             struct smc_clc_msg_accept_confirm *aclc,
 758                             struct smc_init_info *ini)
 759 {
 760         int i, reason_code = 0;
 761         struct smc_link *link;
 762
 763         ini->is_smcd = false;
 764         ini->ib_lcl = &aclc->r0.lcl;
 765         ini->ib_clcqpn = ntoh24(aclc->r0.qpn);
 766         ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
 767
 768         mutex_lock(&smc_client_lgr_pending);
 769         reason_code = smc_conn_create(smc, ini);
 770         if (reason_code) {
 771                 mutex_unlock(&smc_client_lgr_pending);
 772                 return reason_code;
 773         }
 774
 775         smc_conn_save_peer_info(smc, aclc);
 776
 777         if (ini->first_contact_local) {
 778                 link = smc->conn.lnk;
 779         } else {
 780                 /* set link that was assigned by server */
 781                 link = NULL;
 782                 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 783                         struct smc_link *l = &smc->conn.lgr->lnk[i];
 784
 785                         if (l->peer_qpn == ntoh24(aclc->r0.qpn) &&
 786                             !memcmp(l->peer_gid, &aclc->r0.lcl.gid,
 787                                     SMC_GID_SIZE) &&
 788                             !memcmp(l->peer_mac, &aclc->r0.lcl.mac,
 789                                     sizeof(l->peer_mac))) {
 790                                 link = l;
 791                                 break;
 792                         }
 793                 }
 794                 if (!link) {
 795                         reason_code = SMC_CLC_DECL_NOSRVLINK;
 796                         goto connect_abort;
 797                 }
 798                 smc_switch_link_and_count(&smc->conn, link);
 799         }
 800
 801         /* create send buffer and rmb */
 802         if (smc_buf_create(smc, false)) {
 803                 reason_code = SMC_CLC_DECL_MEM;
 804                 goto connect_abort;
 805         }
 806
 807         if (ini->first_contact_local)
 808                 smc_link_save_peer_info(link, aclc);
 809
 810         if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) {
 811                 reason_code = SMC_CLC_DECL_ERR_RTOK;
 812                 goto connect_abort;
 813         }
 814
 815         smc_close_init(smc);
 816         smc_rx_init(smc);
 817
 818         if (ini->first_contact_local) {
 819                 if (smc_ib_ready_link(link)) {
 820                         reason_code = SMC_CLC_DECL_ERR_RDYLNK;
 821                         goto connect_abort;
 822                 }
 823         } else {
 824                 if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) {
 825                         reason_code = SMC_CLC_DECL_ERR_REGRMB;
 826                         goto connect_abort;
 827                 }
 828         }
 829         smc_rmb_sync_sg_for_device(&smc->conn);
 830
 831         reason_code = smc_clc_send_confirm(smc, ini->first_contact_local,
 832                                            SMC_V1);
 833         if (reason_code)
 834                 goto connect_abort;
 835
 836         smc_tx_init(smc);
 837
 838         if (ini->first_contact_local) {
 839                 /* QP confirmation over RoCE fabric */
 840                 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
 841                 reason_code = smcr_clnt_conf_first_link(smc);
 842                 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
 843                 if (reason_code)
 844                         goto connect_abort;
 845         }
 846         mutex_unlock(&smc_client_lgr_pending);
 847
 848         smc_copy_sock_settings_to_clc(smc);
 849         smc->connect_nonblock = 0;
 850         if (smc->sk.sk_state == SMC_INIT)
 851                 smc->sk.sk_state = SMC_ACTIVE;
 852
 853         return 0;
 854 connect_abort:
 855         smc_conn_abort(smc, ini->first_contact_local);
 856         mutex_unlock(&smc_client_lgr_pending);
 857         smc->connect_nonblock = 0;
 858
 859         return reason_code;
 860 }
 861
 862 /* The server has chosen one of the proposed ISM devices for the communication.
 863  * Determine from the CHID of the received CLC ACCEPT the ISM device chosen.
 864  */
 865 static int
 866 smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm_v2 *aclc,
 867                                struct smc_init_info *ini)
 868 {
 869         int i;
 870
 871         for (i = 0; i < ini->ism_offered_cnt + 1; i++) {
 872                 if (ini->ism_chid[i] == ntohs(aclc->chid)) {
 873                         ini->ism_selected = i;
 874                         return 0;
 875                 }
 876         }
 877
 878         return -EPROTO;
 879 }
 880
 881 /* setup for ISM connection of client */
 882 static int smc_connect_ism(struct smc_sock *smc,
 883                            struct smc_clc_msg_accept_confirm *aclc,
 884                            struct smc_init_info *ini)
 885 {
 886         int rc = 0;
 887
 888         ini->is_smcd = true;
 889         ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
 890
 891         if (aclc->hdr.version == SMC_V2) {
 892                 struct smc_clc_msg_accept_confirm_v2 *aclc_v2 =
 893                         (struct smc_clc_msg_accept_confirm_v2 *)aclc;
 894
 895                 rc = smc_v2_determine_accepted_chid(aclc_v2, ini);
 896                 if (rc)
 897                         return rc;
 898         }
 899         ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid;
 900
 901         /* there is only one lgr role for SMC-D; use server lock */
 902         mutex_lock(&smc_server_lgr_pending);
 903         rc = smc_conn_create(smc, ini);
 904         if (rc) {
 905                 mutex_unlock(&smc_server_lgr_pending);
 906                 return rc;
 907         }
 908
 909         /* Create send and receive buffers */
 910         rc = smc_buf_create(smc, true);
 911         if (rc) {
 912                 rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM;
 913                 goto connect_abort;
 914         }
 915
 916         smc_conn_save_peer_info(smc, aclc);
 917         smc_close_init(smc);
 918         smc_rx_init(smc);
 919         smc_tx_init(smc);
 920
 921         rc = smc_clc_send_confirm(smc, ini->first_contact_local,
 922                                   aclc->hdr.version);
 923         if (rc)
 924                 goto connect_abort;
 925         mutex_unlock(&smc_server_lgr_pending);
 926
 927         smc_copy_sock_settings_to_clc(smc);
 928         smc->connect_nonblock = 0;
 929         if (smc->sk.sk_state == SMC_INIT)
 930                 smc->sk.sk_state = SMC_ACTIVE;
 931
 932         return 0;
 933 connect_abort:
 934         smc_conn_abort(smc, ini->first_contact_local);
 935         mutex_unlock(&smc_server_lgr_pending);
 936         smc->connect_nonblock = 0;
 937
 938         return rc;
 939 }
 940
 941 /* check if received accept type and version matches a proposed one */
 942 static int smc_connect_check_aclc(struct smc_init_info *ini,
 943                                   struct smc_clc_msg_accept_confirm *aclc)
 944 {
 945         if ((aclc->hdr.typev1 == SMC_TYPE_R &&
 946              !smcr_indicated(ini->smc_type_v1)) ||
 947             (aclc->hdr.typev1 == SMC_TYPE_D &&
 948              ((!smcd_indicated(ini->smc_type_v1) &&
 949                !smcd_indicated(ini->smc_type_v2)) ||
 950               (aclc->hdr.version == SMC_V1 &&
 951                !smcd_indicated(ini->smc_type_v1)) ||
 952               (aclc->hdr.version == SMC_V2 &&
 953                !smcd_indicated(ini->smc_type_v2)))))
 954                 return SMC_CLC_DECL_MODEUNSUPP;
 955
 956         return 0;
 957 }
 958
 959 /* perform steps before actually connecting */
 960 static int __smc_connect(struct smc_sock *smc)
 961 {
 962         u8 version = smc_ism_is_v2_capable() ? SMC_V2 : SMC_V1;
 963         struct smc_clc_msg_accept_confirm_v2 *aclc2;
 964         struct smc_clc_msg_accept_confirm *aclc;
 965         struct smc_init_info *ini = NULL;
 966         u8 *buf = NULL;
 967         int rc = 0;
 968
 969         if (smc->use_fallback)
 970                 return smc_connect_fallback(smc, smc->fallback_rsn);
 971
 972         /* if peer has not signalled SMC-capability, fall back */
 973         if (!tcp_sk(smc->clcsock->sk)->syn_smc)
 974                 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
 975
 976         /* IPSec connections opt out of SMC optimizations */
 977         if (using_ipsec(smc))
 978                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC,
 979                                                     version);
 980
 981         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
 982         if (!ini)
 983                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM,
 984                                                     version);
 985
 986         ini->smcd_version = SMC_V1;
 987         ini->smcd_version |= smc_ism_is_v2_capable() ? SMC_V2 : 0;
 988         ini->smc_type_v1 = SMC_TYPE_B;
 989         ini->smc_type_v2 = smc_ism_is_v2_capable() ? SMC_TYPE_D : SMC_TYPE_N;
 990
 991         /* get vlan id from IP device */
 992         if (smc_vlan_by_tcpsk(smc->clcsock, ini)) {
 993                 ini->smcd_version &= ~SMC_V1;
 994                 ini->smc_type_v1 = SMC_TYPE_N;
 995                 if (!ini->smcd_version) {
 996                         rc = SMC_CLC_DECL_GETVLANERR;
 997                         goto fallback;
 998                 }
 999         }
1000
1001         rc = smc_find_proposal_devices(smc, ini);
1002         if (rc)
1003                 goto fallback;
1004
1005         buf = kzalloc(SMC_CLC_MAX_ACCEPT_LEN, GFP_KERNEL);
1006         if (!buf) {
1007                 rc = SMC_CLC_DECL_MEM;
1008                 goto fallback;
1009         }
1010         aclc2 = (struct smc_clc_msg_accept_confirm_v2 *)buf;
1011         aclc = (struct smc_clc_msg_accept_confirm *)aclc2;
1012
1013         /* perform CLC handshake */
1014         rc = smc_connect_clc(smc, aclc2, ini);
1015         if (rc)
1016                 goto vlan_cleanup;
1017
1018         /* check if smc modes and versions of CLC proposal and accept match */
1019         rc = smc_connect_check_aclc(ini, aclc);
1020         version = aclc->hdr.version == SMC_V1 ? SMC_V1 : SMC_V2;
1021         ini->smcd_version = version;
1022         if (rc)
1023                 goto vlan_cleanup;
1024
1025         /* depending on previous steps, connect using rdma or ism */
1026         if (aclc->hdr.typev1 == SMC_TYPE_R)
1027                 rc = smc_connect_rdma(smc, aclc, ini);
1028         else if (aclc->hdr.typev1 == SMC_TYPE_D)
1029                 rc = smc_connect_ism(smc, aclc, ini);
1030         if (rc)
1031                 goto vlan_cleanup;
1032
1033         SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc);
1034         smc_connect_ism_vlan_cleanup(smc, ini);
1035         kfree(buf);
1036         kfree(ini);
1037         return 0;
1038
1039 vlan_cleanup:
1040         smc_connect_ism_vlan_cleanup(smc, ini);
1041         kfree(buf);
1042 fallback:
1043         kfree(ini);
1044         return smc_connect_decline_fallback(smc, rc, version);
1045 }
1046
1047 static void smc_connect_work(struct work_struct *work)
1048 {
1049         struct smc_sock *smc = container_of(work, struct smc_sock,
1050                                             connect_work);
1051         long timeo = smc->sk.sk_sndtimeo;
1052         int rc = 0;
1053
1054         if (!timeo)
1055                 timeo = MAX_SCHEDULE_TIMEOUT;
1056         lock_sock(smc->clcsock->sk);
1057         if (smc->clcsock->sk->sk_err) {
1058                 smc->sk.sk_err = smc->clcsock->sk->sk_err;
1059         } else if ((1 << smc->clcsock->sk->sk_state) &
1060                                         (TCPF_SYN_SENT | TCP_SYN_RECV)) {
1061                 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
1062                 if ((rc == -EPIPE) &&
1063                     ((1 << smc->clcsock->sk->sk_state) &
1064                                         (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
1065                         rc = 0;
1066         }
1067         release_sock(smc->clcsock->sk);
1068         lock_sock(&smc->sk);
1069         if (rc != 0 || smc->sk.sk_err) {
1070                 smc->sk.sk_state = SMC_CLOSED;
1071                 if (rc == -EPIPE || rc == -EAGAIN)
1072                         smc->sk.sk_err = EPIPE;
1073                 else if (signal_pending(current))
1074                         smc->sk.sk_err = -sock_intr_errno(timeo);
1075                 sock_put(&smc->sk); /* passive closing */
1076                 goto out;
1077         }
1078
1079         rc = __smc_connect(smc);
1080         if (rc < 0)
1081                 smc->sk.sk_err = -rc;
1082
1083 out:
1084         if (!sock_flag(&smc->sk, SOCK_DEAD)) {
1085                 if (smc->sk.sk_err) {
1086                         smc->sk.sk_state_change(&smc->sk);
1087                 } else { /* allow polling before and after fallback decision */
1088                         smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
1089                         smc->sk.sk_write_space(&smc->sk);
1090                 }
1091         }
1092         release_sock(&smc->sk);
1093 }
1094
1095 static int smc_connect(struct socket *sock, struct sockaddr *addr,
1096                        int alen, int flags)
1097 {
1098         struct sock *sk = sock->sk;
1099         struct smc_sock *smc;
1100         int rc = -EINVAL;
1101
1102         smc = smc_sk(sk);
1103
1104         /* separate smc parameter checking to be safe */
1105         if (alen < sizeof(addr->sa_family))
1106                 goto out_err;
1107         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
1108                 goto out_err;
1109
1110         lock_sock(sk);
1111         switch (sk->sk_state) {
1112         default:
1113                 goto out;
1114         case SMC_ACTIVE:
1115                 rc = -EISCONN;
1116                 goto out;
1117         case SMC_INIT:
1118                 break;
1119         }
1120
1121         smc_copy_sock_settings_to_clc(smc);
1122         tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1123         if (smc->connect_nonblock) {
1124                 rc = -EALREADY;
1125                 goto out;
1126         }
1127         rc = kernel_connect(smc->clcsock, addr, alen, flags);
1128         if (rc && rc != -EINPROGRESS)
1129                 goto out;
1130
1131         sock_hold(&smc->sk); /* sock put in passive closing */
1132         if (smc->use_fallback)
1133                 goto out;
1134         if (flags & O_NONBLOCK) {
1135                 if (queue_work(smc_hs_wq, &smc->connect_work))
1136                         smc->connect_nonblock = 1;
1137                 rc = -EINPROGRESS;
1138         } else {
1139                 rc = __smc_connect(smc);
1140                 if (rc < 0)
1141                         goto out;
1142                 else
1143                         rc = 0; /* success cases including fallback */
1144         }
1145
1146 out:
1147         release_sock(sk);
1148 out_err:
1149         return rc;
1150 }
1151
1152 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
1153 {
1154         struct socket *new_clcsock = NULL;
1155         struct sock *lsk = &lsmc->sk;
1156         struct sock *new_sk;
1157         int rc = -EINVAL;
1158
1159         release_sock(lsk);
1160         new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
1161         if (!new_sk) {
1162                 rc = -ENOMEM;
1163                 lsk->sk_err = ENOMEM;
1164                 *new_smc = NULL;
1165                 lock_sock(lsk);
1166                 goto out;
1167         }
1168         *new_smc = smc_sk(new_sk);
1169
1170         mutex_lock(&lsmc->clcsock_release_lock);
1171         if (lsmc->clcsock)
1172                 rc = kernel_accept(lsmc->clcsock, &new_clcsock, SOCK_NONBLOCK);
1173         mutex_unlock(&lsmc->clcsock_release_lock);
1174         lock_sock(lsk);
1175         if  (rc < 0 && rc != -EAGAIN)
1176                 lsk->sk_err = -rc;
1177         if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
1178                 new_sk->sk_prot->unhash(new_sk);
1179                 if (new_clcsock)
1180                         sock_release(new_clcsock);
1181                 new_sk->sk_state = SMC_CLOSED;
1182                 sock_set_flag(new_sk, SOCK_DEAD);
1183                 sock_put(new_sk); /* final */
1184                 *new_smc = NULL;
1185                 goto out;
1186         }
1187
1188         /* new clcsock has inherited the smc listen-specific sk_data_ready
1189          * function; switch it back to the original sk_data_ready function
1190          */
1191         new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready;
1192         (*new_smc)->clcsock = new_clcsock;
1193 out:
1194         return rc;
1195 }
1196
1197 /* add a just created sock to the accept queue of the listen sock as
1198  * candidate for a following socket accept call from user space
1199  */
1200 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
1201 {
1202         struct smc_sock *par = smc_sk(parent);
1203
1204         sock_hold(sk); /* sock_put in smc_accept_unlink () */
1205         spin_lock(&par->accept_q_lock);
1206         list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
1207         spin_unlock(&par->accept_q_lock);
1208         sk_acceptq_added(parent);
1209 }
1210
1211 /* remove a socket from the accept queue of its parental listening socket */
1212 static void smc_accept_unlink(struct sock *sk)
1213 {
1214         struct smc_sock *par = smc_sk(sk)->listen_smc;
1215
1216         spin_lock(&par->accept_q_lock);
1217         list_del_init(&smc_sk(sk)->accept_q);
1218         spin_unlock(&par->accept_q_lock);
1219         sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
1220         sock_put(sk); /* sock_hold in smc_accept_enqueue */
1221 }
1222
1223 /* remove a sock from the accept queue to bind it to a new socket created
1224  * for a socket accept call from user space
1225  */
1226 struct sock *smc_accept_dequeue(struct sock *parent,
1227                                 struct socket *new_sock)
1228 {
1229         struct smc_sock *isk, *n;
1230         struct sock *new_sk;
1231
1232         list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
1233                 new_sk = (struct sock *)isk;
1234
1235                 smc_accept_unlink(new_sk);
1236                 if (new_sk->sk_state == SMC_CLOSED) {
1237                         new_sk->sk_prot->unhash(new_sk);
1238                         if (isk->clcsock) {
1239                                 sock_release(isk->clcsock);
1240                                 isk->clcsock = NULL;
1241                         }
1242                         sock_put(new_sk); /* final */
1243                         continue;
1244                 }
1245                 if (new_sock) {
1246                         sock_graft(new_sk, new_sock);
1247                         if (isk->use_fallback) {
1248                                 smc_sk(new_sk)->clcsock->file = new_sock->file;
1249                                 isk->clcsock->file->private_data = isk->clcsock;
1250                         }
1251                 }
1252                 return new_sk;
1253         }
1254         return NULL;
1255 }
1256
1257 /* clean up for a created but never accepted sock */
1258 void smc_close_non_accepted(struct sock *sk)
1259 {
1260         struct smc_sock *smc = smc_sk(sk);
1261
1262         sock_hold(sk); /* sock_put below */
1263         lock_sock(sk);
1264         if (!sk->sk_lingertime)
1265                 /* wait for peer closing */
1266                 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
1267         __smc_release(smc);
1268         release_sock(sk);
1269         sock_put(sk); /* sock_hold above */
1270         sock_put(sk); /* final sock_put */
1271 }
1272
1273 static int smcr_serv_conf_first_link(struct smc_sock *smc)
1274 {
1275         struct smc_link *link = smc->conn.lnk;
1276         struct smc_llc_qentry *qentry;
1277         int rc;
1278
1279         if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
1280                 return SMC_CLC_DECL_ERR_REGRMB;
1281
1282         /* send CONFIRM LINK request to client over the RoCE fabric */
1283         rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
1284         if (rc < 0)
1285                 return SMC_CLC_DECL_TIMEOUT_CL;
1286
1287         /* receive CONFIRM LINK response from client over the RoCE fabric */
1288         qentry = smc_llc_wait(link->lgr, link, SMC_LLC_WAIT_TIME,
1289                               SMC_LLC_CONFIRM_LINK);
1290         if (!qentry) {
1291                 struct smc_clc_msg_decline dclc;
1292
1293                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1294                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1295                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1296         }
1297         smc_llc_save_peer_uid(qentry);
1298         rc = smc_llc_eval_conf_link(qentry, SMC_LLC_RESP);
1299         smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
1300         if (rc)
1301                 return SMC_CLC_DECL_RMBE_EC;
1302
1303         /* confirm_rkey is implicit on 1st contact */
1304         smc->conn.rmb_desc->is_conf_rkey = true;
1305
1306         smc_llc_link_active(link);
1307         smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
1308
1309         /* initial contact - try to establish second link */
1310         smc_llc_srv_add_link(link);
1311         return 0;
1312 }
1313
1314 /* listen worker: finish */
1315 static void smc_listen_out(struct smc_sock *new_smc)
1316 {
1317         struct smc_sock *lsmc = new_smc->listen_smc;
1318         struct sock *newsmcsk = &new_smc->sk;
1319
1320         if (lsmc->sk.sk_state == SMC_LISTEN) {
1321                 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1322                 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1323                 release_sock(&lsmc->sk);
1324         } else { /* no longer listening */
1325                 smc_close_non_accepted(newsmcsk);
1326         }
1327
1328         /* Wake up accept */
1329         lsmc->sk.sk_data_ready(&lsmc->sk);
1330         sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1331 }
1332
1333 /* listen worker: finish in state connected */
1334 static void smc_listen_out_connected(struct smc_sock *new_smc)
1335 {
1336         struct sock *newsmcsk = &new_smc->sk;
1337
1338         sk_refcnt_debug_inc(newsmcsk);
1339         if (newsmcsk->sk_state == SMC_INIT)
1340                 newsmcsk->sk_state = SMC_ACTIVE;
1341
1342         smc_listen_out(new_smc);
1343 }
1344
1345 /* listen worker: finish in error state */
1346 static void smc_listen_out_err(struct smc_sock *new_smc)
1347 {
1348         struct sock *newsmcsk = &new_smc->sk;
1349         struct net *net = sock_net(newsmcsk);
1350
1351         this_cpu_inc(net->smc.smc_stats->srv_hshake_err_cnt);
1352         if (newsmcsk->sk_state == SMC_INIT)
1353                 sock_put(&new_smc->sk); /* passive closing */
1354         newsmcsk->sk_state = SMC_CLOSED;
1355
1356         smc_listen_out(new_smc);
1357 }
1358
1359 /* listen worker: decline and fall back if possible */
1360 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1361                                int local_first, u8 version)
1362 {
1363         /* RDMA setup failed, switch back to TCP */
1364         smc_conn_abort(new_smc, local_first);
1365         if (reason_code < 0) { /* error, no fallback possible */
1366                 smc_listen_out_err(new_smc);
1367                 return;
1368         }
1369         smc_switch_to_fallback(new_smc, reason_code);
1370         if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1371                 if (smc_clc_send_decline(new_smc, reason_code, version) < 0) {
1372                         smc_listen_out_err(new_smc);
1373                         return;
1374                 }
1375         }
1376         smc_listen_out_connected(new_smc);
1377 }
1378
1379 /* listen worker: version checking */
1380 static int smc_listen_v2_check(struct smc_sock *new_smc,
1381                                struct smc_clc_msg_proposal *pclc,
1382                                struct smc_init_info *ini)
1383 {
1384         struct smc_clc_smcd_v2_extension *pclc_smcd_v2_ext;
1385         struct smc_clc_v2_extension *pclc_v2_ext;
1386         int rc = SMC_CLC_DECL_PEERNOSMC;
1387
1388         ini->smc_type_v1 = pclc->hdr.typev1;
1389         ini->smc_type_v2 = pclc->hdr.typev2;
1390         ini->smcd_version = ini->smc_type_v1 != SMC_TYPE_N ? SMC_V1 : 0;
1391         if (pclc->hdr.version > SMC_V1)
1392                 ini->smcd_version |=
1393                                 ini->smc_type_v2 != SMC_TYPE_N ? SMC_V2 : 0;
1394         if (!(ini->smcd_version & SMC_V2)) {
1395                 rc = SMC_CLC_DECL_PEERNOSMC;
1396                 goto out;
1397         }
1398         if (!smc_ism_is_v2_capable()) {
1399                 ini->smcd_version &= ~SMC_V2;
1400                 rc = SMC_CLC_DECL_NOISM2SUPP;
1401                 goto out;
1402         }
1403         pclc_v2_ext = smc_get_clc_v2_ext(pclc);
1404         if (!pclc_v2_ext) {
1405                 ini->smcd_version &= ~SMC_V2;
1406                 rc = SMC_CLC_DECL_NOV2EXT;
1407                 goto out;
1408         }
1409         pclc_smcd_v2_ext = smc_get_clc_smcd_v2_ext(pclc_v2_ext);
1410         if (!pclc_smcd_v2_ext) {
1411                 ini->smcd_version &= ~SMC_V2;
1412                 rc = SMC_CLC_DECL_NOV2DEXT;
1413         }
1414
1415 out:
1416         if (!ini->smcd_version)
1417                 return rc;
1418
1419         return 0;
1420 }
1421
1422 /* listen worker: check prefixes */
1423 static int smc_listen_prfx_check(struct smc_sock *new_smc,
1424                                  struct smc_clc_msg_proposal *pclc)
1425 {
1426         struct smc_clc_msg_proposal_prefix *pclc_prfx;
1427         struct socket *newclcsock = new_smc->clcsock;
1428
1429         if (pclc->hdr.typev1 == SMC_TYPE_N)
1430                 return 0;
1431         pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1432         if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1433                 return SMC_CLC_DECL_DIFFPREFIX;
1434
1435         return 0;
1436 }
1437
1438 /* listen worker: initialize connection and buffers */
1439 static int smc_listen_rdma_init(struct smc_sock *new_smc,
1440                                 struct smc_init_info *ini)
1441 {
1442         int rc;
1443
1444         /* allocate connection / link group */
1445         rc = smc_conn_create(new_smc, ini);
1446         if (rc)
1447                 return rc;
1448
1449         /* create send buffer and rmb */
1450         if (smc_buf_create(new_smc, false))
1451                 return SMC_CLC_DECL_MEM;
1452
1453         return 0;
1454 }
1455
1456 /* listen worker: initialize connection and buffers for SMC-D */
1457 static int smc_listen_ism_init(struct smc_sock *new_smc,
1458                                struct smc_init_info *ini)
1459 {
1460         int rc;
1461
1462         rc = smc_conn_create(new_smc, ini);
1463         if (rc)
1464                 return rc;
1465
1466         /* Create send and receive buffers */
1467         rc = smc_buf_create(new_smc, true);
1468         if (rc) {
1469                 smc_conn_abort(new_smc, ini->first_contact_local);
1470                 return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB :
1471                                          SMC_CLC_DECL_MEM;
1472         }
1473
1474         return 0;
1475 }
1476
1477 static bool smc_is_already_selected(struct smcd_dev *smcd,
1478                                     struct smc_init_info *ini,
1479                                     int matches)
1480 {
1481         int i;
1482
1483         for (i = 0; i < matches; i++)
1484                 if (smcd == ini->ism_dev[i])
1485                         return true;
1486
1487         return false;
1488 }
1489
1490 /* check for ISM devices matching proposed ISM devices */
1491 static void smc_check_ism_v2_match(struct smc_init_info *ini,
1492                                    u16 proposed_chid, u64 proposed_gid,
1493                                    unsigned int *matches)
1494 {
1495         struct smcd_dev *smcd;
1496
1497         list_for_each_entry(smcd, &smcd_dev_list.list, list) {
1498                 if (smcd->going_away)
1499                         continue;
1500                 if (smc_is_already_selected(smcd, ini, *matches))
1501                         continue;
1502                 if (smc_ism_get_chid(smcd) == proposed_chid &&
1503                     !smc_ism_cantalk(proposed_gid, ISM_RESERVED_VLANID, smcd)) {
1504                         ini->ism_peer_gid[*matches] = proposed_gid;
1505                         ini->ism_dev[*matches] = smcd;
1506                         (*matches)++;
1507                         break;
1508                 }
1509         }
1510 }
1511
1512 static void smc_find_ism_store_rc(u32 rc, struct smc_init_info *ini)
1513 {
1514         if (!ini->rc)
1515                 ini->rc = rc;
1516 }
1517
1518 static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
1519                                         struct smc_clc_msg_proposal *pclc,
1520                                         struct smc_init_info *ini)
1521 {
1522         struct smc_clc_smcd_v2_extension *smcd_v2_ext;
1523         struct smc_clc_v2_extension *smc_v2_ext;
1524         struct smc_clc_msg_smcd *pclc_smcd;
1525         unsigned int matches = 0;
1526         u8 smcd_version;
1527         u8 *eid = NULL;
1528         int i, rc;
1529
1530         if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2))
1531                 goto not_found;
1532
1533         pclc_smcd = smc_get_clc_msg_smcd(pclc);
1534         smc_v2_ext = smc_get_clc_v2_ext(pclc);
1535         smcd_v2_ext = smc_get_clc_smcd_v2_ext(smc_v2_ext);
1536         if (!smcd_v2_ext ||
1537             !smc_v2_ext->hdr.flag.seid) { /* no system EID support for SMCD */
1538                 smc_find_ism_store_rc(SMC_CLC_DECL_NOSEID, ini);
1539                 goto not_found;
1540         }
1541
1542         mutex_lock(&smcd_dev_list.mutex);
1543         if (pclc_smcd->ism.chid)
1544                 /* check for ISM device matching proposed native ISM device */
1545                 smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid),
1546                                        ntohll(pclc_smcd->ism.gid), &matches);
1547         for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) {
1548                 /* check for ISM devices matching proposed non-native ISM
1549                  * devices
1550                  */
1551                 smc_check_ism_v2_match(ini,
1552                                        ntohs(smcd_v2_ext->gidchid[i - 1].chid),
1553                                        ntohll(smcd_v2_ext->gidchid[i - 1].gid),
1554                                        &matches);
1555         }
1556         mutex_unlock(&smcd_dev_list.mutex);
1557
1558         if (ini->ism_dev[0]) {
1559                 smc_ism_get_system_eid(ini->ism_dev[0], &eid);
1560                 if (memcmp(eid, smcd_v2_ext->system_eid, SMC_MAX_EID_LEN))
1561                         goto not_found;
1562         } else {
1563                 goto not_found;
1564         }
1565
1566         /* separate - outside the smcd_dev_list.lock */
1567         smcd_version = ini->smcd_version;
1568         for (i = 0; i < matches; i++) {
1569                 ini->smcd_version = SMC_V2;
1570                 ini->is_smcd = true;
1571                 ini->ism_selected = i;
1572                 rc = smc_listen_ism_init(new_smc, ini);
1573                 if (rc) {
1574                         smc_find_ism_store_rc(rc, ini);
1575                         /* try next active ISM device */
1576                         continue;
1577                 }
1578                 return; /* matching and usable V2 ISM device found */
1579         }
1580         /* no V2 ISM device could be initialized */
1581         ini->smcd_version = smcd_version;       /* restore original value */
1582
1583 not_found:
1584         ini->smcd_version &= ~SMC_V2;
1585         ini->ism_dev[0] = NULL;
1586         ini->is_smcd = false;
1587 }
1588
1589 static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc,
1590                                         struct smc_clc_msg_proposal *pclc,
1591                                         struct smc_init_info *ini)
1592 {
1593         struct smc_clc_msg_smcd *pclc_smcd = smc_get_clc_msg_smcd(pclc);
1594         int rc = 0;
1595
1596         /* check if ISM V1 is available */
1597         if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1))
1598                 goto not_found;
1599         ini->is_smcd = true; /* prepare ISM check */
1600         ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid);
1601         rc = smc_find_ism_device(new_smc, ini);
1602         if (rc)
1603                 goto not_found;
1604         ini->ism_selected = 0;
1605         rc = smc_listen_ism_init(new_smc, ini);
1606         if (!rc)
1607                 return;         /* V1 ISM device found */
1608
1609 not_found:
1610         smc_find_ism_store_rc(rc, ini);
1611         ini->ism_dev[0] = NULL;
1612         ini->is_smcd = false;
1613 }
1614
1615 /* listen worker: register buffers */
1616 static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first)
1617 {
1618         struct smc_connection *conn = &new_smc->conn;
1619
1620         if (!local_first) {
1621                 if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc))
1622                         return SMC_CLC_DECL_ERR_REGRMB;
1623         }
1624         smc_rmb_sync_sg_for_device(&new_smc->conn);
1625
1626         return 0;
1627 }
1628
1629 static int smc_find_rdma_v1_device_serv(struct smc_sock *new_smc,
1630                                         struct smc_clc_msg_proposal *pclc,
1631                                         struct smc_init_info *ini)
1632 {
1633         int rc;
1634
1635         if (!smcr_indicated(ini->smc_type_v1))
1636                 return SMC_CLC_DECL_NOSMCDEV;
1637
1638         /* prepare RDMA check */
1639         ini->ib_lcl = &pclc->lcl;
1640         rc = smc_find_rdma_device(new_smc, ini);
1641         if (rc) {
1642                 /* no RDMA device found */
1643                 if (ini->smc_type_v1 == SMC_TYPE_B)
1644                         /* neither ISM nor RDMA device found */
1645                         rc = SMC_CLC_DECL_NOSMCDEV;
1646                 return rc;
1647         }
1648         rc = smc_listen_rdma_init(new_smc, ini);
1649         if (rc)
1650                 return rc;
1651         return smc_listen_rdma_reg(new_smc, ini->first_contact_local);
1652 }
1653
1654 /* determine the local device matching to proposal */
1655 static int smc_listen_find_device(struct smc_sock *new_smc,
1656                                   struct smc_clc_msg_proposal *pclc,
1657                                   struct smc_init_info *ini)
1658 {
1659         int rc;
1660
1661         /* check for ISM device matching V2 proposed device */
1662         smc_find_ism_v2_device_serv(new_smc, pclc, ini);
1663         if (ini->ism_dev[0])
1664                 return 0;
1665
1666         if (!(ini->smcd_version & SMC_V1))
1667                 return ini->rc ?: SMC_CLC_DECL_NOSMCD2DEV;
1668
1669         /* check for matching IP prefix and subnet length */
1670         rc = smc_listen_prfx_check(new_smc, pclc);
1671         if (rc)
1672                 return ini->rc ?: rc;
1673
1674         /* get vlan id from IP device */
1675         if (smc_vlan_by_tcpsk(new_smc->clcsock, ini))
1676                 return ini->rc ?: SMC_CLC_DECL_GETVLANERR;
1677
1678         /* check for ISM device matching V1 proposed device */
1679         smc_find_ism_v1_device_serv(new_smc, pclc, ini);
1680         if (ini->ism_dev[0])
1681                 return 0;
1682
1683         if (pclc->hdr.typev1 == SMC_TYPE_D)
1684                 /* skip RDMA and decline */
1685                 return ini->rc ?: SMC_CLC_DECL_NOSMCDDEV;
1686
1687         /* check if RDMA is available */
1688         rc = smc_find_rdma_v1_device_serv(new_smc, pclc, ini);
1689         smc_find_ism_store_rc(rc, ini);
1690
1691         return (!rc) ? 0 : ini->rc;
1692 }
1693
1694 /* listen worker: finish RDMA setup */
1695 static int smc_listen_rdma_finish(struct smc_sock *new_smc,
1696                                   struct smc_clc_msg_accept_confirm *cclc,
1697                                   bool local_first)
1698 {
1699         struct smc_link *link = new_smc->conn.lnk;
1700         int reason_code = 0;
1701
1702         if (local_first)
1703                 smc_link_save_peer_info(link, cclc);
1704
1705         if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc))
1706                 return SMC_CLC_DECL_ERR_RTOK;
1707
1708         if (local_first) {
1709                 if (smc_ib_ready_link(link))
1710                         return SMC_CLC_DECL_ERR_RDYLNK;
1711                 /* QP confirmation over RoCE fabric */
1712                 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
1713                 reason_code = smcr_serv_conf_first_link(new_smc);
1714                 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
1715         }
1716         return reason_code;
1717 }
1718
1719 /* setup for connection of server */
1720 static void smc_listen_work(struct work_struct *work)
1721 {
1722         struct smc_sock *new_smc = container_of(work, struct smc_sock,
1723                                                 smc_listen_work);
1724         u8 version = smc_ism_is_v2_capable() ? SMC_V2 : SMC_V1;
1725         struct socket *newclcsock = new_smc->clcsock;
1726         struct smc_clc_msg_accept_confirm *cclc;
1727         struct smc_clc_msg_proposal_area *buf;
1728         struct smc_clc_msg_proposal *pclc;
1729         struct smc_init_info *ini = NULL;
1730         int rc = 0;
1731
1732         if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
1733                 return smc_listen_out_err(new_smc);
1734
1735         if (new_smc->use_fallback) {
1736                 smc_listen_out_connected(new_smc);
1737                 return;
1738         }
1739
1740         /* check if peer is smc capable */
1741         if (!tcp_sk(newclcsock->sk)->syn_smc) {
1742                 smc_switch_to_fallback(new_smc, SMC_CLC_DECL_PEERNOSMC);
1743                 smc_listen_out_connected(new_smc);
1744                 return;
1745         }
1746
1747         /* do inband token exchange -
1748          * wait for and receive SMC Proposal CLC message
1749          */
1750         buf = kzalloc(sizeof(*buf), GFP_KERNEL);
1751         if (!buf) {
1752                 rc = SMC_CLC_DECL_MEM;
1753                 goto out_decl;
1754         }
1755         pclc = (struct smc_clc_msg_proposal *)buf;
1756         rc = smc_clc_wait_msg(new_smc, pclc, sizeof(*buf),
1757                               SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
1758         if (rc)
1759                 goto out_decl;
1760         version = pclc->hdr.version == SMC_V1 ? SMC_V1 : version;
1761
1762         /* IPSec connections opt out of SMC optimizations */
1763         if (using_ipsec(new_smc)) {
1764                 rc = SMC_CLC_DECL_IPSEC;
1765                 goto out_decl;
1766         }
1767
1768         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1769         if (!ini) {
1770                 rc = SMC_CLC_DECL_MEM;
1771                 goto out_decl;
1772         }
1773
1774         /* initial version checking */
1775         rc = smc_listen_v2_check(new_smc, pclc, ini);
1776         if (rc)
1777                 goto out_decl;
1778
1779         mutex_lock(&smc_server_lgr_pending);
1780         smc_close_init(new_smc);
1781         smc_rx_init(new_smc);
1782         smc_tx_init(new_smc);
1783
1784         /* determine ISM or RoCE device used for connection */
1785         rc = smc_listen_find_device(new_smc, pclc, ini);
1786         if (rc)
1787                 goto out_unlock;
1788
1789         /* send SMC Accept CLC message */
1790         rc = smc_clc_send_accept(new_smc, ini->first_contact_local,
1791                                  ini->smcd_version == SMC_V2 ? SMC_V2 : SMC_V1);
1792         if (rc)
1793                 goto out_unlock;
1794
1795         /* SMC-D does not need this lock any more */
1796         if (ini->is_smcd)
1797                 mutex_unlock(&smc_server_lgr_pending);
1798
1799         /* receive SMC Confirm CLC message */
1800         memset(buf, 0, sizeof(*buf));
1801         cclc = (struct smc_clc_msg_accept_confirm *)buf;
1802         rc = smc_clc_wait_msg(new_smc, cclc, sizeof(*buf),
1803                               SMC_CLC_CONFIRM, CLC_WAIT_TIME);
1804         if (rc) {
1805                 if (!ini->is_smcd)
1806                         goto out_unlock;
1807                 goto out_decl;
1808         }
1809
1810         /* finish worker */
1811         if (!ini->is_smcd) {
1812                 rc = smc_listen_rdma_finish(new_smc, cclc,
1813                                             ini->first_contact_local);
1814                 if (rc)
1815                         goto out_unlock;
1816                 mutex_unlock(&smc_server_lgr_pending);
1817         }
1818         smc_conn_save_peer_info(new_smc, cclc);
1819         smc_listen_out_connected(new_smc);
1820         SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini);
1821         goto out_free;
1822
1823 out_unlock:
1824         mutex_unlock(&smc_server_lgr_pending);
1825 out_decl:
1826         smc_listen_decline(new_smc, rc, ini ? ini->first_contact_local : 0,
1827                            version);
1828 out_free:
1829         kfree(ini);
1830         kfree(buf);
1831 }
1832
1833 static void smc_tcp_listen_work(struct work_struct *work)
1834 {
1835         struct smc_sock *lsmc = container_of(work, struct smc_sock,
1836                                              tcp_listen_work);
1837         struct sock *lsk = &lsmc->sk;
1838         struct smc_sock *new_smc;
1839         int rc = 0;
1840
1841         lock_sock(lsk);
1842         while (lsk->sk_state == SMC_LISTEN) {
1843                 rc = smc_clcsock_accept(lsmc, &new_smc);
1844                 if (rc) /* clcsock accept queue empty or error */
1845                         goto out;
1846                 if (!new_smc)
1847                         continue;
1848
1849                 new_smc->listen_smc = lsmc;
1850                 new_smc->use_fallback = lsmc->use_fallback;
1851                 new_smc->fallback_rsn = lsmc->fallback_rsn;
1852                 sock_hold(lsk); /* sock_put in smc_listen_work */
1853                 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1854                 smc_copy_sock_settings_to_smc(new_smc);
1855                 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
1856                 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
1857                 sock_hold(&new_smc->sk); /* sock_put in passive closing */
1858                 if (!queue_work(smc_hs_wq, &new_smc->smc_listen_work))
1859                         sock_put(&new_smc->sk);
1860         }
1861
1862 out:
1863         release_sock(lsk);
1864         sock_put(&lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */
1865 }
1866
1867 static void smc_clcsock_data_ready(struct sock *listen_clcsock)
1868 {
1869         struct smc_sock *lsmc;
1870
1871         lsmc = (struct smc_sock *)
1872                ((uintptr_t)listen_clcsock->sk_user_data & ~SK_USER_DATA_NOCOPY);
1873         if (!lsmc)
1874                 return;
1875         lsmc->clcsk_data_ready(listen_clcsock);
1876         if (lsmc->sk.sk_state == SMC_LISTEN) {
1877                 sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */
1878                 if (!queue_work(smc_hs_wq, &lsmc->tcp_listen_work))
1879                         sock_put(&lsmc->sk);
1880         }
1881 }
1882
1883 static int smc_listen(struct socket *sock, int backlog)
1884 {
1885         struct sock *sk = sock->sk;
1886         struct smc_sock *smc;
1887         int rc;
1888
1889         smc = smc_sk(sk);
1890         lock_sock(sk);
1891
1892         rc = -EINVAL;
1893         if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
1894             smc->connect_nonblock)
1895                 goto out;
1896
1897         rc = 0;
1898         if (sk->sk_state == SMC_LISTEN) {
1899                 sk->sk_max_ack_backlog = backlog;
1900                 goto out;
1901         }
1902         /* some socket options are handled in core, so we could not apply
1903          * them to the clc socket -- copy smc socket options to clc socket
1904          */
1905         smc_copy_sock_settings_to_clc(smc);
1906         if (!smc->use_fallback)
1907                 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1908
1909         /* save original sk_data_ready function and establish
1910          * smc-specific sk_data_ready function
1911          */
1912         smc->clcsk_data_ready = smc->clcsock->sk->sk_data_ready;
1913         smc->clcsock->sk->sk_data_ready = smc_clcsock_data_ready;
1914         smc->clcsock->sk->sk_user_data =
1915                 (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
1916         rc = kernel_listen(smc->clcsock, backlog);
1917         if (rc)
1918                 goto out;
1919         sk->sk_max_ack_backlog = backlog;
1920         sk->sk_ack_backlog = 0;
1921         sk->sk_state = SMC_LISTEN;
1922
1923 out:
1924         release_sock(sk);
1925         return rc;
1926 }
1927
1928 static int smc_accept(struct socket *sock, struct socket *new_sock,
1929                       int flags, bool kern)
1930 {
1931         struct sock *sk = sock->sk, *nsk;
1932         DECLARE_WAITQUEUE(wait, current);
1933         struct smc_sock *lsmc;
1934         long timeo;
1935         int rc = 0;
1936
1937         lsmc = smc_sk(sk);
1938         sock_hold(sk); /* sock_put below */
1939         lock_sock(sk);
1940
1941         if (lsmc->sk.sk_state != SMC_LISTEN) {
1942                 rc = -EINVAL;
1943                 release_sock(sk);
1944                 goto out;
1945         }
1946
1947         /* Wait for an incoming connection */
1948         timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1949         add_wait_queue_exclusive(sk_sleep(sk), &wait);
1950         while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1951                 set_current_state(TASK_INTERRUPTIBLE);
1952                 if (!timeo) {
1953                         rc = -EAGAIN;
1954                         break;
1955                 }
1956                 release_sock(sk);
1957                 timeo = schedule_timeout(timeo);
1958                 /* wakeup by sk_data_ready in smc_listen_work() */
1959                 sched_annotate_sleep();
1960                 lock_sock(sk);
1961                 if (signal_pending(current)) {
1962                         rc = sock_intr_errno(timeo);
1963                         break;
1964                 }
1965         }
1966         set_current_state(TASK_RUNNING);
1967         remove_wait_queue(sk_sleep(sk), &wait);
1968
1969         if (!rc)
1970                 rc = sock_error(nsk);
1971         release_sock(sk);
1972         if (rc)
1973                 goto out;
1974
1975         if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1976                 /* wait till data arrives on the socket */
1977                 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1978                                                                 MSEC_PER_SEC);
1979                 if (smc_sk(nsk)->use_fallback) {
1980                         struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1981
1982                         lock_sock(clcsk);
1983                         if (skb_queue_empty(&clcsk->sk_receive_queue))
1984                                 sk_wait_data(clcsk, &timeo, NULL);
1985                         release_sock(clcsk);
1986                 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1987                         lock_sock(nsk);
1988                         smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1989                         release_sock(nsk);
1990                 }
1991         }
1992
1993 out:
1994         sock_put(sk); /* sock_hold above */
1995         return rc;
1996 }
1997
1998 static int smc_getname(struct socket *sock, struct sockaddr *addr,
1999                        int peer)
2000 {
2001         struct smc_sock *smc;
2002
2003         if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
2004             (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
2005                 return -ENOTCONN;
2006
2007         smc = smc_sk(sock->sk);
2008
2009         return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
2010 }
2011
2012 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
2013 {
2014         struct sock *sk = sock->sk;
2015         struct smc_sock *smc;
2016         int rc = -EPIPE;
2017
2018         smc = smc_sk(sk);
2019         lock_sock(sk);
2020         if ((sk->sk_state != SMC_ACTIVE) &&
2021             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2022             (sk->sk_state != SMC_INIT))
2023                 goto out;
2024
2025         if (msg->msg_flags & MSG_FASTOPEN) {
2026                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2027                         smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2028                 } else {
2029                         rc = -EINVAL;
2030                         goto out;
2031                 }
2032         }
2033
2034         if (smc->use_fallback) {
2035                 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
2036         } else {
2037                 rc = smc_tx_sendmsg(smc, msg, len);
2038                 SMC_STAT_TX_PAYLOAD(smc, len, rc);
2039         }
2040 out:
2041         release_sock(sk);
2042         return rc;
2043 }
2044
2045 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2046                        int flags)
2047 {
2048         struct sock *sk = sock->sk;
2049         struct smc_sock *smc;
2050         int rc = -ENOTCONN;
2051
2052         smc = smc_sk(sk);
2053         lock_sock(sk);
2054         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2055                 /* socket was connected before, no more data to read */
2056                 rc = 0;
2057                 goto out;
2058         }
2059         if ((sk->sk_state == SMC_INIT) ||
2060             (sk->sk_state == SMC_LISTEN) ||
2061             (sk->sk_state == SMC_CLOSED))
2062                 goto out;
2063
2064         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2065                 rc = 0;
2066                 goto out;
2067         }
2068
2069         if (smc->use_fallback) {
2070                 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
2071         } else {
2072                 msg->msg_namelen = 0;
2073                 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
2074                 SMC_STAT_RX_PAYLOAD(smc, rc, rc);
2075         }
2076
2077 out:
2078         release_sock(sk);
2079         return rc;
2080 }
2081
2082 static __poll_t smc_accept_poll(struct sock *parent)
2083 {
2084         struct smc_sock *isk = smc_sk(parent);
2085         __poll_t mask = 0;
2086
2087         spin_lock(&isk->accept_q_lock);
2088         if (!list_empty(&isk->accept_q))
2089                 mask = EPOLLIN | EPOLLRDNORM;
2090         spin_unlock(&isk->accept_q_lock);
2091
2092         return mask;
2093 }
2094
2095 static __poll_t smc_poll(struct file *file, struct socket *sock,
2096                              poll_table *wait)
2097 {
2098         struct sock *sk = sock->sk;
2099         struct smc_sock *smc;
2100         __poll_t mask = 0;
2101
2102         if (!sk)
2103                 return EPOLLNVAL;
2104
2105         smc = smc_sk(sock->sk);
2106         if (smc->use_fallback) {
2107                 /* delegate to CLC child sock */
2108                 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
2109                 sk->sk_err = smc->clcsock->sk->sk_err;
2110         } else {
2111                 if (sk->sk_state != SMC_CLOSED)
2112                         sock_poll_wait(file, sock, wait);
2113                 if (sk->sk_err)
2114                         mask |= EPOLLERR;
2115                 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
2116                     (sk->sk_state == SMC_CLOSED))
2117                         mask |= EPOLLHUP;
2118                 if (sk->sk_state == SMC_LISTEN) {
2119                         /* woken up by sk_data_ready in smc_listen_work() */
2120                         mask |= smc_accept_poll(sk);
2121                 } else if (smc->use_fallback) { /* as result of connect_work()*/
2122                         mask |= smc->clcsock->ops->poll(file, smc->clcsock,
2123                                                            wait);
2124                         sk->sk_err = smc->clcsock->sk->sk_err;
2125                 } else {
2126                         if ((sk->sk_state != SMC_INIT &&
2127                              atomic_read(&smc->conn.sndbuf_space)) ||
2128                             sk->sk_shutdown & SEND_SHUTDOWN) {
2129                                 mask |= EPOLLOUT | EPOLLWRNORM;
2130                         } else {
2131                                 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2132                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2133                         }
2134                         if (atomic_read(&smc->conn.bytes_to_rcv))
2135                                 mask |= EPOLLIN | EPOLLRDNORM;
2136                         if (sk->sk_shutdown & RCV_SHUTDOWN)
2137                                 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2138                         if (sk->sk_state == SMC_APPCLOSEWAIT1)
2139                                 mask |= EPOLLIN;
2140                         if (smc->conn.urg_state == SMC_URG_VALID)
2141                                 mask |= EPOLLPRI;
2142                 }
2143         }
2144
2145         return mask;
2146 }
2147
2148 static int smc_shutdown(struct socket *sock, int how)
2149 {
2150         struct sock *sk = sock->sk;
2151         struct smc_sock *smc;
2152         int rc = -EINVAL;
2153         int rc1 = 0;
2154
2155         smc = smc_sk(sk);
2156
2157         if ((how < SHUT_RD) || (how > SHUT_RDWR))
2158                 return rc;
2159
2160         lock_sock(sk);
2161
2162         rc = -ENOTCONN;
2163         if ((sk->sk_state != SMC_ACTIVE) &&
2164             (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
2165             (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
2166             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2167             (sk->sk_state != SMC_APPCLOSEWAIT2) &&
2168             (sk->sk_state != SMC_APPFINCLOSEWAIT))
2169                 goto out;
2170         if (smc->use_fallback) {
2171                 rc = kernel_sock_shutdown(smc->clcsock, how);
2172                 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
2173                 if (sk->sk_shutdown == SHUTDOWN_MASK)
2174                         sk->sk_state = SMC_CLOSED;
2175                 goto out;
2176         }
2177         switch (how) {
2178         case SHUT_RDWR:         /* shutdown in both directions */
2179                 rc = smc_close_active(smc);
2180                 break;
2181         case SHUT_WR:
2182                 rc = smc_close_shutdown_write(smc);
2183                 break;
2184         case SHUT_RD:
2185                 rc = 0;
2186                 /* nothing more to do because peer is not involved */
2187                 break;
2188         }
2189         if (smc->clcsock)
2190                 rc1 = kernel_sock_shutdown(smc->clcsock, how);
2191         /* map sock_shutdown_cmd constants to sk_shutdown value range */
2192         sk->sk_shutdown |= how + 1;
2193
2194 out:
2195         release_sock(sk);
2196         return rc ? rc : rc1;
2197 }
2198
2199 static int smc_setsockopt(struct socket *sock, int level, int optname,
2200                           sockptr_t optval, unsigned int optlen)
2201 {
2202         struct sock *sk = sock->sk;
2203         struct smc_sock *smc;
2204         int val, rc;
2205
2206         if (level == SOL_TCP && optname == TCP_ULP)
2207                 return -EOPNOTSUPP;
2208
2209         smc = smc_sk(sk);
2210
2211         /* generic setsockopts reaching us here always apply to the
2212          * CLC socket
2213          */
2214         if (unlikely(!smc->clcsock->ops->setsockopt))
2215                 rc = -EOPNOTSUPP;
2216         else
2217                 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
2218                                                    optval, optlen);
2219         if (smc->clcsock->sk->sk_err) {
2220                 sk->sk_err = smc->clcsock->sk->sk_err;
2221                 sk_error_report(sk);
2222         }
2223
2224         if (optlen < sizeof(int))
2225                 return -EINVAL;
2226         if (copy_from_sockptr(&val, optval, sizeof(int)))
2227                 return -EFAULT;
2228
2229         lock_sock(sk);
2230         if (rc || smc->use_fallback)
2231                 goto out;
2232         switch (optname) {
2233         case TCP_FASTOPEN:
2234         case TCP_FASTOPEN_CONNECT:
2235         case TCP_FASTOPEN_KEY:
2236         case TCP_FASTOPEN_NO_COOKIE:
2237                 /* option not supported by SMC */
2238                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2239                         smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2240                 } else {
2241                         rc = -EINVAL;
2242                 }
2243                 break;
2244         case TCP_NODELAY:
2245                 if (sk->sk_state != SMC_INIT &&
2246                     sk->sk_state != SMC_LISTEN &&
2247                     sk->sk_state != SMC_CLOSED) {
2248                         if (val) {
2249                                 SMC_STAT_INC(smc, ndly_cnt);
2250                                 mod_delayed_work(smc->conn.lgr->tx_wq,
2251                                                  &smc->conn.tx_work, 0);
2252                         }
2253                 }
2254                 break;
2255         case TCP_CORK:
2256                 if (sk->sk_state != SMC_INIT &&
2257                     sk->sk_state != SMC_LISTEN &&
2258                     sk->sk_state != SMC_CLOSED) {
2259                         if (!val) {
2260                                 SMC_STAT_INC(smc, cork_cnt);
2261                                 mod_delayed_work(smc->conn.lgr->tx_wq,
2262                                                  &smc->conn.tx_work, 0);
2263                         }
2264                 }
2265                 break;
2266         case TCP_DEFER_ACCEPT:
2267                 smc->sockopt_defer_accept = val;
2268                 break;
2269         default:
2270                 break;
2271         }
2272 out:
2273         release_sock(sk);
2274
2275         return rc;
2276 }
2277
2278 static int smc_getsockopt(struct socket *sock, int level, int optname,
2279                           char __user *optval, int __user *optlen)
2280 {
2281         struct smc_sock *smc;
2282
2283         smc = smc_sk(sock->sk);
2284         /* socket options apply to the CLC socket */
2285         if (unlikely(!smc->clcsock->ops->getsockopt))
2286                 return -EOPNOTSUPP;
2287         return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
2288                                              optval, optlen);
2289 }
2290
2291 static int smc_ioctl(struct socket *sock, unsigned int cmd,
2292                      unsigned long arg)
2293 {
2294         union smc_host_cursor cons, urg;
2295         struct smc_connection *conn;
2296         struct smc_sock *smc;
2297         int answ;
2298
2299         smc = smc_sk(sock->sk);
2300         conn = &smc->conn;
2301         lock_sock(&smc->sk);
2302         if (smc->use_fallback) {
2303                 if (!smc->clcsock) {
2304                         release_sock(&smc->sk);
2305                         return -EBADF;
2306                 }
2307                 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
2308                 release_sock(&smc->sk);
2309                 return answ;
2310         }
2311         switch (cmd) {
2312         case SIOCINQ: /* same as FIONREAD */
2313                 if (smc->sk.sk_state == SMC_LISTEN) {
2314                         release_sock(&smc->sk);
2315                         return -EINVAL;
2316                 }
2317                 if (smc->sk.sk_state == SMC_INIT ||
2318                     smc->sk.sk_state == SMC_CLOSED)
2319                         answ = 0;
2320                 else
2321                         answ = atomic_read(&smc->conn.bytes_to_rcv);
2322                 break;
2323         case SIOCOUTQ:
2324                 /* output queue size (not send + not acked) */
2325                 if (smc->sk.sk_state == SMC_LISTEN) {
2326                         release_sock(&smc->sk);
2327                         return -EINVAL;
2328                 }
2329                 if (smc->sk.sk_state == SMC_INIT ||
2330                     smc->sk.sk_state == SMC_CLOSED)
2331                         answ = 0;
2332                 else
2333                         answ = smc->conn.sndbuf_desc->len -
2334                                         atomic_read(&smc->conn.sndbuf_space);
2335                 break;
2336         case SIOCOUTQNSD:
2337                 /* output queue size (not send only) */
2338                 if (smc->sk.sk_state == SMC_LISTEN) {
2339                         release_sock(&smc->sk);
2340                         return -EINVAL;
2341                 }
2342                 if (smc->sk.sk_state == SMC_INIT ||
2343                     smc->sk.sk_state == SMC_CLOSED)
2344                         answ = 0;
2345                 else
2346                         answ = smc_tx_prepared_sends(&smc->conn);
2347                 break;
2348         case SIOCATMARK:
2349                 if (smc->sk.sk_state == SMC_LISTEN) {
2350                         release_sock(&smc->sk);
2351                         return -EINVAL;
2352                 }
2353                 if (smc->sk.sk_state == SMC_INIT ||
2354                     smc->sk.sk_state == SMC_CLOSED) {
2355                         answ = 0;
2356                 } else {
2357                         smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
2358                         smc_curs_copy(&urg, &conn->urg_curs, conn);
2359                         answ = smc_curs_diff(conn->rmb_desc->len,
2360                                              &cons, &urg) == 1;
2361                 }
2362                 break;
2363         default:
2364                 release_sock(&smc->sk);
2365                 return -ENOIOCTLCMD;
2366         }
2367         release_sock(&smc->sk);
2368
2369         return put_user(answ, (int __user *)arg);
2370 }
2371
2372 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
2373                             int offset, size_t size, int flags)
2374 {
2375         struct sock *sk = sock->sk;
2376         struct smc_sock *smc;
2377         int rc = -EPIPE;
2378
2379         smc = smc_sk(sk);
2380         lock_sock(sk);
2381         if (sk->sk_state != SMC_ACTIVE) {
2382                 release_sock(sk);
2383                 goto out;
2384         }
2385         release_sock(sk);
2386         if (smc->use_fallback) {
2387                 rc = kernel_sendpage(smc->clcsock, page, offset,
2388                                      size, flags);
2389         } else {
2390                 SMC_STAT_INC(smc, sendpage_cnt);
2391                 rc = sock_no_sendpage(sock, page, offset, size, flags);
2392         }
2393
2394 out:
2395         return rc;
2396 }
2397
2398 /* Map the affected portions of the rmbe into an spd, note the number of bytes
2399  * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
2400  * updates till whenever a respective page has been fully processed.
2401  * Note that subsequent recv() calls have to wait till all splice() processing
2402  * completed.
2403  */
2404 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
2405                                struct pipe_inode_info *pipe, size_t len,
2406                                unsigned int flags)
2407 {
2408         struct sock *sk = sock->sk;
2409         struct smc_sock *smc;
2410         int rc = -ENOTCONN;
2411
2412         smc = smc_sk(sk);
2413         lock_sock(sk);
2414         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2415                 /* socket was connected before, no more data to read */
2416                 rc = 0;
2417                 goto out;
2418         }
2419         if (sk->sk_state == SMC_INIT ||
2420             sk->sk_state == SMC_LISTEN ||
2421             sk->sk_state == SMC_CLOSED)
2422                 goto out;
2423
2424         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2425                 rc = 0;
2426                 goto out;
2427         }
2428
2429         if (smc->use_fallback) {
2430                 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
2431                                                     pipe, len, flags);
2432         } else {
2433                 if (*ppos) {
2434                         rc = -ESPIPE;
2435                         goto out;
2436                 }
2437                 if (flags & SPLICE_F_NONBLOCK)
2438                         flags = MSG_DONTWAIT;
2439                 else
2440                         flags = 0;
2441                 SMC_STAT_INC(smc, splice_cnt);
2442                 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
2443         }
2444 out:
2445         release_sock(sk);
2446
2447         return rc;
2448 }
2449
2450 /* must look like tcp */
2451 static const struct proto_ops smc_sock_ops = {
2452         .family         = PF_SMC,
2453         .owner          = THIS_MODULE,
2454         .release        = smc_release,
2455         .bind           = smc_bind,
2456         .connect        = smc_connect,
2457         .socketpair     = sock_no_socketpair,
2458         .accept         = smc_accept,
2459         .getname        = smc_getname,
2460         .poll           = smc_poll,
2461         .ioctl          = smc_ioctl,
2462         .listen         = smc_listen,
2463         .shutdown       = smc_shutdown,
2464         .setsockopt     = smc_setsockopt,
2465         .getsockopt     = smc_getsockopt,
2466         .sendmsg        = smc_sendmsg,
2467         .recvmsg        = smc_recvmsg,
2468         .mmap           = sock_no_mmap,
2469         .sendpage       = smc_sendpage,
2470         .splice_read    = smc_splice_read,
2471 };
2472
2473 static int smc_create(struct net *net, struct socket *sock, int protocol,
2474                       int kern)
2475 {
2476         int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
2477         struct smc_sock *smc;
2478         struct sock *sk;
2479         int rc;
2480
2481         rc = -ESOCKTNOSUPPORT;
2482         if (sock->type != SOCK_STREAM)
2483                 goto out;
2484
2485         rc = -EPROTONOSUPPORT;
2486         if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
2487                 goto out;
2488
2489         rc = -ENOBUFS;
2490         sock->ops = &smc_sock_ops;
2491         sk = smc_sock_alloc(net, sock, protocol);
2492         if (!sk)
2493                 goto out;
2494
2495         /* create internal TCP socket for CLC handshake and fallback */
2496         smc = smc_sk(sk);
2497         smc->use_fallback = false; /* assume rdma capability first */
2498         smc->fallback_rsn = 0;
2499         rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
2500                               &smc->clcsock);
2501         if (rc) {
2502                 sk_common_release(sk);
2503                 goto out;
2504         }
2505         smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
2506         smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
2507
2508 out:
2509         return rc;
2510 }
2511
2512 static const struct net_proto_family smc_sock_family_ops = {
2513         .family = PF_SMC,
2514         .owner  = THIS_MODULE,
2515         .create = smc_create,
2516 };
2517
2518 unsigned int smc_net_id;
2519
2520 static __net_init int smc_net_init(struct net *net)
2521 {
2522         return smc_pnet_net_init(net);
2523 }
2524
2525 static void __net_exit smc_net_exit(struct net *net)
2526 {
2527         smc_pnet_net_exit(net);
2528 }
2529
2530 static __net_init int smc_net_stat_init(struct net *net)
2531 {
2532         return smc_stats_init(net);
2533 }
2534
2535 static void __net_exit smc_net_stat_exit(struct net *net)
2536 {
2537         smc_stats_exit(net);
2538 }
2539
2540 static struct pernet_operations smc_net_ops = {
2541         .init = smc_net_init,
2542         .exit = smc_net_exit,
2543         .id   = &smc_net_id,
2544         .size = sizeof(struct smc_net),
2545 };
2546
2547 static struct pernet_operations smc_net_stat_ops = {
2548         .init = smc_net_stat_init,
2549         .exit = smc_net_stat_exit,
2550 };
2551
2552 static int __init smc_init(void)
2553 {
2554         int rc;
2555
2556         rc = register_pernet_subsys(&smc_net_ops);
2557         if (rc)
2558                 return rc;
2559
2560         rc = register_pernet_subsys(&smc_net_stat_ops);
2561         if (rc)
2562                 return rc;
2563
2564         smc_ism_init();
2565         smc_clc_init();
2566
2567         rc = smc_nl_init();
2568         if (rc)
2569                 goto out_pernet_subsys;
2570
2571         rc = smc_pnet_init();
2572         if (rc)
2573                 goto out_nl;
2574
2575         rc = -ENOMEM;
2576         smc_hs_wq = alloc_workqueue("smc_hs_wq", 0, 0);
2577         if (!smc_hs_wq)
2578                 goto out_pnet;
2579
2580         smc_close_wq = alloc_workqueue("smc_close_wq", 0, 0);
2581         if (!smc_close_wq)
2582                 goto out_alloc_hs_wq;
2583
2584         rc = smc_core_init();
2585         if (rc) {
2586                 pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
2587                 goto out_alloc_wqs;
2588         }
2589
2590         rc = smc_llc_init();
2591         if (rc) {
2592                 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
2593                 goto out_core;
2594         }
2595
2596         rc = smc_cdc_init();
2597         if (rc) {
2598                 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
2599                 goto out_core;
2600         }
2601
2602         rc = proto_register(&smc_proto, 1);
2603         if (rc) {
2604                 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
2605                 goto out_core;
2606         }
2607
2608         rc = proto_register(&smc_proto6, 1);
2609         if (rc) {
2610                 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
2611                 goto out_proto;
2612         }
2613
2614         rc = sock_register(&smc_sock_family_ops);
2615         if (rc) {
2616                 pr_err("%s: sock_register fails with %d\n", __func__, rc);
2617                 goto out_proto6;
2618         }
2619         INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
2620         INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
2621
2622         rc = smc_ib_register_client();
2623         if (rc) {
2624                 pr_err("%s: ib_register fails with %d\n", __func__, rc);
2625                 goto out_sock;
2626         }
2627
2628         static_branch_enable(&tcp_have_smc);
2629         return 0;
2630
2631 out_sock:
2632         sock_unregister(PF_SMC);
2633 out_proto6:
2634         proto_unregister(&smc_proto6);
2635 out_proto:
2636         proto_unregister(&smc_proto);
2637 out_core:
2638         smc_core_exit();
2639 out_alloc_wqs:
2640         destroy_workqueue(smc_close_wq);
2641 out_alloc_hs_wq:
2642         destroy_workqueue(smc_hs_wq);
2643 out_pnet:
2644         smc_pnet_exit();
2645 out_nl:
2646         smc_nl_exit();
2647 out_pernet_subsys:
2648         unregister_pernet_subsys(&smc_net_ops);
2649
2650         return rc;
2651 }
2652
2653 static void __exit smc_exit(void)
2654 {
2655         static_branch_disable(&tcp_have_smc);
2656         sock_unregister(PF_SMC);
2657         smc_core_exit();
2658         smc_ib_unregister_client();
2659         destroy_workqueue(smc_close_wq);
2660         destroy_workqueue(smc_hs_wq);
2661         proto_unregister(&smc_proto6);
2662         proto_unregister(&smc_proto);
2663         smc_pnet_exit();
2664         smc_nl_exit();
2665         unregister_pernet_subsys(&smc_net_stat_ops);
2666         unregister_pernet_subsys(&smc_net_ops);
2667         rcu_barrier();
2668 }
2669
2670 module_init(smc_init);
2671 module_exit(smc_exit);
2672
2673 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
2674 MODULE_DESCRIPTION("smc socket address family");
2675 MODULE_LICENSE("GPL");
2676 MODULE_ALIAS_NETPROTO(PF_SMC);