net/smc/af_smc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
   6  *  applies to SOCK_STREAM sockets only
   7  *  offers an alternative communication option for TCP-protocol sockets
   8  *  applicable with RoCE-cards only
   9  *
  10  *  Initial restrictions:
  11  *    - support for alternate links postponed
  12  *
  13  *  Copyright IBM Corp. 2016, 2018
  14  *
  15  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
  16  *              based on prototype from Frank Blaschka
  17  */
  18
  19 #define KMSG_COMPONENT "smc"
  20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  21
  22 #include <linux/module.h>
  23 #include <linux/socket.h>
  24 #include <linux/workqueue.h>
  25 #include <linux/in.h>
  26 #include <linux/sched/signal.h>
  27 #include <linux/if_vlan.h>
  28 #include <linux/rcupdate_wait.h>
  29
  30 #include <net/sock.h>
  31 #include <net/tcp.h>
  32 #include <net/smc.h>
  33 #include <asm/ioctls.h>
  34
  35 #include <net/net_namespace.h>
  36 #include <net/netns/generic.h>
  37 #include "smc_netns.h"
  38
  39 #include "smc.h"
  40 #include "smc_clc.h"
  41 #include "smc_llc.h"
  42 #include "smc_cdc.h"
  43 #include "smc_core.h"
  44 #include "smc_ib.h"
  45 #include "smc_ism.h"
  46 #include "smc_pnet.h"
  47 #include "smc_tx.h"
  48 #include "smc_rx.h"
  49 #include "smc_close.h"
  50
  51 static DEFINE_MUTEX(smc_server_lgr_pending);    /* serialize link group
  52                                                  * creation on server
  53                                                  */
  54 static DEFINE_MUTEX(smc_client_lgr_pending);    /* serialize link group
  55                                                  * creation on client
  56                                                  */
  57
  58 static void smc_tcp_listen_work(struct work_struct *);
  59 static void smc_connect_work(struct work_struct *);
  60
  61 static void smc_set_keepalive(struct sock *sk, int val)
  62 {
  63         struct smc_sock *smc = smc_sk(sk);
  64
  65         smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
  66 }
  67
  68 static struct smc_hashinfo smc_v4_hashinfo = {
  69         .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
  70 };
  71
  72 static struct smc_hashinfo smc_v6_hashinfo = {
  73         .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
  74 };
  75
  76 int smc_hash_sk(struct sock *sk)
  77 {
  78         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  79         struct hlist_head *head;
  80
  81         head = &h->ht;
  82
  83         write_lock_bh(&h->lock);
  84         sk_add_node(sk, head);
  85         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  86         write_unlock_bh(&h->lock);
  87
  88         return 0;
  89 }
  90 EXPORT_SYMBOL_GPL(smc_hash_sk);
  91
  92 void smc_unhash_sk(struct sock *sk)
  93 {
  94         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  95
  96         write_lock_bh(&h->lock);
  97         if (sk_del_node_init(sk))
  98                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
  99         write_unlock_bh(&h->lock);
 100 }
 101 EXPORT_SYMBOL_GPL(smc_unhash_sk);
 102
 103 struct proto smc_proto = {
 104         .name           = "SMC",
 105         .owner          = THIS_MODULE,
 106         .keepalive      = smc_set_keepalive,
 107         .hash           = smc_hash_sk,
 108         .unhash         = smc_unhash_sk,
 109         .obj_size       = sizeof(struct smc_sock),
 110         .h.smc_hash     = &smc_v4_hashinfo,
 111         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 112 };
 113 EXPORT_SYMBOL_GPL(smc_proto);
 114
 115 struct proto smc_proto6 = {
 116         .name           = "SMC6",
 117         .owner          = THIS_MODULE,
 118         .keepalive      = smc_set_keepalive,
 119         .hash           = smc_hash_sk,
 120         .unhash         = smc_unhash_sk,
 121         .obj_size       = sizeof(struct smc_sock),
 122         .h.smc_hash     = &smc_v6_hashinfo,
 123         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 124 };
 125 EXPORT_SYMBOL_GPL(smc_proto6);
 126
 127 static void smc_restore_fallback_changes(struct smc_sock *smc)
 128 {
 129         smc->clcsock->file->private_data = smc->sk.sk_socket;
 130         smc->clcsock->file = NULL;
 131 }
 132
 133 static int __smc_release(struct smc_sock *smc)
 134 {
 135         struct sock *sk = &smc->sk;
 136         int rc = 0;
 137
 138         if (!smc->use_fallback) {
 139                 rc = smc_close_active(smc);
 140                 sock_set_flag(sk, SOCK_DEAD);
 141                 sk->sk_shutdown |= SHUTDOWN_MASK;
 142         } else {
 143                 if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
 144                         sock_put(sk); /* passive closing */
 145                 if (sk->sk_state == SMC_LISTEN) {
 146                         /* wake up clcsock accept */
 147                         rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR);
 148                 }
 149                 sk->sk_state = SMC_CLOSED;
 150                 sk->sk_state_change(sk);
 151                 smc_restore_fallback_changes(smc);
 152         }
 153
 154         sk->sk_prot->unhash(sk);
 155
 156         if (sk->sk_state == SMC_CLOSED) {
 157                 if (smc->clcsock) {
 158                         release_sock(sk);
 159                         smc_clcsock_release(smc);
 160                         lock_sock(sk);
 161                 }
 162                 if (!smc->use_fallback)
 163                         smc_conn_free(&smc->conn);
 164         }
 165
 166         return rc;
 167 }
 168
 169 static int smc_release(struct socket *sock)
 170 {
 171         struct sock *sk = sock->sk;
 172         struct smc_sock *smc;
 173         int rc = 0;
 174
 175         if (!sk)
 176                 goto out;
 177
 178         sock_hold(sk); /* sock_put below */
 179         smc = smc_sk(sk);
 180
 181         /* cleanup for a dangling non-blocking connect */
 182         if (smc->connect_nonblock && sk->sk_state == SMC_INIT)
 183                 tcp_abort(smc->clcsock->sk, ECONNABORTED);
 184         flush_work(&smc->connect_work);
 185
 186         if (sk->sk_state == SMC_LISTEN)
 187                 /* smc_close_non_accepted() is called and acquires
 188                  * sock lock for child sockets again
 189                  */
 190                 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 191         else
 192                 lock_sock(sk);
 193
 194         rc = __smc_release(smc);
 195
 196         /* detach socket */
 197         sock_orphan(sk);
 198         sock->sk = NULL;
 199         release_sock(sk);
 200
 201         sock_put(sk); /* sock_hold above */
 202         sock_put(sk); /* final sock_put */
 203 out:
 204         return rc;
 205 }
 206
 207 static void smc_destruct(struct sock *sk)
 208 {
 209         if (sk->sk_state != SMC_CLOSED)
 210                 return;
 211         if (!sock_flag(sk, SOCK_DEAD))
 212                 return;
 213
 214         sk_refcnt_debug_dec(sk);
 215 }
 216
 217 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 218                                    int protocol)
 219 {
 220         struct smc_sock *smc;
 221         struct proto *prot;
 222         struct sock *sk;
 223
 224         prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
 225         sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
 226         if (!sk)
 227                 return NULL;
 228
 229         sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
 230         sk->sk_state = SMC_INIT;
 231         sk->sk_destruct = smc_destruct;
 232         sk->sk_protocol = protocol;
 233         smc = smc_sk(sk);
 234         INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 235         INIT_WORK(&smc->connect_work, smc_connect_work);
 236         INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
 237         INIT_LIST_HEAD(&smc->accept_q);
 238         spin_lock_init(&smc->accept_q_lock);
 239         spin_lock_init(&smc->conn.send_lock);
 240         sk->sk_prot->hash(sk);
 241         sk_refcnt_debug_inc(sk);
 242         mutex_init(&smc->clcsock_release_lock);
 243
 244         return sk;
 245 }
 246
 247 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 248                     int addr_len)
 249 {
 250         struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
 251         struct sock *sk = sock->sk;
 252         struct smc_sock *smc;
 253         int rc;
 254
 255         smc = smc_sk(sk);
 256
 257         /* replicate tests from inet_bind(), to be safe wrt. future changes */
 258         rc = -EINVAL;
 259         if (addr_len < sizeof(struct sockaddr_in))
 260                 goto out;
 261
 262         rc = -EAFNOSUPPORT;
 263         if (addr->sin_family != AF_INET &&
 264             addr->sin_family != AF_INET6 &&
 265             addr->sin_family != AF_UNSPEC)
 266                 goto out;
 267         /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
 268         if (addr->sin_family == AF_UNSPEC &&
 269             addr->sin_addr.s_addr != htonl(INADDR_ANY))
 270                 goto out;
 271
 272         lock_sock(sk);
 273
 274         /* Check if socket is already active */
 275         rc = -EINVAL;
 276         if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
 277                 goto out_rel;
 278
 279         smc->clcsock->sk->sk_reuse = sk->sk_reuse;
 280         rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 281
 282 out_rel:
 283         release_sock(sk);
 284 out:
 285         return rc;
 286 }
 287
 288 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
 289                                    unsigned long mask)
 290 {
 291         /* options we don't get control via setsockopt for */
 292         nsk->sk_type = osk->sk_type;
 293         nsk->sk_sndbuf = osk->sk_sndbuf;
 294         nsk->sk_rcvbuf = osk->sk_rcvbuf;
 295         nsk->sk_sndtimeo = osk->sk_sndtimeo;
 296         nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
 297         nsk->sk_mark = osk->sk_mark;
 298         nsk->sk_priority = osk->sk_priority;
 299         nsk->sk_rcvlowat = osk->sk_rcvlowat;
 300         nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
 301         nsk->sk_err = osk->sk_err;
 302
 303         nsk->sk_flags &= ~mask;
 304         nsk->sk_flags |= osk->sk_flags & mask;
 305 }
 306
 307 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
 308                              (1UL << SOCK_KEEPOPEN) | \
 309                              (1UL << SOCK_LINGER) | \
 310                              (1UL << SOCK_BROADCAST) | \
 311                              (1UL << SOCK_TIMESTAMP) | \
 312                              (1UL << SOCK_DBG) | \
 313                              (1UL << SOCK_RCVTSTAMP) | \
 314                              (1UL << SOCK_RCVTSTAMPNS) | \
 315                              (1UL << SOCK_LOCALROUTE) | \
 316                              (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
 317                              (1UL << SOCK_RXQ_OVFL) | \
 318                              (1UL << SOCK_WIFI_STATUS) | \
 319                              (1UL << SOCK_NOFCS) | \
 320                              (1UL << SOCK_FILTER_LOCKED) | \
 321                              (1UL << SOCK_TSTAMP_NEW))
 322 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
 323  * clc socket (since smc is not called for these options from net/core)
 324  */
 325 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
 326 {
 327         smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
 328 }
 329
 330 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
 331                              (1UL << SOCK_KEEPOPEN) | \
 332                              (1UL << SOCK_LINGER) | \
 333                              (1UL << SOCK_DBG))
 334 /* copy only settings and flags relevant for smc from clc to smc socket */
 335 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
 336 {
 337         smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 338 }
 339
 340 /* register a new rmb, send confirm_rkey msg to register with peer */
 341 static int smcr_link_reg_rmb(struct smc_link *link,
 342                              struct smc_buf_desc *rmb_desc, bool conf_rkey)
 343 {
 344         if (!rmb_desc->is_reg_mr[link->link_idx]) {
 345                 /* register memory region for new rmb */
 346                 if (smc_wr_reg_send(link, rmb_desc->mr_rx[link->link_idx])) {
 347                         rmb_desc->is_reg_err = true;
 348                         return -EFAULT;
 349                 }
 350                 rmb_desc->is_reg_mr[link->link_idx] = true;
 351         }
 352         if (!conf_rkey)
 353                 return 0;
 354
 355         /* exchange confirm_rkey msg with peer */
 356         if (!rmb_desc->is_conf_rkey) {
 357                 if (smc_llc_do_confirm_rkey(link, rmb_desc)) {
 358                         rmb_desc->is_reg_err = true;
 359                         return -EFAULT;
 360                 }
 361                 rmb_desc->is_conf_rkey = true;
 362         }
 363         return 0;
 364 }
 365
 366 /* register the new rmb on all links */
 367 static int smcr_lgr_reg_rmbs(struct smc_link_group *lgr,
 368                              struct smc_buf_desc *rmb_desc)
 369 {
 370         int i, rc;
 371
 372         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 373                 if (lgr->lnk[i].state != SMC_LNK_ACTIVE)
 374                         continue;
 375                 rc = smcr_link_reg_rmb(&lgr->lnk[i], rmb_desc, true);
 376                 if (rc)
 377                         return rc;
 378         }
 379         return 0;
 380 }
 381
 382 static int smcr_clnt_conf_first_link(struct smc_sock *smc)
 383 {
 384         struct smc_link *link = smc->conn.lnk;
 385         int rest;
 386         int rc;
 387
 388         /* receive CONFIRM LINK request from server over RoCE fabric */
 389         rest = wait_for_completion_interruptible_timeout(
 390                 &link->llc_confirm,
 391                 SMC_LLC_WAIT_FIRST_TIME);
 392         if (rest <= 0) {
 393                 struct smc_clc_msg_decline dclc;
 394
 395                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 396                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 397                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
 398         }
 399
 400         if (link->llc_confirm_rc)
 401                 return SMC_CLC_DECL_RMBE_EC;
 402
 403         rc = smc_ib_modify_qp_rts(link);
 404         if (rc)
 405                 return SMC_CLC_DECL_ERR_RDYLNK;
 406
 407         smc_wr_remember_qp_attr(link);
 408
 409         if (smcr_link_reg_rmb(link, smc->conn.rmb_desc, false))
 410                 return SMC_CLC_DECL_ERR_REGRMB;
 411
 412         /* send CONFIRM LINK response over RoCE fabric */
 413         rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
 414         if (rc < 0)
 415                 return SMC_CLC_DECL_TIMEOUT_CL;
 416
 417         /* receive ADD LINK request from server over RoCE fabric */
 418         rest = wait_for_completion_interruptible_timeout(&link->llc_add,
 419                                                          SMC_LLC_WAIT_TIME);
 420         if (rest <= 0) {
 421                 struct smc_clc_msg_decline dclc;
 422
 423                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 424                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 425                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
 426         }
 427
 428         /* send add link reject message, only one link supported for now */
 429         rc = smc_llc_send_add_link(link,
 430                                    link->smcibdev->mac[link->ibport - 1],
 431                                    link->gid, SMC_LLC_RESP);
 432         if (rc < 0)
 433                 return SMC_CLC_DECL_TIMEOUT_AL;
 434
 435         smc_llc_link_active(link);
 436
 437         return 0;
 438 }
 439
 440 static void smcr_conn_save_peer_info(struct smc_sock *smc,
 441                                      struct smc_clc_msg_accept_confirm *clc)
 442 {
 443         int bufsize = smc_uncompress_bufsize(clc->rmbe_size);
 444
 445         smc->conn.peer_rmbe_idx = clc->rmbe_idx;
 446         smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
 447         smc->conn.peer_rmbe_size = bufsize;
 448         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 449         smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
 450 }
 451
 452 static void smcd_conn_save_peer_info(struct smc_sock *smc,
 453                                      struct smc_clc_msg_accept_confirm *clc)
 454 {
 455         int bufsize = smc_uncompress_bufsize(clc->dmbe_size);
 456
 457         smc->conn.peer_rmbe_idx = clc->dmbe_idx;
 458         smc->conn.peer_token = clc->token;
 459         /* msg header takes up space in the buffer */
 460         smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
 461         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 462         smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
 463 }
 464
 465 static void smc_conn_save_peer_info(struct smc_sock *smc,
 466                                     struct smc_clc_msg_accept_confirm *clc)
 467 {
 468         if (smc->conn.lgr->is_smcd)
 469                 smcd_conn_save_peer_info(smc, clc);
 470         else
 471                 smcr_conn_save_peer_info(smc, clc);
 472 }
 473
 474 static void smc_link_save_peer_info(struct smc_link *link,
 475                                     struct smc_clc_msg_accept_confirm *clc)
 476 {
 477         link->peer_qpn = ntoh24(clc->qpn);
 478         memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
 479         memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
 480         link->peer_psn = ntoh24(clc->psn);
 481         link->peer_mtu = clc->qp_mtu;
 482 }
 483
 484 static void smc_switch_to_fallback(struct smc_sock *smc)
 485 {
 486         smc->use_fallback = true;
 487         if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
 488                 smc->clcsock->file = smc->sk.sk_socket->file;
 489                 smc->clcsock->file->private_data = smc->clcsock;
 490                 smc->clcsock->wq.fasync_list =
 491                         smc->sk.sk_socket->wq.fasync_list;
 492         }
 493 }
 494
 495 /* fall back during connect */
 496 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
 497 {
 498         smc_switch_to_fallback(smc);
 499         smc->fallback_rsn = reason_code;
 500         smc_copy_sock_settings_to_clc(smc);
 501         smc->connect_nonblock = 0;
 502         if (smc->sk.sk_state == SMC_INIT)
 503                 smc->sk.sk_state = SMC_ACTIVE;
 504         return 0;
 505 }
 506
 507 /* decline and fall back during connect */
 508 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
 509 {
 510         int rc;
 511
 512         if (reason_code < 0) { /* error, fallback is not possible */
 513                 if (smc->sk.sk_state == SMC_INIT)
 514                         sock_put(&smc->sk); /* passive closing */
 515                 return reason_code;
 516         }
 517         if (reason_code != SMC_CLC_DECL_PEERDECL) {
 518                 rc = smc_clc_send_decline(smc, reason_code);
 519                 if (rc < 0) {
 520                         if (smc->sk.sk_state == SMC_INIT)
 521                                 sock_put(&smc->sk); /* passive closing */
 522                         return rc;
 523                 }
 524         }
 525         return smc_connect_fallback(smc, reason_code);
 526 }
 527
 528 /* abort connecting */
 529 static int smc_connect_abort(struct smc_sock *smc, int reason_code,
 530                              int local_contact)
 531 {
 532         bool is_smcd = smc->conn.lgr->is_smcd;
 533
 534         if (local_contact == SMC_FIRST_CONTACT)
 535                 smc_lgr_cleanup_early(&smc->conn);
 536         else
 537                 smc_conn_free(&smc->conn);
 538         if (is_smcd)
 539                 /* there is only one lgr role for SMC-D; use server lock */
 540                 mutex_unlock(&smc_server_lgr_pending);
 541         else
 542                 mutex_unlock(&smc_client_lgr_pending);
 543
 544         smc->connect_nonblock = 0;
 545         return reason_code;
 546 }
 547
 548 /* check if there is a rdma device available for this connection. */
 549 /* called for connect and listen */
 550 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
 551 {
 552         /* PNET table look up: search active ib_device and port
 553          * within same PNETID that also contains the ethernet device
 554          * used for the internal TCP socket
 555          */
 556         smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
 557         if (!ini->ib_dev)
 558                 return SMC_CLC_DECL_NOSMCRDEV;
 559         return 0;
 560 }
 561
 562 /* check if there is an ISM device available for this connection. */
 563 /* called for connect and listen */
 564 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
 565 {
 566         /* Find ISM device with same PNETID as connecting interface  */
 567         smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
 568         if (!ini->ism_dev)
 569                 return SMC_CLC_DECL_NOSMCDDEV;
 570         return 0;
 571 }
 572
 573 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
 574 static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
 575                                       struct smc_init_info *ini)
 576 {
 577         if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev, ini->vlan_id))
 578                 return SMC_CLC_DECL_ISMVLANERR;
 579         return 0;
 580 }
 581
 582 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
 583  * used, the VLAN ID will be registered again during the connection setup.
 584  */
 585 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd,
 586                                         struct smc_init_info *ini)
 587 {
 588         if (!is_smcd)
 589                 return 0;
 590         if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev, ini->vlan_id))
 591                 return SMC_CLC_DECL_CNFERR;
 592         return 0;
 593 }
 594
 595 /* CLC handshake during connect */
 596 static int smc_connect_clc(struct smc_sock *smc, int smc_type,
 597                            struct smc_clc_msg_accept_confirm *aclc,
 598                            struct smc_init_info *ini)
 599 {
 600         int rc = 0;
 601
 602         /* do inband token exchange */
 603         rc = smc_clc_send_proposal(smc, smc_type, ini);
 604         if (rc)
 605                 return rc;
 606         /* receive SMC Accept CLC message */
 607         return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT,
 608                                 CLC_WAIT_TIME);
 609 }
 610
 611 /* setup for RDMA connection of client */
 612 static int smc_connect_rdma(struct smc_sock *smc,
 613                             struct smc_clc_msg_accept_confirm *aclc,
 614                             struct smc_init_info *ini)
 615 {
 616         struct smc_link *link;
 617         int reason_code = 0;
 618
 619         ini->is_smcd = false;
 620         ini->ib_lcl = &aclc->lcl;
 621         ini->ib_clcqpn = ntoh24(aclc->qpn);
 622         ini->srv_first_contact = aclc->hdr.flag;
 623
 624         mutex_lock(&smc_client_lgr_pending);
 625         reason_code = smc_conn_create(smc, ini);
 626         if (reason_code) {
 627                 mutex_unlock(&smc_client_lgr_pending);
 628                 return reason_code;
 629         }
 630         link = smc->conn.lnk;
 631
 632         smc_conn_save_peer_info(smc, aclc);
 633
 634         /* create send buffer and rmb */
 635         if (smc_buf_create(smc, false))
 636                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
 637                                          ini->cln_first_contact);
 638
 639         if (ini->cln_first_contact == SMC_FIRST_CONTACT)
 640                 smc_link_save_peer_info(link, aclc);
 641
 642         if (smc_rmb_rtoken_handling(&smc->conn, link, aclc))
 643                 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RTOK,
 644                                          ini->cln_first_contact);
 645
 646         smc_close_init(smc);
 647         smc_rx_init(smc);
 648
 649         if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
 650                 if (smc_ib_ready_link(link))
 651                         return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK,
 652                                                  ini->cln_first_contact);
 653         } else {
 654                 if (smcr_lgr_reg_rmbs(smc->conn.lgr, smc->conn.rmb_desc))
 655                         return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB,
 656                                                  ini->cln_first_contact);
 657         }
 658         smc_rmb_sync_sg_for_device(&smc->conn);
 659
 660         reason_code = smc_clc_send_confirm(smc);
 661         if (reason_code)
 662                 return smc_connect_abort(smc, reason_code,
 663                                          ini->cln_first_contact);
 664
 665         smc_tx_init(smc);
 666
 667         if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
 668                 /* QP confirmation over RoCE fabric */
 669                 reason_code = smcr_clnt_conf_first_link(smc);
 670                 if (reason_code)
 671                         return smc_connect_abort(smc, reason_code,
 672                                                  ini->cln_first_contact);
 673         }
 674         mutex_unlock(&smc_client_lgr_pending);
 675
 676         smc_copy_sock_settings_to_clc(smc);
 677         smc->connect_nonblock = 0;
 678         if (smc->sk.sk_state == SMC_INIT)
 679                 smc->sk.sk_state = SMC_ACTIVE;
 680
 681         return 0;
 682 }
 683
 684 /* setup for ISM connection of client */
 685 static int smc_connect_ism(struct smc_sock *smc,
 686                            struct smc_clc_msg_accept_confirm *aclc,
 687                            struct smc_init_info *ini)
 688 {
 689         int rc = 0;
 690
 691         ini->is_smcd = true;
 692         ini->ism_gid = aclc->gid;
 693         ini->srv_first_contact = aclc->hdr.flag;
 694
 695         /* there is only one lgr role for SMC-D; use server lock */
 696         mutex_lock(&smc_server_lgr_pending);
 697         rc = smc_conn_create(smc, ini);
 698         if (rc) {
 699                 mutex_unlock(&smc_server_lgr_pending);
 700                 return rc;
 701         }
 702
 703         /* Create send and receive buffers */
 704         if (smc_buf_create(smc, true))
 705                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
 706                                          ini->cln_first_contact);
 707
 708         smc_conn_save_peer_info(smc, aclc);
 709         smc_close_init(smc);
 710         smc_rx_init(smc);
 711         smc_tx_init(smc);
 712
 713         rc = smc_clc_send_confirm(smc);
 714         if (rc)
 715                 return smc_connect_abort(smc, rc, ini->cln_first_contact);
 716         mutex_unlock(&smc_server_lgr_pending);
 717
 718         smc_copy_sock_settings_to_clc(smc);
 719         smc->connect_nonblock = 0;
 720         if (smc->sk.sk_state == SMC_INIT)
 721                 smc->sk.sk_state = SMC_ACTIVE;
 722
 723         return 0;
 724 }
 725
 726 /* perform steps before actually connecting */
 727 static int __smc_connect(struct smc_sock *smc)
 728 {
 729         bool ism_supported = false, rdma_supported = false;
 730         struct smc_clc_msg_accept_confirm aclc;
 731         struct smc_init_info ini = {0};
 732         int smc_type;
 733         int rc = 0;
 734
 735         if (smc->use_fallback)
 736                 return smc_connect_fallback(smc, smc->fallback_rsn);
 737
 738         /* if peer has not signalled SMC-capability, fall back */
 739         if (!tcp_sk(smc->clcsock->sk)->syn_smc)
 740                 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
 741
 742         /* IPSec connections opt out of SMC-R optimizations */
 743         if (using_ipsec(smc))
 744                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC);
 745
 746         /* get vlan id from IP device */
 747         if (smc_vlan_by_tcpsk(smc->clcsock, &ini))
 748                 return smc_connect_decline_fallback(smc,
 749                                                     SMC_CLC_DECL_GETVLANERR);
 750
 751         /* check if there is an ism device available */
 752         if (!smc_find_ism_device(smc, &ini) &&
 753             !smc_connect_ism_vlan_setup(smc, &ini)) {
 754                 /* ISM is supported for this connection */
 755                 ism_supported = true;
 756                 smc_type = SMC_TYPE_D;
 757         }
 758
 759         /* check if there is a rdma device available */
 760         if (!smc_find_rdma_device(smc, &ini)) {
 761                 /* RDMA is supported for this connection */
 762                 rdma_supported = true;
 763                 if (ism_supported)
 764                         smc_type = SMC_TYPE_B; /* both */
 765                 else
 766                         smc_type = SMC_TYPE_R; /* only RDMA */
 767         }
 768
 769         /* if neither ISM nor RDMA are supported, fallback */
 770         if (!rdma_supported && !ism_supported)
 771                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV);
 772
 773         /* perform CLC handshake */
 774         rc = smc_connect_clc(smc, smc_type, &aclc, &ini);
 775         if (rc) {
 776                 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 777                 return smc_connect_decline_fallback(smc, rc);
 778         }
 779
 780         /* depending on previous steps, connect using rdma or ism */
 781         if (rdma_supported && aclc.hdr.path == SMC_TYPE_R)
 782                 rc = smc_connect_rdma(smc, &aclc, &ini);
 783         else if (ism_supported && aclc.hdr.path == SMC_TYPE_D)
 784                 rc = smc_connect_ism(smc, &aclc, &ini);
 785         else
 786                 rc = SMC_CLC_DECL_MODEUNSUPP;
 787         if (rc) {
 788                 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 789                 return smc_connect_decline_fallback(smc, rc);
 790         }
 791
 792         smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 793         return 0;
 794 }
 795
 796 static void smc_connect_work(struct work_struct *work)
 797 {
 798         struct smc_sock *smc = container_of(work, struct smc_sock,
 799                                             connect_work);
 800         long timeo = smc->sk.sk_sndtimeo;
 801         int rc = 0;
 802
 803         if (!timeo)
 804                 timeo = MAX_SCHEDULE_TIMEOUT;
 805         lock_sock(smc->clcsock->sk);
 806         if (smc->clcsock->sk->sk_err) {
 807                 smc->sk.sk_err = smc->clcsock->sk->sk_err;
 808         } else if ((1 << smc->clcsock->sk->sk_state) &
 809                                         (TCPF_SYN_SENT | TCP_SYN_RECV)) {
 810                 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
 811                 if ((rc == -EPIPE) &&
 812                     ((1 << smc->clcsock->sk->sk_state) &
 813                                         (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
 814                         rc = 0;
 815         }
 816         release_sock(smc->clcsock->sk);
 817         lock_sock(&smc->sk);
 818         if (rc != 0 || smc->sk.sk_err) {
 819                 smc->sk.sk_state = SMC_CLOSED;
 820                 if (rc == -EPIPE || rc == -EAGAIN)
 821                         smc->sk.sk_err = EPIPE;
 822                 else if (signal_pending(current))
 823                         smc->sk.sk_err = -sock_intr_errno(timeo);
 824                 sock_put(&smc->sk); /* passive closing */
 825                 goto out;
 826         }
 827
 828         rc = __smc_connect(smc);
 829         if (rc < 0)
 830                 smc->sk.sk_err = -rc;
 831
 832 out:
 833         if (!sock_flag(&smc->sk, SOCK_DEAD)) {
 834                 if (smc->sk.sk_err) {
 835                         smc->sk.sk_state_change(&smc->sk);
 836                 } else { /* allow polling before and after fallback decision */
 837                         smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
 838                         smc->sk.sk_write_space(&smc->sk);
 839                 }
 840         }
 841         release_sock(&smc->sk);
 842 }
 843
 844 static int smc_connect(struct socket *sock, struct sockaddr *addr,
 845                        int alen, int flags)
 846 {
 847         struct sock *sk = sock->sk;
 848         struct smc_sock *smc;
 849         int rc = -EINVAL;
 850
 851         smc = smc_sk(sk);
 852
 853         /* separate smc parameter checking to be safe */
 854         if (alen < sizeof(addr->sa_family))
 855                 goto out_err;
 856         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
 857                 goto out_err;
 858
 859         lock_sock(sk);
 860         switch (sk->sk_state) {
 861         default:
 862                 goto out;
 863         case SMC_ACTIVE:
 864                 rc = -EISCONN;
 865                 goto out;
 866         case SMC_INIT:
 867                 rc = 0;
 868                 break;
 869         }
 870
 871         smc_copy_sock_settings_to_clc(smc);
 872         tcp_sk(smc->clcsock->sk)->syn_smc = 1;
 873         if (smc->connect_nonblock) {
 874                 rc = -EALREADY;
 875                 goto out;
 876         }
 877         rc = kernel_connect(smc->clcsock, addr, alen, flags);
 878         if (rc && rc != -EINPROGRESS)
 879                 goto out;
 880
 881         sock_hold(&smc->sk); /* sock put in passive closing */
 882         if (smc->use_fallback)
 883                 goto out;
 884         if (flags & O_NONBLOCK) {
 885                 if (schedule_work(&smc->connect_work))
 886                         smc->connect_nonblock = 1;
 887                 rc = -EINPROGRESS;
 888         } else {
 889                 rc = __smc_connect(smc);
 890                 if (rc < 0)
 891                         goto out;
 892                 else
 893                         rc = 0; /* success cases including fallback */
 894         }
 895
 896 out:
 897         release_sock(sk);
 898 out_err:
 899         return rc;
 900 }
 901
 902 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
 903 {
 904         struct socket *new_clcsock = NULL;
 905         struct sock *lsk = &lsmc->sk;
 906         struct sock *new_sk;
 907         int rc = -EINVAL;
 908
 909         release_sock(lsk);
 910         new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
 911         if (!new_sk) {
 912                 rc = -ENOMEM;
 913                 lsk->sk_err = ENOMEM;
 914                 *new_smc = NULL;
 915                 lock_sock(lsk);
 916                 goto out;
 917         }
 918         *new_smc = smc_sk(new_sk);
 919
 920         mutex_lock(&lsmc->clcsock_release_lock);
 921         if (lsmc->clcsock)
 922                 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
 923         mutex_unlock(&lsmc->clcsock_release_lock);
 924         lock_sock(lsk);
 925         if  (rc < 0)
 926                 lsk->sk_err = -rc;
 927         if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
 928                 new_sk->sk_prot->unhash(new_sk);
 929                 if (new_clcsock)
 930                         sock_release(new_clcsock);
 931                 new_sk->sk_state = SMC_CLOSED;
 932                 sock_set_flag(new_sk, SOCK_DEAD);
 933                 sock_put(new_sk); /* final */
 934                 *new_smc = NULL;
 935                 goto out;
 936         }
 937
 938         (*new_smc)->clcsock = new_clcsock;
 939 out:
 940         return rc;
 941 }
 942
 943 /* add a just created sock to the accept queue of the listen sock as
 944  * candidate for a following socket accept call from user space
 945  */
 946 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
 947 {
 948         struct smc_sock *par = smc_sk(parent);
 949
 950         sock_hold(sk); /* sock_put in smc_accept_unlink () */
 951         spin_lock(&par->accept_q_lock);
 952         list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
 953         spin_unlock(&par->accept_q_lock);
 954         sk_acceptq_added(parent);
 955 }
 956
 957 /* remove a socket from the accept queue of its parental listening socket */
 958 static void smc_accept_unlink(struct sock *sk)
 959 {
 960         struct smc_sock *par = smc_sk(sk)->listen_smc;
 961
 962         spin_lock(&par->accept_q_lock);
 963         list_del_init(&smc_sk(sk)->accept_q);
 964         spin_unlock(&par->accept_q_lock);
 965         sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
 966         sock_put(sk); /* sock_hold in smc_accept_enqueue */
 967 }
 968
 969 /* remove a sock from the accept queue to bind it to a new socket created
 970  * for a socket accept call from user space
 971  */
 972 struct sock *smc_accept_dequeue(struct sock *parent,
 973                                 struct socket *new_sock)
 974 {
 975         struct smc_sock *isk, *n;
 976         struct sock *new_sk;
 977
 978         list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
 979                 new_sk = (struct sock *)isk;
 980
 981                 smc_accept_unlink(new_sk);
 982                 if (new_sk->sk_state == SMC_CLOSED) {
 983                         new_sk->sk_prot->unhash(new_sk);
 984                         if (isk->clcsock) {
 985                                 sock_release(isk->clcsock);
 986                                 isk->clcsock = NULL;
 987                         }
 988                         sock_put(new_sk); /* final */
 989                         continue;
 990                 }
 991                 if (new_sock) {
 992                         sock_graft(new_sk, new_sock);
 993                         if (isk->use_fallback) {
 994                                 smc_sk(new_sk)->clcsock->file = new_sock->file;
 995                                 isk->clcsock->file->private_data = isk->clcsock;
 996                         }
 997                 }
 998                 return new_sk;
 999         }
1000         return NULL;
1001 }
1002
1003 /* clean up for a created but never accepted sock */
1004 void smc_close_non_accepted(struct sock *sk)
1005 {
1006         struct smc_sock *smc = smc_sk(sk);
1007
1008         sock_hold(sk); /* sock_put below */
1009         lock_sock(sk);
1010         if (!sk->sk_lingertime)
1011                 /* wait for peer closing */
1012                 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
1013         __smc_release(smc);
1014         release_sock(sk);
1015         sock_put(sk); /* sock_hold above */
1016         sock_put(sk); /* final sock_put */
1017 }
1018
1019 static int smcr_serv_conf_first_link(struct smc_sock *smc)
1020 {
1021         struct smc_link *link = smc->conn.lnk;
1022         int rest;
1023         int rc;
1024
1025         if (smcr_link_reg_rmb(link, smc->conn.rmb_desc, false))
1026                 return SMC_CLC_DECL_ERR_REGRMB;
1027
1028         /* send CONFIRM LINK request to client over the RoCE fabric */
1029         rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
1030         if (rc < 0)
1031                 return SMC_CLC_DECL_TIMEOUT_CL;
1032
1033         /* receive CONFIRM LINK response from client over the RoCE fabric */
1034         rest = wait_for_completion_interruptible_timeout(
1035                 &link->llc_confirm_resp,
1036                 SMC_LLC_WAIT_FIRST_TIME);
1037         if (rest <= 0) {
1038                 struct smc_clc_msg_decline dclc;
1039
1040                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1041                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1042                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1043         }
1044
1045         if (link->llc_confirm_resp_rc)
1046                 return SMC_CLC_DECL_RMBE_EC;
1047
1048         /* send ADD LINK request to client over the RoCE fabric */
1049         rc = smc_llc_send_add_link(link,
1050                                    link->smcibdev->mac[link->ibport - 1],
1051                                    link->gid, SMC_LLC_REQ);
1052         if (rc < 0)
1053                 return SMC_CLC_DECL_TIMEOUT_AL;
1054
1055         /* receive ADD LINK response from client over the RoCE fabric */
1056         rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
1057                                                          SMC_LLC_WAIT_TIME);
1058         if (rest <= 0) {
1059                 struct smc_clc_msg_decline dclc;
1060
1061                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1062                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1063                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
1064         }
1065
1066         smc_llc_link_active(link);
1067
1068         return 0;
1069 }
1070
1071 /* listen worker: finish */
1072 static void smc_listen_out(struct smc_sock *new_smc)
1073 {
1074         struct smc_sock *lsmc = new_smc->listen_smc;
1075         struct sock *newsmcsk = &new_smc->sk;
1076
1077         if (lsmc->sk.sk_state == SMC_LISTEN) {
1078                 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1079                 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1080                 release_sock(&lsmc->sk);
1081         } else { /* no longer listening */
1082                 smc_close_non_accepted(newsmcsk);
1083         }
1084
1085         /* Wake up accept */
1086         lsmc->sk.sk_data_ready(&lsmc->sk);
1087         sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1088 }
1089
1090 /* listen worker: finish in state connected */
1091 static void smc_listen_out_connected(struct smc_sock *new_smc)
1092 {
1093         struct sock *newsmcsk = &new_smc->sk;
1094
1095         sk_refcnt_debug_inc(newsmcsk);
1096         if (newsmcsk->sk_state == SMC_INIT)
1097                 newsmcsk->sk_state = SMC_ACTIVE;
1098
1099         smc_listen_out(new_smc);
1100 }
1101
1102 /* listen worker: finish in error state */
1103 static void smc_listen_out_err(struct smc_sock *new_smc)
1104 {
1105         struct sock *newsmcsk = &new_smc->sk;
1106
1107         if (newsmcsk->sk_state == SMC_INIT)
1108                 sock_put(&new_smc->sk); /* passive closing */
1109         newsmcsk->sk_state = SMC_CLOSED;
1110
1111         smc_listen_out(new_smc);
1112 }
1113
1114 /* listen worker: decline and fall back if possible */
1115 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1116                                int local_contact)
1117 {
1118         /* RDMA setup failed, switch back to TCP */
1119         if (local_contact == SMC_FIRST_CONTACT)
1120                 smc_lgr_cleanup_early(&new_smc->conn);
1121         else
1122                 smc_conn_free(&new_smc->conn);
1123         if (reason_code < 0) { /* error, no fallback possible */
1124                 smc_listen_out_err(new_smc);
1125                 return;
1126         }
1127         smc_switch_to_fallback(new_smc);
1128         new_smc->fallback_rsn = reason_code;
1129         if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1130                 if (smc_clc_send_decline(new_smc, reason_code) < 0) {
1131                         smc_listen_out_err(new_smc);
1132                         return;
1133                 }
1134         }
1135         smc_listen_out_connected(new_smc);
1136 }
1137
1138 /* listen worker: check prefixes */
1139 static int smc_listen_prfx_check(struct smc_sock *new_smc,
1140                                  struct smc_clc_msg_proposal *pclc)
1141 {
1142         struct smc_clc_msg_proposal_prefix *pclc_prfx;
1143         struct socket *newclcsock = new_smc->clcsock;
1144
1145         pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1146         if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1147                 return SMC_CLC_DECL_DIFFPREFIX;
1148
1149         return 0;
1150 }
1151
1152 /* listen worker: initialize connection and buffers */
1153 static int smc_listen_rdma_init(struct smc_sock *new_smc,
1154                                 struct smc_init_info *ini)
1155 {
1156         int rc;
1157
1158         /* allocate connection / link group */
1159         rc = smc_conn_create(new_smc, ini);
1160         if (rc)
1161                 return rc;
1162
1163         /* create send buffer and rmb */
1164         if (smc_buf_create(new_smc, false))
1165                 return SMC_CLC_DECL_MEM;
1166
1167         return 0;
1168 }
1169
1170 /* listen worker: initialize connection and buffers for SMC-D */
1171 static int smc_listen_ism_init(struct smc_sock *new_smc,
1172                                struct smc_clc_msg_proposal *pclc,
1173                                struct smc_init_info *ini)
1174 {
1175         struct smc_clc_msg_smcd *pclc_smcd;
1176         int rc;
1177
1178         pclc_smcd = smc_get_clc_msg_smcd(pclc);
1179         ini->ism_gid = pclc_smcd->gid;
1180         rc = smc_conn_create(new_smc, ini);
1181         if (rc)
1182                 return rc;
1183
1184         /* Check if peer can be reached via ISM device */
1185         if (smc_ism_cantalk(new_smc->conn.lgr->peer_gid,
1186                             new_smc->conn.lgr->vlan_id,
1187                             new_smc->conn.lgr->smcd)) {
1188                 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
1189                         smc_lgr_cleanup_early(&new_smc->conn);
1190                 else
1191                         smc_conn_free(&new_smc->conn);
1192                 return SMC_CLC_DECL_SMCDNOTALK;
1193         }
1194
1195         /* Create send and receive buffers */
1196         if (smc_buf_create(new_smc, true)) {
1197                 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
1198                         smc_lgr_cleanup_early(&new_smc->conn);
1199                 else
1200                         smc_conn_free(&new_smc->conn);
1201                 return SMC_CLC_DECL_MEM;
1202         }
1203
1204         return 0;
1205 }
1206
1207 /* listen worker: register buffers */
1208 static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
1209 {
1210         struct smc_connection *conn = &new_smc->conn;
1211
1212         if (local_contact != SMC_FIRST_CONTACT) {
1213                 if (smcr_lgr_reg_rmbs(conn->lgr, conn->rmb_desc))
1214                         return SMC_CLC_DECL_ERR_REGRMB;
1215         }
1216         smc_rmb_sync_sg_for_device(&new_smc->conn);
1217
1218         return 0;
1219 }
1220
1221 /* listen worker: finish RDMA setup */
1222 static int smc_listen_rdma_finish(struct smc_sock *new_smc,
1223                                   struct smc_clc_msg_accept_confirm *cclc,
1224                                   int local_contact)
1225 {
1226         struct smc_link *link = new_smc->conn.lnk;
1227         int reason_code = 0;
1228
1229         if (local_contact == SMC_FIRST_CONTACT)
1230                 smc_link_save_peer_info(link, cclc);
1231
1232         if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc)) {
1233                 reason_code = SMC_CLC_DECL_ERR_RTOK;
1234                 goto decline;
1235         }
1236
1237         if (local_contact == SMC_FIRST_CONTACT) {
1238                 if (smc_ib_ready_link(link)) {
1239                         reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1240                         goto decline;
1241                 }
1242                 /* QP confirmation over RoCE fabric */
1243                 reason_code = smcr_serv_conf_first_link(new_smc);
1244                 if (reason_code)
1245                         goto decline;
1246         }
1247         return 0;
1248
1249 decline:
1250         smc_listen_decline(new_smc, reason_code, local_contact);
1251         return reason_code;
1252 }
1253
1254 /* setup for RDMA connection of server */
1255 static void smc_listen_work(struct work_struct *work)
1256 {
1257         struct smc_sock *new_smc = container_of(work, struct smc_sock,
1258                                                 smc_listen_work);
1259         struct socket *newclcsock = new_smc->clcsock;
1260         struct smc_clc_msg_accept_confirm cclc;
1261         struct smc_clc_msg_proposal *pclc;
1262         struct smc_init_info ini = {0};
1263         bool ism_supported = false;
1264         u8 buf[SMC_CLC_MAX_LEN];
1265         int rc = 0;
1266
1267         if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
1268                 return smc_listen_out_err(new_smc);
1269
1270         if (new_smc->use_fallback) {
1271                 smc_listen_out_connected(new_smc);
1272                 return;
1273         }
1274
1275         /* check if peer is smc capable */
1276         if (!tcp_sk(newclcsock->sk)->syn_smc) {
1277                 smc_switch_to_fallback(new_smc);
1278                 new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;
1279                 smc_listen_out_connected(new_smc);
1280                 return;
1281         }
1282
1283         /* do inband token exchange -
1284          * wait for and receive SMC Proposal CLC message
1285          */
1286         pclc = (struct smc_clc_msg_proposal *)&buf;
1287         rc = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN,
1288                               SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
1289         if (rc)
1290                 goto out_decl;
1291
1292         /* IPSec connections opt out of SMC-R optimizations */
1293         if (using_ipsec(new_smc)) {
1294                 rc = SMC_CLC_DECL_IPSEC;
1295                 goto out_decl;
1296         }
1297
1298         /* check for matching IP prefix and subnet length */
1299         rc = smc_listen_prfx_check(new_smc, pclc);
1300         if (rc)
1301                 goto out_decl;
1302
1303         /* get vlan id from IP device */
1304         if (smc_vlan_by_tcpsk(new_smc->clcsock, &ini)) {
1305                 rc = SMC_CLC_DECL_GETVLANERR;
1306                 goto out_decl;
1307         }
1308
1309         mutex_lock(&smc_server_lgr_pending);
1310         smc_close_init(new_smc);
1311         smc_rx_init(new_smc);
1312         smc_tx_init(new_smc);
1313
1314         /* check if ISM is available */
1315         if (pclc->hdr.path == SMC_TYPE_D || pclc->hdr.path == SMC_TYPE_B) {
1316                 ini.is_smcd = true; /* prepare ISM check */
1317                 rc = smc_find_ism_device(new_smc, &ini);
1318                 if (!rc)
1319                         rc = smc_listen_ism_init(new_smc, pclc, &ini);
1320                 if (!rc)
1321                         ism_supported = true;
1322                 else if (pclc->hdr.path == SMC_TYPE_D)
1323                         goto out_unlock; /* skip RDMA and decline */
1324         }
1325
1326         /* check if RDMA is available */
1327         if (!ism_supported) { /* SMC_TYPE_R or SMC_TYPE_B */
1328                 /* prepare RDMA check */
1329                 ini.is_smcd = false;
1330                 ini.ism_dev = NULL;
1331                 ini.ib_lcl = &pclc->lcl;
1332                 rc = smc_find_rdma_device(new_smc, &ini);
1333                 if (rc) {
1334                         /* no RDMA device found */
1335                         if (pclc->hdr.path == SMC_TYPE_B)
1336                                 /* neither ISM nor RDMA device found */
1337                                 rc = SMC_CLC_DECL_NOSMCDEV;
1338                         goto out_unlock;
1339                 }
1340                 rc = smc_listen_rdma_init(new_smc, &ini);
1341                 if (rc)
1342                         goto out_unlock;
1343                 rc = smc_listen_rdma_reg(new_smc, ini.cln_first_contact);
1344                 if (rc)
1345                         goto out_unlock;
1346         }
1347
1348         /* send SMC Accept CLC message */
1349         rc = smc_clc_send_accept(new_smc, ini.cln_first_contact);
1350         if (rc)
1351                 goto out_unlock;
1352
1353         /* SMC-D does not need this lock any more */
1354         if (ism_supported)
1355                 mutex_unlock(&smc_server_lgr_pending);
1356
1357         /* receive SMC Confirm CLC message */
1358         rc = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
1359                               SMC_CLC_CONFIRM, CLC_WAIT_TIME);
1360         if (rc) {
1361                 if (!ism_supported)
1362                         goto out_unlock;
1363                 goto out_decl;
1364         }
1365
1366         /* finish worker */
1367         if (!ism_supported) {
1368                 rc = smc_listen_rdma_finish(new_smc, &cclc,
1369                                             ini.cln_first_contact);
1370                 mutex_unlock(&smc_server_lgr_pending);
1371                 if (rc)
1372                         return;
1373         }
1374         smc_conn_save_peer_info(new_smc, &cclc);
1375         smc_listen_out_connected(new_smc);
1376         return;
1377
1378 out_unlock:
1379         mutex_unlock(&smc_server_lgr_pending);
1380 out_decl:
1381         smc_listen_decline(new_smc, rc, ini.cln_first_contact);
1382 }
1383
1384 static void smc_tcp_listen_work(struct work_struct *work)
1385 {
1386         struct smc_sock *lsmc = container_of(work, struct smc_sock,
1387                                              tcp_listen_work);
1388         struct sock *lsk = &lsmc->sk;
1389         struct smc_sock *new_smc;
1390         int rc = 0;
1391
1392         lock_sock(lsk);
1393         while (lsk->sk_state == SMC_LISTEN) {
1394                 rc = smc_clcsock_accept(lsmc, &new_smc);
1395                 if (rc)
1396                         goto out;
1397                 if (!new_smc)
1398                         continue;
1399
1400                 new_smc->listen_smc = lsmc;
1401                 new_smc->use_fallback = lsmc->use_fallback;
1402                 new_smc->fallback_rsn = lsmc->fallback_rsn;
1403                 sock_hold(lsk); /* sock_put in smc_listen_work */
1404                 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1405                 smc_copy_sock_settings_to_smc(new_smc);
1406                 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
1407                 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
1408                 sock_hold(&new_smc->sk); /* sock_put in passive closing */
1409                 if (!schedule_work(&new_smc->smc_listen_work))
1410                         sock_put(&new_smc->sk);
1411         }
1412
1413 out:
1414         release_sock(lsk);
1415         sock_put(&lsmc->sk); /* sock_hold in smc_listen */
1416 }
1417
1418 static int smc_listen(struct socket *sock, int backlog)
1419 {
1420         struct sock *sk = sock->sk;
1421         struct smc_sock *smc;
1422         int rc;
1423
1424         smc = smc_sk(sk);
1425         lock_sock(sk);
1426
1427         rc = -EINVAL;
1428         if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
1429             smc->connect_nonblock)
1430                 goto out;
1431
1432         rc = 0;
1433         if (sk->sk_state == SMC_LISTEN) {
1434                 sk->sk_max_ack_backlog = backlog;
1435                 goto out;
1436         }
1437         /* some socket options are handled in core, so we could not apply
1438          * them to the clc socket -- copy smc socket options to clc socket
1439          */
1440         smc_copy_sock_settings_to_clc(smc);
1441         if (!smc->use_fallback)
1442                 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1443
1444         rc = kernel_listen(smc->clcsock, backlog);
1445         if (rc)
1446                 goto out;
1447         sk->sk_max_ack_backlog = backlog;
1448         sk->sk_ack_backlog = 0;
1449         sk->sk_state = SMC_LISTEN;
1450         sock_hold(sk); /* sock_hold in tcp_listen_worker */
1451         if (!schedule_work(&smc->tcp_listen_work))
1452                 sock_put(sk);
1453
1454 out:
1455         release_sock(sk);
1456         return rc;
1457 }
1458
1459 static int smc_accept(struct socket *sock, struct socket *new_sock,
1460                       int flags, bool kern)
1461 {
1462         struct sock *sk = sock->sk, *nsk;
1463         DECLARE_WAITQUEUE(wait, current);
1464         struct smc_sock *lsmc;
1465         long timeo;
1466         int rc = 0;
1467
1468         lsmc = smc_sk(sk);
1469         sock_hold(sk); /* sock_put below */
1470         lock_sock(sk);
1471
1472         if (lsmc->sk.sk_state != SMC_LISTEN) {
1473                 rc = -EINVAL;
1474                 release_sock(sk);
1475                 goto out;
1476         }
1477
1478         /* Wait for an incoming connection */
1479         timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1480         add_wait_queue_exclusive(sk_sleep(sk), &wait);
1481         while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1482                 set_current_state(TASK_INTERRUPTIBLE);
1483                 if (!timeo) {
1484                         rc = -EAGAIN;
1485                         break;
1486                 }
1487                 release_sock(sk);
1488                 timeo = schedule_timeout(timeo);
1489                 /* wakeup by sk_data_ready in smc_listen_work() */
1490                 sched_annotate_sleep();
1491                 lock_sock(sk);
1492                 if (signal_pending(current)) {
1493                         rc = sock_intr_errno(timeo);
1494                         break;
1495                 }
1496         }
1497         set_current_state(TASK_RUNNING);
1498         remove_wait_queue(sk_sleep(sk), &wait);
1499
1500         if (!rc)
1501                 rc = sock_error(nsk);
1502         release_sock(sk);
1503         if (rc)
1504                 goto out;
1505
1506         if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1507                 /* wait till data arrives on the socket */
1508                 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1509                                                                 MSEC_PER_SEC);
1510                 if (smc_sk(nsk)->use_fallback) {
1511                         struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1512
1513                         lock_sock(clcsk);
1514                         if (skb_queue_empty(&clcsk->sk_receive_queue))
1515                                 sk_wait_data(clcsk, &timeo, NULL);
1516                         release_sock(clcsk);
1517                 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1518                         lock_sock(nsk);
1519                         smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1520                         release_sock(nsk);
1521                 }
1522         }
1523
1524 out:
1525         sock_put(sk); /* sock_hold above */
1526         return rc;
1527 }
1528
1529 static int smc_getname(struct socket *sock, struct sockaddr *addr,
1530                        int peer)
1531 {
1532         struct smc_sock *smc;
1533
1534         if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
1535             (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
1536                 return -ENOTCONN;
1537
1538         smc = smc_sk(sock->sk);
1539
1540         return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
1541 }
1542
1543 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
1544 {
1545         struct sock *sk = sock->sk;
1546         struct smc_sock *smc;
1547         int rc = -EPIPE;
1548
1549         smc = smc_sk(sk);
1550         lock_sock(sk);
1551         if ((sk->sk_state != SMC_ACTIVE) &&
1552             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1553             (sk->sk_state != SMC_INIT))
1554                 goto out;
1555
1556         if (msg->msg_flags & MSG_FASTOPEN) {
1557                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
1558                         smc_switch_to_fallback(smc);
1559                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1560                 } else {
1561                         rc = -EINVAL;
1562                         goto out;
1563                 }
1564         }
1565
1566         if (smc->use_fallback)
1567                 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
1568         else
1569                 rc = smc_tx_sendmsg(smc, msg, len);
1570 out:
1571         release_sock(sk);
1572         return rc;
1573 }
1574
1575 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
1576                        int flags)
1577 {
1578         struct sock *sk = sock->sk;
1579         struct smc_sock *smc;
1580         int rc = -ENOTCONN;
1581
1582         smc = smc_sk(sk);
1583         lock_sock(sk);
1584         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
1585                 /* socket was connected before, no more data to read */
1586                 rc = 0;
1587                 goto out;
1588         }
1589         if ((sk->sk_state == SMC_INIT) ||
1590             (sk->sk_state == SMC_LISTEN) ||
1591             (sk->sk_state == SMC_CLOSED))
1592                 goto out;
1593
1594         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1595                 rc = 0;
1596                 goto out;
1597         }
1598
1599         if (smc->use_fallback) {
1600                 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
1601         } else {
1602                 msg->msg_namelen = 0;
1603                 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
1604         }
1605
1606 out:
1607         release_sock(sk);
1608         return rc;
1609 }
1610
1611 static __poll_t smc_accept_poll(struct sock *parent)
1612 {
1613         struct smc_sock *isk = smc_sk(parent);
1614         __poll_t mask = 0;
1615
1616         spin_lock(&isk->accept_q_lock);
1617         if (!list_empty(&isk->accept_q))
1618                 mask = EPOLLIN | EPOLLRDNORM;
1619         spin_unlock(&isk->accept_q_lock);
1620
1621         return mask;
1622 }
1623
1624 static __poll_t smc_poll(struct file *file, struct socket *sock,
1625                              poll_table *wait)
1626 {
1627         struct sock *sk = sock->sk;
1628         struct smc_sock *smc;
1629         __poll_t mask = 0;
1630
1631         if (!sk)
1632                 return EPOLLNVAL;
1633
1634         smc = smc_sk(sock->sk);
1635         if (smc->use_fallback) {
1636                 /* delegate to CLC child sock */
1637                 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
1638                 sk->sk_err = smc->clcsock->sk->sk_err;
1639         } else {
1640                 if (sk->sk_state != SMC_CLOSED)
1641                         sock_poll_wait(file, sock, wait);
1642                 if (sk->sk_err)
1643                         mask |= EPOLLERR;
1644                 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
1645                     (sk->sk_state == SMC_CLOSED))
1646                         mask |= EPOLLHUP;
1647                 if (sk->sk_state == SMC_LISTEN) {
1648                         /* woken up by sk_data_ready in smc_listen_work() */
1649                         mask |= smc_accept_poll(sk);
1650                 } else if (smc->use_fallback) { /* as result of connect_work()*/
1651                         mask |= smc->clcsock->ops->poll(file, smc->clcsock,
1652                                                            wait);
1653                         sk->sk_err = smc->clcsock->sk->sk_err;
1654                 } else {
1655                         if ((sk->sk_state != SMC_INIT &&
1656                              atomic_read(&smc->conn.sndbuf_space)) ||
1657                             sk->sk_shutdown & SEND_SHUTDOWN) {
1658                                 mask |= EPOLLOUT | EPOLLWRNORM;
1659                         } else {
1660                                 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1661                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1662                         }
1663                         if (atomic_read(&smc->conn.bytes_to_rcv))
1664                                 mask |= EPOLLIN | EPOLLRDNORM;
1665                         if (sk->sk_shutdown & RCV_SHUTDOWN)
1666                                 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
1667                         if (sk->sk_state == SMC_APPCLOSEWAIT1)
1668                                 mask |= EPOLLIN;
1669                         if (smc->conn.urg_state == SMC_URG_VALID)
1670                                 mask |= EPOLLPRI;
1671                 }
1672         }
1673
1674         return mask;
1675 }
1676
1677 static int smc_shutdown(struct socket *sock, int how)
1678 {
1679         struct sock *sk = sock->sk;
1680         struct smc_sock *smc;
1681         int rc = -EINVAL;
1682         int rc1 = 0;
1683
1684         smc = smc_sk(sk);
1685
1686         if ((how < SHUT_RD) || (how > SHUT_RDWR))
1687                 return rc;
1688
1689         lock_sock(sk);
1690
1691         rc = -ENOTCONN;
1692         if ((sk->sk_state != SMC_ACTIVE) &&
1693             (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
1694             (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
1695             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1696             (sk->sk_state != SMC_APPCLOSEWAIT2) &&
1697             (sk->sk_state != SMC_APPFINCLOSEWAIT))
1698                 goto out;
1699         if (smc->use_fallback) {
1700                 rc = kernel_sock_shutdown(smc->clcsock, how);
1701                 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
1702                 if (sk->sk_shutdown == SHUTDOWN_MASK)
1703                         sk->sk_state = SMC_CLOSED;
1704                 goto out;
1705         }
1706         switch (how) {
1707         case SHUT_RDWR:         /* shutdown in both directions */
1708                 rc = smc_close_active(smc);
1709                 break;
1710         case SHUT_WR:
1711                 rc = smc_close_shutdown_write(smc);
1712                 break;
1713         case SHUT_RD:
1714                 rc = 0;
1715                 /* nothing more to do because peer is not involved */
1716                 break;
1717         }
1718         if (smc->clcsock)
1719                 rc1 = kernel_sock_shutdown(smc->clcsock, how);
1720         /* map sock_shutdown_cmd constants to sk_shutdown value range */
1721         sk->sk_shutdown |= how + 1;
1722
1723 out:
1724         release_sock(sk);
1725         return rc ? rc : rc1;
1726 }
1727
1728 static int smc_setsockopt(struct socket *sock, int level, int optname,
1729                           char __user *optval, unsigned int optlen)
1730 {
1731         struct sock *sk = sock->sk;
1732         struct smc_sock *smc;
1733         int val, rc;
1734
1735         smc = smc_sk(sk);
1736
1737         /* generic setsockopts reaching us here always apply to the
1738          * CLC socket
1739          */
1740         rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
1741                                            optval, optlen);
1742         if (smc->clcsock->sk->sk_err) {
1743                 sk->sk_err = smc->clcsock->sk->sk_err;
1744                 sk->sk_error_report(sk);
1745         }
1746
1747         if (optlen < sizeof(int))
1748                 return -EINVAL;
1749         if (get_user(val, (int __user *)optval))
1750                 return -EFAULT;
1751
1752         lock_sock(sk);
1753         if (rc || smc->use_fallback)
1754                 goto out;
1755         switch (optname) {
1756         case TCP_ULP:
1757         case TCP_FASTOPEN:
1758         case TCP_FASTOPEN_CONNECT:
1759         case TCP_FASTOPEN_KEY:
1760         case TCP_FASTOPEN_NO_COOKIE:
1761                 /* option not supported by SMC */
1762                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
1763                         smc_switch_to_fallback(smc);
1764                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1765                 } else {
1766                         rc = -EINVAL;
1767                 }
1768                 break;
1769         case TCP_NODELAY:
1770                 if (sk->sk_state != SMC_INIT &&
1771                     sk->sk_state != SMC_LISTEN &&
1772                     sk->sk_state != SMC_CLOSED) {
1773                         if (val)
1774                                 mod_delayed_work(system_wq, &smc->conn.tx_work,
1775                                                  0);
1776                 }
1777                 break;
1778         case TCP_CORK:
1779                 if (sk->sk_state != SMC_INIT &&
1780                     sk->sk_state != SMC_LISTEN &&
1781                     sk->sk_state != SMC_CLOSED) {
1782                         if (!val)
1783                                 mod_delayed_work(system_wq, &smc->conn.tx_work,
1784                                                  0);
1785                 }
1786                 break;
1787         case TCP_DEFER_ACCEPT:
1788                 smc->sockopt_defer_accept = val;
1789                 break;
1790         default:
1791                 break;
1792         }
1793 out:
1794         release_sock(sk);
1795
1796         return rc;
1797 }
1798
1799 static int smc_getsockopt(struct socket *sock, int level, int optname,
1800                           char __user *optval, int __user *optlen)
1801 {
1802         struct smc_sock *smc;
1803
1804         smc = smc_sk(sock->sk);
1805         /* socket options apply to the CLC socket */
1806         return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
1807                                              optval, optlen);
1808 }
1809
1810 static int smc_ioctl(struct socket *sock, unsigned int cmd,
1811                      unsigned long arg)
1812 {
1813         union smc_host_cursor cons, urg;
1814         struct smc_connection *conn;
1815         struct smc_sock *smc;
1816         int answ;
1817
1818         smc = smc_sk(sock->sk);
1819         conn = &smc->conn;
1820         lock_sock(&smc->sk);
1821         if (smc->use_fallback) {
1822                 if (!smc->clcsock) {
1823                         release_sock(&smc->sk);
1824                         return -EBADF;
1825                 }
1826                 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
1827                 release_sock(&smc->sk);
1828                 return answ;
1829         }
1830         switch (cmd) {
1831         case SIOCINQ: /* same as FIONREAD */
1832                 if (smc->sk.sk_state == SMC_LISTEN) {
1833                         release_sock(&smc->sk);
1834                         return -EINVAL;
1835                 }
1836                 if (smc->sk.sk_state == SMC_INIT ||
1837                     smc->sk.sk_state == SMC_CLOSED)
1838                         answ = 0;
1839                 else
1840                         answ = atomic_read(&smc->conn.bytes_to_rcv);
1841                 break;
1842         case SIOCOUTQ:
1843                 /* output queue size (not send + not acked) */
1844                 if (smc->sk.sk_state == SMC_LISTEN) {
1845                         release_sock(&smc->sk);
1846                         return -EINVAL;
1847                 }
1848                 if (smc->sk.sk_state == SMC_INIT ||
1849                     smc->sk.sk_state == SMC_CLOSED)
1850                         answ = 0;
1851                 else
1852                         answ = smc->conn.sndbuf_desc->len -
1853                                         atomic_read(&smc->conn.sndbuf_space);
1854                 break;
1855         case SIOCOUTQNSD:
1856                 /* output queue size (not send only) */
1857                 if (smc->sk.sk_state == SMC_LISTEN) {
1858                         release_sock(&smc->sk);
1859                         return -EINVAL;
1860                 }
1861                 if (smc->sk.sk_state == SMC_INIT ||
1862                     smc->sk.sk_state == SMC_CLOSED)
1863                         answ = 0;
1864                 else
1865                         answ = smc_tx_prepared_sends(&smc->conn);
1866                 break;
1867         case SIOCATMARK:
1868                 if (smc->sk.sk_state == SMC_LISTEN) {
1869                         release_sock(&smc->sk);
1870                         return -EINVAL;
1871                 }
1872                 if (smc->sk.sk_state == SMC_INIT ||
1873                     smc->sk.sk_state == SMC_CLOSED) {
1874                         answ = 0;
1875                 } else {
1876                         smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
1877                         smc_curs_copy(&urg, &conn->urg_curs, conn);
1878                         answ = smc_curs_diff(conn->rmb_desc->len,
1879                                              &cons, &urg) == 1;
1880                 }
1881                 break;
1882         default:
1883                 release_sock(&smc->sk);
1884                 return -ENOIOCTLCMD;
1885         }
1886         release_sock(&smc->sk);
1887
1888         return put_user(answ, (int __user *)arg);
1889 }
1890
1891 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
1892                             int offset, size_t size, int flags)
1893 {
1894         struct sock *sk = sock->sk;
1895         struct smc_sock *smc;
1896         int rc = -EPIPE;
1897
1898         smc = smc_sk(sk);
1899         lock_sock(sk);
1900         if (sk->sk_state != SMC_ACTIVE) {
1901                 release_sock(sk);
1902                 goto out;
1903         }
1904         release_sock(sk);
1905         if (smc->use_fallback)
1906                 rc = kernel_sendpage(smc->clcsock, page, offset,
1907                                      size, flags);
1908         else
1909                 rc = sock_no_sendpage(sock, page, offset, size, flags);
1910
1911 out:
1912         return rc;
1913 }
1914
1915 /* Map the affected portions of the rmbe into an spd, note the number of bytes
1916  * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
1917  * updates till whenever a respective page has been fully processed.
1918  * Note that subsequent recv() calls have to wait till all splice() processing
1919  * completed.
1920  */
1921 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
1922                                struct pipe_inode_info *pipe, size_t len,
1923                                unsigned int flags)
1924 {
1925         struct sock *sk = sock->sk;
1926         struct smc_sock *smc;
1927         int rc = -ENOTCONN;
1928
1929         smc = smc_sk(sk);
1930         lock_sock(sk);
1931         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
1932                 /* socket was connected before, no more data to read */
1933                 rc = 0;
1934                 goto out;
1935         }
1936         if (sk->sk_state == SMC_INIT ||
1937             sk->sk_state == SMC_LISTEN ||
1938             sk->sk_state == SMC_CLOSED)
1939                 goto out;
1940
1941         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1942                 rc = 0;
1943                 goto out;
1944         }
1945
1946         if (smc->use_fallback) {
1947                 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
1948                                                     pipe, len, flags);
1949         } else {
1950                 if (*ppos) {
1951                         rc = -ESPIPE;
1952                         goto out;
1953                 }
1954                 if (flags & SPLICE_F_NONBLOCK)
1955                         flags = MSG_DONTWAIT;
1956                 else
1957                         flags = 0;
1958                 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
1959         }
1960 out:
1961         release_sock(sk);
1962
1963         return rc;
1964 }
1965
1966 /* must look like tcp */
1967 static const struct proto_ops smc_sock_ops = {
1968         .family         = PF_SMC,
1969         .owner          = THIS_MODULE,
1970         .release        = smc_release,
1971         .bind           = smc_bind,
1972         .connect        = smc_connect,
1973         .socketpair     = sock_no_socketpair,
1974         .accept         = smc_accept,
1975         .getname        = smc_getname,
1976         .poll           = smc_poll,
1977         .ioctl          = smc_ioctl,
1978         .listen         = smc_listen,
1979         .shutdown       = smc_shutdown,
1980         .setsockopt     = smc_setsockopt,
1981         .getsockopt     = smc_getsockopt,
1982         .sendmsg        = smc_sendmsg,
1983         .recvmsg        = smc_recvmsg,
1984         .mmap           = sock_no_mmap,
1985         .sendpage       = smc_sendpage,
1986         .splice_read    = smc_splice_read,
1987 };
1988
1989 static int smc_create(struct net *net, struct socket *sock, int protocol,
1990                       int kern)
1991 {
1992         int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
1993         struct smc_sock *smc;
1994         struct sock *sk;
1995         int rc;
1996
1997         rc = -ESOCKTNOSUPPORT;
1998         if (sock->type != SOCK_STREAM)
1999                 goto out;
2000
2001         rc = -EPROTONOSUPPORT;
2002         if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
2003                 goto out;
2004
2005         rc = -ENOBUFS;
2006         sock->ops = &smc_sock_ops;
2007         sk = smc_sock_alloc(net, sock, protocol);
2008         if (!sk)
2009                 goto out;
2010
2011         /* create internal TCP socket for CLC handshake and fallback */
2012         smc = smc_sk(sk);
2013         smc->use_fallback = false; /* assume rdma capability first */
2014         smc->fallback_rsn = 0;
2015         rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
2016                               &smc->clcsock);
2017         if (rc) {
2018                 sk_common_release(sk);
2019                 goto out;
2020         }
2021         smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
2022         smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
2023
2024 out:
2025         return rc;
2026 }
2027
2028 static const struct net_proto_family smc_sock_family_ops = {
2029         .family = PF_SMC,
2030         .owner  = THIS_MODULE,
2031         .create = smc_create,
2032 };
2033
2034 unsigned int smc_net_id;
2035
2036 static __net_init int smc_net_init(struct net *net)
2037 {
2038         return smc_pnet_net_init(net);
2039 }
2040
2041 static void __net_exit smc_net_exit(struct net *net)
2042 {
2043         smc_pnet_net_exit(net);
2044 }
2045
2046 static struct pernet_operations smc_net_ops = {
2047         .init = smc_net_init,
2048         .exit = smc_net_exit,
2049         .id   = &smc_net_id,
2050         .size = sizeof(struct smc_net),
2051 };
2052
2053 static int __init smc_init(void)
2054 {
2055         int rc;
2056
2057         rc = register_pernet_subsys(&smc_net_ops);
2058         if (rc)
2059                 return rc;
2060
2061         rc = smc_pnet_init();
2062         if (rc)
2063                 goto out_pernet_subsys;
2064
2065         rc = smc_core_init();
2066         if (rc) {
2067                 pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
2068                 goto out_pnet;
2069         }
2070
2071         rc = smc_llc_init();
2072         if (rc) {
2073                 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
2074                 goto out_core;
2075         }
2076
2077         rc = smc_cdc_init();
2078         if (rc) {
2079                 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
2080                 goto out_core;
2081         }
2082
2083         rc = proto_register(&smc_proto, 1);
2084         if (rc) {
2085                 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
2086                 goto out_core;
2087         }
2088
2089         rc = proto_register(&smc_proto6, 1);
2090         if (rc) {
2091                 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
2092                 goto out_proto;
2093         }
2094
2095         rc = sock_register(&smc_sock_family_ops);
2096         if (rc) {
2097                 pr_err("%s: sock_register fails with %d\n", __func__, rc);
2098                 goto out_proto6;
2099         }
2100         INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
2101         INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
2102
2103         rc = smc_ib_register_client();
2104         if (rc) {
2105                 pr_err("%s: ib_register fails with %d\n", __func__, rc);
2106                 goto out_sock;
2107         }
2108
2109         static_branch_enable(&tcp_have_smc);
2110         return 0;
2111
2112 out_sock:
2113         sock_unregister(PF_SMC);
2114 out_proto6:
2115         proto_unregister(&smc_proto6);
2116 out_proto:
2117         proto_unregister(&smc_proto);
2118 out_core:
2119         smc_core_exit();
2120 out_pnet:
2121         smc_pnet_exit();
2122 out_pernet_subsys:
2123         unregister_pernet_subsys(&smc_net_ops);
2124
2125         return rc;
2126 }
2127
2128 static void __exit smc_exit(void)
2129 {
2130         static_branch_disable(&tcp_have_smc);
2131         sock_unregister(PF_SMC);
2132         smc_core_exit();
2133         smc_ib_unregister_client();
2134         proto_unregister(&smc_proto6);
2135         proto_unregister(&smc_proto);
2136         smc_pnet_exit();
2137         unregister_pernet_subsys(&smc_net_ops);
2138         rcu_barrier();
2139 }
2140
2141 module_init(smc_init);
2142 module_exit(smc_exit);
2143
2144 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
2145 MODULE_DESCRIPTION("smc socket address family");
2146 MODULE_LICENSE("GPL");
2147 MODULE_ALIAS_NETPROTO(PF_SMC);