net/smc/af_smc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
   6  *  applies to SOCK_STREAM sockets only
   7  *  offers an alternative communication option for TCP-protocol sockets
   8  *  applicable with RoCE-cards only
   9  *
  10  *  Initial restrictions:
  11  *    - support for alternate links postponed
  12  *
  13  *  Copyright IBM Corp. 2016, 2018
  14  *
  15  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
  16  *              based on prototype from Frank Blaschka
  17  */
  18
  19 #define KMSG_COMPONENT "smc"
  20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  21
  22 #include <linux/module.h>
  23 #include <linux/socket.h>
  24 #include <linux/workqueue.h>
  25 #include <linux/in.h>
  26 #include <linux/sched/signal.h>
  27 #include <linux/if_vlan.h>
  28 #include <linux/rcupdate_wait.h>
  29
  30 #include <net/sock.h>
  31 #include <net/tcp.h>
  32 #include <net/smc.h>
  33 #include <asm/ioctls.h>
  34
  35 #include <net/net_namespace.h>
  36 #include <net/netns/generic.h>
  37 #include "smc_netns.h"
  38
  39 #include "smc.h"
  40 #include "smc_clc.h"
  41 #include "smc_llc.h"
  42 #include "smc_cdc.h"
  43 #include "smc_core.h"
  44 #include "smc_ib.h"
  45 #include "smc_ism.h"
  46 #include "smc_pnet.h"
  47 #include "smc_tx.h"
  48 #include "smc_rx.h"
  49 #include "smc_close.h"
  50
  51 static DEFINE_MUTEX(smc_server_lgr_pending);    /* serialize link group
  52                                                  * creation on server
  53                                                  */
  54 static DEFINE_MUTEX(smc_client_lgr_pending);    /* serialize link group
  55                                                  * creation on client
  56                                                  */
  57
  58 static void smc_tcp_listen_work(struct work_struct *);
  59 static void smc_connect_work(struct work_struct *);
  60
  61 static void smc_set_keepalive(struct sock *sk, int val)
  62 {
  63         struct smc_sock *smc = smc_sk(sk);
  64
  65         smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
  66 }
  67
  68 static struct smc_hashinfo smc_v4_hashinfo = {
  69         .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
  70 };
  71
  72 static struct smc_hashinfo smc_v6_hashinfo = {
  73         .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
  74 };
  75
  76 int smc_hash_sk(struct sock *sk)
  77 {
  78         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  79         struct hlist_head *head;
  80
  81         head = &h->ht;
  82
  83         write_lock_bh(&h->lock);
  84         sk_add_node(sk, head);
  85         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  86         write_unlock_bh(&h->lock);
  87
  88         return 0;
  89 }
  90 EXPORT_SYMBOL_GPL(smc_hash_sk);
  91
  92 void smc_unhash_sk(struct sock *sk)
  93 {
  94         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  95
  96         write_lock_bh(&h->lock);
  97         if (sk_del_node_init(sk))
  98                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
  99         write_unlock_bh(&h->lock);
 100 }
 101 EXPORT_SYMBOL_GPL(smc_unhash_sk);
 102
 103 struct proto smc_proto = {
 104         .name           = "SMC",
 105         .owner          = THIS_MODULE,
 106         .keepalive      = smc_set_keepalive,
 107         .hash           = smc_hash_sk,
 108         .unhash         = smc_unhash_sk,
 109         .obj_size       = sizeof(struct smc_sock),
 110         .h.smc_hash     = &smc_v4_hashinfo,
 111         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 112 };
 113 EXPORT_SYMBOL_GPL(smc_proto);
 114
 115 struct proto smc_proto6 = {
 116         .name           = "SMC6",
 117         .owner          = THIS_MODULE,
 118         .keepalive      = smc_set_keepalive,
 119         .hash           = smc_hash_sk,
 120         .unhash         = smc_unhash_sk,
 121         .obj_size       = sizeof(struct smc_sock),
 122         .h.smc_hash     = &smc_v6_hashinfo,
 123         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 124 };
 125 EXPORT_SYMBOL_GPL(smc_proto6);
 126
 127 static void smc_restore_fallback_changes(struct smc_sock *smc)
 128 {
 129         smc->clcsock->file->private_data = smc->sk.sk_socket;
 130         smc->clcsock->file = NULL;
 131 }
 132
 133 static int __smc_release(struct smc_sock *smc)
 134 {
 135         struct sock *sk = &smc->sk;
 136         int rc = 0;
 137
 138         if (!smc->use_fallback) {
 139                 rc = smc_close_active(smc);
 140                 sock_set_flag(sk, SOCK_DEAD);
 141                 sk->sk_shutdown |= SHUTDOWN_MASK;
 142         } else {
 143                 if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
 144                         sock_put(sk); /* passive closing */
 145                 if (sk->sk_state == SMC_LISTEN) {
 146                         /* wake up clcsock accept */
 147                         rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR);
 148                 }
 149                 sk->sk_state = SMC_CLOSED;
 150                 sk->sk_state_change(sk);
 151                 smc_restore_fallback_changes(smc);
 152         }
 153
 154         sk->sk_prot->unhash(sk);
 155
 156         if (sk->sk_state == SMC_CLOSED) {
 157                 if (smc->clcsock) {
 158                         release_sock(sk);
 159                         smc_clcsock_release(smc);
 160                         lock_sock(sk);
 161                 }
 162                 if (!smc->use_fallback)
 163                         smc_conn_free(&smc->conn);
 164         }
 165
 166         return rc;
 167 }
 168
 169 static int smc_release(struct socket *sock)
 170 {
 171         struct sock *sk = sock->sk;
 172         struct smc_sock *smc;
 173         int rc = 0;
 174
 175         if (!sk)
 176                 goto out;
 177
 178         sock_hold(sk); /* sock_put below */
 179         smc = smc_sk(sk);
 180
 181         /* cleanup for a dangling non-blocking connect */
 182         if (smc->connect_nonblock && sk->sk_state == SMC_INIT)
 183                 tcp_abort(smc->clcsock->sk, ECONNABORTED);
 184         flush_work(&smc->connect_work);
 185
 186         if (sk->sk_state == SMC_LISTEN)
 187                 /* smc_close_non_accepted() is called and acquires
 188                  * sock lock for child sockets again
 189                  */
 190                 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 191         else
 192                 lock_sock(sk);
 193
 194         rc = __smc_release(smc);
 195
 196         /* detach socket */
 197         sock_orphan(sk);
 198         sock->sk = NULL;
 199         release_sock(sk);
 200
 201         sock_put(sk); /* sock_hold above */
 202         sock_put(sk); /* final sock_put */
 203 out:
 204         return rc;
 205 }
 206
 207 static void smc_destruct(struct sock *sk)
 208 {
 209         if (sk->sk_state != SMC_CLOSED)
 210                 return;
 211         if (!sock_flag(sk, SOCK_DEAD))
 212                 return;
 213
 214         sk_refcnt_debug_dec(sk);
 215 }
 216
 217 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 218                                    int protocol)
 219 {
 220         struct smc_sock *smc;
 221         struct proto *prot;
 222         struct sock *sk;
 223
 224         prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
 225         sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
 226         if (!sk)
 227                 return NULL;
 228
 229         sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
 230         sk->sk_state = SMC_INIT;
 231         sk->sk_destruct = smc_destruct;
 232         sk->sk_protocol = protocol;
 233         smc = smc_sk(sk);
 234         INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 235         INIT_WORK(&smc->connect_work, smc_connect_work);
 236         INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
 237         INIT_LIST_HEAD(&smc->accept_q);
 238         spin_lock_init(&smc->accept_q_lock);
 239         spin_lock_init(&smc->conn.send_lock);
 240         sk->sk_prot->hash(sk);
 241         sk_refcnt_debug_inc(sk);
 242         mutex_init(&smc->clcsock_release_lock);
 243
 244         return sk;
 245 }
 246
 247 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 248                     int addr_len)
 249 {
 250         struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
 251         struct sock *sk = sock->sk;
 252         struct smc_sock *smc;
 253         int rc;
 254
 255         smc = smc_sk(sk);
 256
 257         /* replicate tests from inet_bind(), to be safe wrt. future changes */
 258         rc = -EINVAL;
 259         if (addr_len < sizeof(struct sockaddr_in))
 260                 goto out;
 261
 262         rc = -EAFNOSUPPORT;
 263         if (addr->sin_family != AF_INET &&
 264             addr->sin_family != AF_INET6 &&
 265             addr->sin_family != AF_UNSPEC)
 266                 goto out;
 267         /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
 268         if (addr->sin_family == AF_UNSPEC &&
 269             addr->sin_addr.s_addr != htonl(INADDR_ANY))
 270                 goto out;
 271
 272         lock_sock(sk);
 273
 274         /* Check if socket is already active */
 275         rc = -EINVAL;
 276         if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
 277                 goto out_rel;
 278
 279         smc->clcsock->sk->sk_reuse = sk->sk_reuse;
 280         rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 281
 282 out_rel:
 283         release_sock(sk);
 284 out:
 285         return rc;
 286 }
 287
 288 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
 289                                    unsigned long mask)
 290 {
 291         /* options we don't get control via setsockopt for */
 292         nsk->sk_type = osk->sk_type;
 293         nsk->sk_sndbuf = osk->sk_sndbuf;
 294         nsk->sk_rcvbuf = osk->sk_rcvbuf;
 295         nsk->sk_sndtimeo = osk->sk_sndtimeo;
 296         nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
 297         nsk->sk_mark = osk->sk_mark;
 298         nsk->sk_priority = osk->sk_priority;
 299         nsk->sk_rcvlowat = osk->sk_rcvlowat;
 300         nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
 301         nsk->sk_err = osk->sk_err;
 302
 303         nsk->sk_flags &= ~mask;
 304         nsk->sk_flags |= osk->sk_flags & mask;
 305 }
 306
 307 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
 308                              (1UL << SOCK_KEEPOPEN) | \
 309                              (1UL << SOCK_LINGER) | \
 310                              (1UL << SOCK_BROADCAST) | \
 311                              (1UL << SOCK_TIMESTAMP) | \
 312                              (1UL << SOCK_DBG) | \
 313                              (1UL << SOCK_RCVTSTAMP) | \
 314                              (1UL << SOCK_RCVTSTAMPNS) | \
 315                              (1UL << SOCK_LOCALROUTE) | \
 316                              (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
 317                              (1UL << SOCK_RXQ_OVFL) | \
 318                              (1UL << SOCK_WIFI_STATUS) | \
 319                              (1UL << SOCK_NOFCS) | \
 320                              (1UL << SOCK_FILTER_LOCKED) | \
 321                              (1UL << SOCK_TSTAMP_NEW))
 322 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
 323  * clc socket (since smc is not called for these options from net/core)
 324  */
 325 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
 326 {
 327         smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
 328 }
 329
 330 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
 331                              (1UL << SOCK_KEEPOPEN) | \
 332                              (1UL << SOCK_LINGER) | \
 333                              (1UL << SOCK_DBG))
 334 /* copy only settings and flags relevant for smc from clc to smc socket */
 335 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
 336 {
 337         smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 338 }
 339
 340 /* register a new rmb, send confirm_rkey msg to register with peer */
 341 static int smc_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc,
 342                        bool conf_rkey)
 343 {
 344         if (!rmb_desc->wr_reg) {
 345                 /* register memory region for new rmb */
 346                 if (smc_wr_reg_send(link, rmb_desc->mr_rx[link->link_idx])) {
 347                         rmb_desc->regerr = 1;
 348                         return -EFAULT;
 349                 }
 350                 rmb_desc->wr_reg = 1;
 351         }
 352         if (!conf_rkey)
 353                 return 0;
 354         /* exchange confirm_rkey msg with peer */
 355         if (smc_llc_do_confirm_rkey(link, rmb_desc)) {
 356                 rmb_desc->regerr = 1;
 357                 return -EFAULT;
 358         }
 359         return 0;
 360 }
 361
 362 static int smc_clnt_conf_first_link(struct smc_sock *smc)
 363 {
 364         struct net *net = sock_net(smc->clcsock->sk);
 365         struct smc_link *link = smc->conn.lnk;
 366         int rest;
 367         int rc;
 368
 369         /* receive CONFIRM LINK request from server over RoCE fabric */
 370         rest = wait_for_completion_interruptible_timeout(
 371                 &link->llc_confirm,
 372                 SMC_LLC_WAIT_FIRST_TIME);
 373         if (rest <= 0) {
 374                 struct smc_clc_msg_decline dclc;
 375
 376                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 377                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 378                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
 379         }
 380
 381         if (link->llc_confirm_rc)
 382                 return SMC_CLC_DECL_RMBE_EC;
 383
 384         rc = smc_ib_modify_qp_rts(link);
 385         if (rc)
 386                 return SMC_CLC_DECL_ERR_RDYLNK;
 387
 388         smc_wr_remember_qp_attr(link);
 389
 390         if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
 391                 return SMC_CLC_DECL_ERR_REGRMB;
 392
 393         /* send CONFIRM LINK response over RoCE fabric */
 394         rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
 395         if (rc < 0)
 396                 return SMC_CLC_DECL_TIMEOUT_CL;
 397
 398         /* receive ADD LINK request from server over RoCE fabric */
 399         rest = wait_for_completion_interruptible_timeout(&link->llc_add,
 400                                                          SMC_LLC_WAIT_TIME);
 401         if (rest <= 0) {
 402                 struct smc_clc_msg_decline dclc;
 403
 404                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 405                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 406                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
 407         }
 408
 409         /* send add link reject message, only one link supported for now */
 410         rc = smc_llc_send_add_link(link,
 411                                    link->smcibdev->mac[link->ibport - 1],
 412                                    link->gid, SMC_LLC_RESP);
 413         if (rc < 0)
 414                 return SMC_CLC_DECL_TIMEOUT_AL;
 415
 416         smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
 417
 418         return 0;
 419 }
 420
 421 static void smcr_conn_save_peer_info(struct smc_sock *smc,
 422                                      struct smc_clc_msg_accept_confirm *clc)
 423 {
 424         int bufsize = smc_uncompress_bufsize(clc->rmbe_size);
 425
 426         smc->conn.peer_rmbe_idx = clc->rmbe_idx;
 427         smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
 428         smc->conn.peer_rmbe_size = bufsize;
 429         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 430         smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
 431 }
 432
 433 static void smcd_conn_save_peer_info(struct smc_sock *smc,
 434                                      struct smc_clc_msg_accept_confirm *clc)
 435 {
 436         int bufsize = smc_uncompress_bufsize(clc->dmbe_size);
 437
 438         smc->conn.peer_rmbe_idx = clc->dmbe_idx;
 439         smc->conn.peer_token = clc->token;
 440         /* msg header takes up space in the buffer */
 441         smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
 442         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 443         smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
 444 }
 445
 446 static void smc_conn_save_peer_info(struct smc_sock *smc,
 447                                     struct smc_clc_msg_accept_confirm *clc)
 448 {
 449         if (smc->conn.lgr->is_smcd)
 450                 smcd_conn_save_peer_info(smc, clc);
 451         else
 452                 smcr_conn_save_peer_info(smc, clc);
 453 }
 454
 455 static void smc_link_save_peer_info(struct smc_link *link,
 456                                     struct smc_clc_msg_accept_confirm *clc)
 457 {
 458         link->peer_qpn = ntoh24(clc->qpn);
 459         memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
 460         memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
 461         link->peer_psn = ntoh24(clc->psn);
 462         link->peer_mtu = clc->qp_mtu;
 463 }
 464
 465 static void smc_switch_to_fallback(struct smc_sock *smc)
 466 {
 467         smc->use_fallback = true;
 468         if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
 469                 smc->clcsock->file = smc->sk.sk_socket->file;
 470                 smc->clcsock->file->private_data = smc->clcsock;
 471                 smc->clcsock->wq.fasync_list =
 472                         smc->sk.sk_socket->wq.fasync_list;
 473         }
 474 }
 475
 476 /* fall back during connect */
 477 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
 478 {
 479         smc_switch_to_fallback(smc);
 480         smc->fallback_rsn = reason_code;
 481         smc_copy_sock_settings_to_clc(smc);
 482         smc->connect_nonblock = 0;
 483         if (smc->sk.sk_state == SMC_INIT)
 484                 smc->sk.sk_state = SMC_ACTIVE;
 485         return 0;
 486 }
 487
 488 /* decline and fall back during connect */
 489 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
 490 {
 491         int rc;
 492
 493         if (reason_code < 0) { /* error, fallback is not possible */
 494                 if (smc->sk.sk_state == SMC_INIT)
 495                         sock_put(&smc->sk); /* passive closing */
 496                 return reason_code;
 497         }
 498         if (reason_code != SMC_CLC_DECL_PEERDECL) {
 499                 rc = smc_clc_send_decline(smc, reason_code);
 500                 if (rc < 0) {
 501                         if (smc->sk.sk_state == SMC_INIT)
 502                                 sock_put(&smc->sk); /* passive closing */
 503                         return rc;
 504                 }
 505         }
 506         return smc_connect_fallback(smc, reason_code);
 507 }
 508
 509 /* abort connecting */
 510 static int smc_connect_abort(struct smc_sock *smc, int reason_code,
 511                              int local_contact)
 512 {
 513         bool is_smcd = smc->conn.lgr->is_smcd;
 514
 515         if (local_contact == SMC_FIRST_CONTACT)
 516                 smc_lgr_cleanup_early(&smc->conn);
 517         else
 518                 smc_conn_free(&smc->conn);
 519         if (is_smcd)
 520                 /* there is only one lgr role for SMC-D; use server lock */
 521                 mutex_unlock(&smc_server_lgr_pending);
 522         else
 523                 mutex_unlock(&smc_client_lgr_pending);
 524
 525         smc->connect_nonblock = 0;
 526         return reason_code;
 527 }
 528
 529 /* check if there is a rdma device available for this connection. */
 530 /* called for connect and listen */
 531 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
 532 {
 533         /* PNET table look up: search active ib_device and port
 534          * within same PNETID that also contains the ethernet device
 535          * used for the internal TCP socket
 536          */
 537         smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
 538         if (!ini->ib_dev)
 539                 return SMC_CLC_DECL_NOSMCRDEV;
 540         return 0;
 541 }
 542
 543 /* check if there is an ISM device available for this connection. */
 544 /* called for connect and listen */
 545 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
 546 {
 547         /* Find ISM device with same PNETID as connecting interface  */
 548         smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
 549         if (!ini->ism_dev)
 550                 return SMC_CLC_DECL_NOSMCDDEV;
 551         return 0;
 552 }
 553
 554 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
 555 static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
 556                                       struct smc_init_info *ini)
 557 {
 558         if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev, ini->vlan_id))
 559                 return SMC_CLC_DECL_ISMVLANERR;
 560         return 0;
 561 }
 562
 563 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
 564  * used, the VLAN ID will be registered again during the connection setup.
 565  */
 566 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd,
 567                                         struct smc_init_info *ini)
 568 {
 569         if (!is_smcd)
 570                 return 0;
 571         if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev, ini->vlan_id))
 572                 return SMC_CLC_DECL_CNFERR;
 573         return 0;
 574 }
 575
 576 /* CLC handshake during connect */
 577 static int smc_connect_clc(struct smc_sock *smc, int smc_type,
 578                            struct smc_clc_msg_accept_confirm *aclc,
 579                            struct smc_init_info *ini)
 580 {
 581         int rc = 0;
 582
 583         /* do inband token exchange */
 584         rc = smc_clc_send_proposal(smc, smc_type, ini);
 585         if (rc)
 586                 return rc;
 587         /* receive SMC Accept CLC message */
 588         return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT,
 589                                 CLC_WAIT_TIME);
 590 }
 591
 592 /* setup for RDMA connection of client */
 593 static int smc_connect_rdma(struct smc_sock *smc,
 594                             struct smc_clc_msg_accept_confirm *aclc,
 595                             struct smc_init_info *ini)
 596 {
 597         struct smc_link *link;
 598         int reason_code = 0;
 599
 600         ini->is_smcd = false;
 601         ini->ib_lcl = &aclc->lcl;
 602         ini->ib_clcqpn = ntoh24(aclc->qpn);
 603         ini->srv_first_contact = aclc->hdr.flag;
 604
 605         mutex_lock(&smc_client_lgr_pending);
 606         reason_code = smc_conn_create(smc, ini);
 607         if (reason_code) {
 608                 mutex_unlock(&smc_client_lgr_pending);
 609                 return reason_code;
 610         }
 611         link = smc->conn.lnk;
 612
 613         smc_conn_save_peer_info(smc, aclc);
 614
 615         /* create send buffer and rmb */
 616         if (smc_buf_create(smc, false))
 617                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
 618                                          ini->cln_first_contact);
 619
 620         if (ini->cln_first_contact == SMC_FIRST_CONTACT)
 621                 smc_link_save_peer_info(link, aclc);
 622
 623         if (smc_rmb_rtoken_handling(&smc->conn, aclc))
 624                 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RTOK,
 625                                          ini->cln_first_contact);
 626
 627         smc_close_init(smc);
 628         smc_rx_init(smc);
 629
 630         if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
 631                 if (smc_ib_ready_link(link))
 632                         return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK,
 633                                                  ini->cln_first_contact);
 634         } else {
 635                 if (smc_reg_rmb(link, smc->conn.rmb_desc, true))
 636                         return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB,
 637                                                  ini->cln_first_contact);
 638         }
 639         smc_rmb_sync_sg_for_device(&smc->conn);
 640
 641         reason_code = smc_clc_send_confirm(smc);
 642         if (reason_code)
 643                 return smc_connect_abort(smc, reason_code,
 644                                          ini->cln_first_contact);
 645
 646         smc_tx_init(smc);
 647
 648         if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
 649                 /* QP confirmation over RoCE fabric */
 650                 reason_code = smc_clnt_conf_first_link(smc);
 651                 if (reason_code)
 652                         return smc_connect_abort(smc, reason_code,
 653                                                  ini->cln_first_contact);
 654         }
 655         mutex_unlock(&smc_client_lgr_pending);
 656
 657         smc_copy_sock_settings_to_clc(smc);
 658         smc->connect_nonblock = 0;
 659         if (smc->sk.sk_state == SMC_INIT)
 660                 smc->sk.sk_state = SMC_ACTIVE;
 661
 662         return 0;
 663 }
 664
 665 /* setup for ISM connection of client */
 666 static int smc_connect_ism(struct smc_sock *smc,
 667                            struct smc_clc_msg_accept_confirm *aclc,
 668                            struct smc_init_info *ini)
 669 {
 670         int rc = 0;
 671
 672         ini->is_smcd = true;
 673         ini->ism_gid = aclc->gid;
 674         ini->srv_first_contact = aclc->hdr.flag;
 675
 676         /* there is only one lgr role for SMC-D; use server lock */
 677         mutex_lock(&smc_server_lgr_pending);
 678         rc = smc_conn_create(smc, ini);
 679         if (rc) {
 680                 mutex_unlock(&smc_server_lgr_pending);
 681                 return rc;
 682         }
 683
 684         /* Create send and receive buffers */
 685         if (smc_buf_create(smc, true))
 686                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
 687                                          ini->cln_first_contact);
 688
 689         smc_conn_save_peer_info(smc, aclc);
 690         smc_close_init(smc);
 691         smc_rx_init(smc);
 692         smc_tx_init(smc);
 693
 694         rc = smc_clc_send_confirm(smc);
 695         if (rc)
 696                 return smc_connect_abort(smc, rc, ini->cln_first_contact);
 697         mutex_unlock(&smc_server_lgr_pending);
 698
 699         smc_copy_sock_settings_to_clc(smc);
 700         smc->connect_nonblock = 0;
 701         if (smc->sk.sk_state == SMC_INIT)
 702                 smc->sk.sk_state = SMC_ACTIVE;
 703
 704         return 0;
 705 }
 706
 707 /* perform steps before actually connecting */
 708 static int __smc_connect(struct smc_sock *smc)
 709 {
 710         bool ism_supported = false, rdma_supported = false;
 711         struct smc_clc_msg_accept_confirm aclc;
 712         struct smc_init_info ini = {0};
 713         int smc_type;
 714         int rc = 0;
 715
 716         if (smc->use_fallback)
 717                 return smc_connect_fallback(smc, smc->fallback_rsn);
 718
 719         /* if peer has not signalled SMC-capability, fall back */
 720         if (!tcp_sk(smc->clcsock->sk)->syn_smc)
 721                 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
 722
 723         /* IPSec connections opt out of SMC-R optimizations */
 724         if (using_ipsec(smc))
 725                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC);
 726
 727         /* get vlan id from IP device */
 728         if (smc_vlan_by_tcpsk(smc->clcsock, &ini))
 729                 return smc_connect_decline_fallback(smc,
 730                                                     SMC_CLC_DECL_GETVLANERR);
 731
 732         /* check if there is an ism device available */
 733         if (!smc_find_ism_device(smc, &ini) &&
 734             !smc_connect_ism_vlan_setup(smc, &ini)) {
 735                 /* ISM is supported for this connection */
 736                 ism_supported = true;
 737                 smc_type = SMC_TYPE_D;
 738         }
 739
 740         /* check if there is a rdma device available */
 741         if (!smc_find_rdma_device(smc, &ini)) {
 742                 /* RDMA is supported for this connection */
 743                 rdma_supported = true;
 744                 if (ism_supported)
 745                         smc_type = SMC_TYPE_B; /* both */
 746                 else
 747                         smc_type = SMC_TYPE_R; /* only RDMA */
 748         }
 749
 750         /* if neither ISM nor RDMA are supported, fallback */
 751         if (!rdma_supported && !ism_supported)
 752                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV);
 753
 754         /* perform CLC handshake */
 755         rc = smc_connect_clc(smc, smc_type, &aclc, &ini);
 756         if (rc) {
 757                 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 758                 return smc_connect_decline_fallback(smc, rc);
 759         }
 760
 761         /* depending on previous steps, connect using rdma or ism */
 762         if (rdma_supported && aclc.hdr.path == SMC_TYPE_R)
 763                 rc = smc_connect_rdma(smc, &aclc, &ini);
 764         else if (ism_supported && aclc.hdr.path == SMC_TYPE_D)
 765                 rc = smc_connect_ism(smc, &aclc, &ini);
 766         else
 767                 rc = SMC_CLC_DECL_MODEUNSUPP;
 768         if (rc) {
 769                 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 770                 return smc_connect_decline_fallback(smc, rc);
 771         }
 772
 773         smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 774         return 0;
 775 }
 776
 777 static void smc_connect_work(struct work_struct *work)
 778 {
 779         struct smc_sock *smc = container_of(work, struct smc_sock,
 780                                             connect_work);
 781         long timeo = smc->sk.sk_sndtimeo;
 782         int rc = 0;
 783
 784         if (!timeo)
 785                 timeo = MAX_SCHEDULE_TIMEOUT;
 786         lock_sock(smc->clcsock->sk);
 787         if (smc->clcsock->sk->sk_err) {
 788                 smc->sk.sk_err = smc->clcsock->sk->sk_err;
 789         } else if ((1 << smc->clcsock->sk->sk_state) &
 790                                         (TCPF_SYN_SENT | TCP_SYN_RECV)) {
 791                 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
 792                 if ((rc == -EPIPE) &&
 793                     ((1 << smc->clcsock->sk->sk_state) &
 794                                         (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
 795                         rc = 0;
 796         }
 797         release_sock(smc->clcsock->sk);
 798         lock_sock(&smc->sk);
 799         if (rc != 0 || smc->sk.sk_err) {
 800                 smc->sk.sk_state = SMC_CLOSED;
 801                 if (rc == -EPIPE || rc == -EAGAIN)
 802                         smc->sk.sk_err = EPIPE;
 803                 else if (signal_pending(current))
 804                         smc->sk.sk_err = -sock_intr_errno(timeo);
 805                 sock_put(&smc->sk); /* passive closing */
 806                 goto out;
 807         }
 808
 809         rc = __smc_connect(smc);
 810         if (rc < 0)
 811                 smc->sk.sk_err = -rc;
 812
 813 out:
 814         if (!sock_flag(&smc->sk, SOCK_DEAD)) {
 815                 if (smc->sk.sk_err) {
 816                         smc->sk.sk_state_change(&smc->sk);
 817                 } else { /* allow polling before and after fallback decision */
 818                         smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
 819                         smc->sk.sk_write_space(&smc->sk);
 820                 }
 821         }
 822         release_sock(&smc->sk);
 823 }
 824
 825 static int smc_connect(struct socket *sock, struct sockaddr *addr,
 826                        int alen, int flags)
 827 {
 828         struct sock *sk = sock->sk;
 829         struct smc_sock *smc;
 830         int rc = -EINVAL;
 831
 832         smc = smc_sk(sk);
 833
 834         /* separate smc parameter checking to be safe */
 835         if (alen < sizeof(addr->sa_family))
 836                 goto out_err;
 837         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
 838                 goto out_err;
 839
 840         lock_sock(sk);
 841         switch (sk->sk_state) {
 842         default:
 843                 goto out;
 844         case SMC_ACTIVE:
 845                 rc = -EISCONN;
 846                 goto out;
 847         case SMC_INIT:
 848                 rc = 0;
 849                 break;
 850         }
 851
 852         smc_copy_sock_settings_to_clc(smc);
 853         tcp_sk(smc->clcsock->sk)->syn_smc = 1;
 854         if (smc->connect_nonblock) {
 855                 rc = -EALREADY;
 856                 goto out;
 857         }
 858         rc = kernel_connect(smc->clcsock, addr, alen, flags);
 859         if (rc && rc != -EINPROGRESS)
 860                 goto out;
 861
 862         sock_hold(&smc->sk); /* sock put in passive closing */
 863         if (smc->use_fallback)
 864                 goto out;
 865         if (flags & O_NONBLOCK) {
 866                 if (schedule_work(&smc->connect_work))
 867                         smc->connect_nonblock = 1;
 868                 rc = -EINPROGRESS;
 869         } else {
 870                 rc = __smc_connect(smc);
 871                 if (rc < 0)
 872                         goto out;
 873                 else
 874                         rc = 0; /* success cases including fallback */
 875         }
 876
 877 out:
 878         release_sock(sk);
 879 out_err:
 880         return rc;
 881 }
 882
 883 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
 884 {
 885         struct socket *new_clcsock = NULL;
 886         struct sock *lsk = &lsmc->sk;
 887         struct sock *new_sk;
 888         int rc = -EINVAL;
 889
 890         release_sock(lsk);
 891         new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
 892         if (!new_sk) {
 893                 rc = -ENOMEM;
 894                 lsk->sk_err = ENOMEM;
 895                 *new_smc = NULL;
 896                 lock_sock(lsk);
 897                 goto out;
 898         }
 899         *new_smc = smc_sk(new_sk);
 900
 901         mutex_lock(&lsmc->clcsock_release_lock);
 902         if (lsmc->clcsock)
 903                 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
 904         mutex_unlock(&lsmc->clcsock_release_lock);
 905         lock_sock(lsk);
 906         if  (rc < 0)
 907                 lsk->sk_err = -rc;
 908         if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
 909                 new_sk->sk_prot->unhash(new_sk);
 910                 if (new_clcsock)
 911                         sock_release(new_clcsock);
 912                 new_sk->sk_state = SMC_CLOSED;
 913                 sock_set_flag(new_sk, SOCK_DEAD);
 914                 sock_put(new_sk); /* final */
 915                 *new_smc = NULL;
 916                 goto out;
 917         }
 918
 919         (*new_smc)->clcsock = new_clcsock;
 920 out:
 921         return rc;
 922 }
 923
 924 /* add a just created sock to the accept queue of the listen sock as
 925  * candidate for a following socket accept call from user space
 926  */
 927 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
 928 {
 929         struct smc_sock *par = smc_sk(parent);
 930
 931         sock_hold(sk); /* sock_put in smc_accept_unlink () */
 932         spin_lock(&par->accept_q_lock);
 933         list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
 934         spin_unlock(&par->accept_q_lock);
 935         sk_acceptq_added(parent);
 936 }
 937
 938 /* remove a socket from the accept queue of its parental listening socket */
 939 static void smc_accept_unlink(struct sock *sk)
 940 {
 941         struct smc_sock *par = smc_sk(sk)->listen_smc;
 942
 943         spin_lock(&par->accept_q_lock);
 944         list_del_init(&smc_sk(sk)->accept_q);
 945         spin_unlock(&par->accept_q_lock);
 946         sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
 947         sock_put(sk); /* sock_hold in smc_accept_enqueue */
 948 }
 949
 950 /* remove a sock from the accept queue to bind it to a new socket created
 951  * for a socket accept call from user space
 952  */
 953 struct sock *smc_accept_dequeue(struct sock *parent,
 954                                 struct socket *new_sock)
 955 {
 956         struct smc_sock *isk, *n;
 957         struct sock *new_sk;
 958
 959         list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
 960                 new_sk = (struct sock *)isk;
 961
 962                 smc_accept_unlink(new_sk);
 963                 if (new_sk->sk_state == SMC_CLOSED) {
 964                         new_sk->sk_prot->unhash(new_sk);
 965                         if (isk->clcsock) {
 966                                 sock_release(isk->clcsock);
 967                                 isk->clcsock = NULL;
 968                         }
 969                         sock_put(new_sk); /* final */
 970                         continue;
 971                 }
 972                 if (new_sock) {
 973                         sock_graft(new_sk, new_sock);
 974                         if (isk->use_fallback) {
 975                                 smc_sk(new_sk)->clcsock->file = new_sock->file;
 976                                 isk->clcsock->file->private_data = isk->clcsock;
 977                         }
 978                 }
 979                 return new_sk;
 980         }
 981         return NULL;
 982 }
 983
 984 /* clean up for a created but never accepted sock */
 985 void smc_close_non_accepted(struct sock *sk)
 986 {
 987         struct smc_sock *smc = smc_sk(sk);
 988
 989         sock_hold(sk); /* sock_put below */
 990         lock_sock(sk);
 991         if (!sk->sk_lingertime)
 992                 /* wait for peer closing */
 993                 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
 994         __smc_release(smc);
 995         release_sock(sk);
 996         sock_put(sk); /* sock_hold above */
 997         sock_put(sk); /* final sock_put */
 998 }
 999
1000 static int smc_serv_conf_first_link(struct smc_sock *smc)
1001 {
1002         struct net *net = sock_net(smc->clcsock->sk);
1003         struct smc_link *link = smc->conn.lnk;
1004         int rest;
1005         int rc;
1006
1007         if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
1008                 return SMC_CLC_DECL_ERR_REGRMB;
1009
1010         /* send CONFIRM LINK request to client over the RoCE fabric */
1011         rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
1012         if (rc < 0)
1013                 return SMC_CLC_DECL_TIMEOUT_CL;
1014
1015         /* receive CONFIRM LINK response from client over the RoCE fabric */
1016         rest = wait_for_completion_interruptible_timeout(
1017                 &link->llc_confirm_resp,
1018                 SMC_LLC_WAIT_FIRST_TIME);
1019         if (rest <= 0) {
1020                 struct smc_clc_msg_decline dclc;
1021
1022                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1023                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1024                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1025         }
1026
1027         if (link->llc_confirm_resp_rc)
1028                 return SMC_CLC_DECL_RMBE_EC;
1029
1030         /* send ADD LINK request to client over the RoCE fabric */
1031         rc = smc_llc_send_add_link(link,
1032                                    link->smcibdev->mac[link->ibport - 1],
1033                                    link->gid, SMC_LLC_REQ);
1034         if (rc < 0)
1035                 return SMC_CLC_DECL_TIMEOUT_AL;
1036
1037         /* receive ADD LINK response from client over the RoCE fabric */
1038         rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
1039                                                          SMC_LLC_WAIT_TIME);
1040         if (rest <= 0) {
1041                 struct smc_clc_msg_decline dclc;
1042
1043                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1044                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1045                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
1046         }
1047
1048         smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
1049
1050         return 0;
1051 }
1052
1053 /* listen worker: finish */
1054 static void smc_listen_out(struct smc_sock *new_smc)
1055 {
1056         struct smc_sock *lsmc = new_smc->listen_smc;
1057         struct sock *newsmcsk = &new_smc->sk;
1058
1059         if (lsmc->sk.sk_state == SMC_LISTEN) {
1060                 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1061                 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1062                 release_sock(&lsmc->sk);
1063         } else { /* no longer listening */
1064                 smc_close_non_accepted(newsmcsk);
1065         }
1066
1067         /* Wake up accept */
1068         lsmc->sk.sk_data_ready(&lsmc->sk);
1069         sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1070 }
1071
1072 /* listen worker: finish in state connected */
1073 static void smc_listen_out_connected(struct smc_sock *new_smc)
1074 {
1075         struct sock *newsmcsk = &new_smc->sk;
1076
1077         sk_refcnt_debug_inc(newsmcsk);
1078         if (newsmcsk->sk_state == SMC_INIT)
1079                 newsmcsk->sk_state = SMC_ACTIVE;
1080
1081         smc_listen_out(new_smc);
1082 }
1083
1084 /* listen worker: finish in error state */
1085 static void smc_listen_out_err(struct smc_sock *new_smc)
1086 {
1087         struct sock *newsmcsk = &new_smc->sk;
1088
1089         if (newsmcsk->sk_state == SMC_INIT)
1090                 sock_put(&new_smc->sk); /* passive closing */
1091         newsmcsk->sk_state = SMC_CLOSED;
1092
1093         smc_listen_out(new_smc);
1094 }
1095
1096 /* listen worker: decline and fall back if possible */
1097 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1098                                int local_contact)
1099 {
1100         /* RDMA setup failed, switch back to TCP */
1101         if (local_contact == SMC_FIRST_CONTACT)
1102                 smc_lgr_cleanup_early(&new_smc->conn);
1103         else
1104                 smc_conn_free(&new_smc->conn);
1105         if (reason_code < 0) { /* error, no fallback possible */
1106                 smc_listen_out_err(new_smc);
1107                 return;
1108         }
1109         smc_switch_to_fallback(new_smc);
1110         new_smc->fallback_rsn = reason_code;
1111         if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1112                 if (smc_clc_send_decline(new_smc, reason_code) < 0) {
1113                         smc_listen_out_err(new_smc);
1114                         return;
1115                 }
1116         }
1117         smc_listen_out_connected(new_smc);
1118 }
1119
1120 /* listen worker: check prefixes */
1121 static int smc_listen_prfx_check(struct smc_sock *new_smc,
1122                                  struct smc_clc_msg_proposal *pclc)
1123 {
1124         struct smc_clc_msg_proposal_prefix *pclc_prfx;
1125         struct socket *newclcsock = new_smc->clcsock;
1126
1127         pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1128         if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1129                 return SMC_CLC_DECL_DIFFPREFIX;
1130
1131         return 0;
1132 }
1133
1134 /* listen worker: initialize connection and buffers */
1135 static int smc_listen_rdma_init(struct smc_sock *new_smc,
1136                                 struct smc_init_info *ini)
1137 {
1138         int rc;
1139
1140         /* allocate connection / link group */
1141         rc = smc_conn_create(new_smc, ini);
1142         if (rc)
1143                 return rc;
1144
1145         /* create send buffer and rmb */
1146         if (smc_buf_create(new_smc, false))
1147                 return SMC_CLC_DECL_MEM;
1148
1149         return 0;
1150 }
1151
1152 /* listen worker: initialize connection and buffers for SMC-D */
1153 static int smc_listen_ism_init(struct smc_sock *new_smc,
1154                                struct smc_clc_msg_proposal *pclc,
1155                                struct smc_init_info *ini)
1156 {
1157         struct smc_clc_msg_smcd *pclc_smcd;
1158         int rc;
1159
1160         pclc_smcd = smc_get_clc_msg_smcd(pclc);
1161         ini->ism_gid = pclc_smcd->gid;
1162         rc = smc_conn_create(new_smc, ini);
1163         if (rc)
1164                 return rc;
1165
1166         /* Check if peer can be reached via ISM device */
1167         if (smc_ism_cantalk(new_smc->conn.lgr->peer_gid,
1168                             new_smc->conn.lgr->vlan_id,
1169                             new_smc->conn.lgr->smcd)) {
1170                 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
1171                         smc_lgr_cleanup_early(&new_smc->conn);
1172                 else
1173                         smc_conn_free(&new_smc->conn);
1174                 return SMC_CLC_DECL_SMCDNOTALK;
1175         }
1176
1177         /* Create send and receive buffers */
1178         if (smc_buf_create(new_smc, true)) {
1179                 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
1180                         smc_lgr_cleanup_early(&new_smc->conn);
1181                 else
1182                         smc_conn_free(&new_smc->conn);
1183                 return SMC_CLC_DECL_MEM;
1184         }
1185
1186         return 0;
1187 }
1188
1189 /* listen worker: register buffers */
1190 static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
1191 {
1192         struct smc_link *link = new_smc->conn.lnk;
1193
1194         if (local_contact != SMC_FIRST_CONTACT) {
1195                 if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
1196                         return SMC_CLC_DECL_ERR_REGRMB;
1197         }
1198         smc_rmb_sync_sg_for_device(&new_smc->conn);
1199
1200         return 0;
1201 }
1202
1203 /* listen worker: finish RDMA setup */
1204 static int smc_listen_rdma_finish(struct smc_sock *new_smc,
1205                                   struct smc_clc_msg_accept_confirm *cclc,
1206                                   int local_contact)
1207 {
1208         struct smc_link *link = new_smc->conn.lnk;
1209         int reason_code = 0;
1210
1211         if (local_contact == SMC_FIRST_CONTACT)
1212                 smc_link_save_peer_info(link, cclc);
1213
1214         if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {
1215                 reason_code = SMC_CLC_DECL_ERR_RTOK;
1216                 goto decline;
1217         }
1218
1219         if (local_contact == SMC_FIRST_CONTACT) {
1220                 if (smc_ib_ready_link(link)) {
1221                         reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1222                         goto decline;
1223                 }
1224                 /* QP confirmation over RoCE fabric */
1225                 reason_code = smc_serv_conf_first_link(new_smc);
1226                 if (reason_code)
1227                         goto decline;
1228         }
1229         return 0;
1230
1231 decline:
1232         smc_listen_decline(new_smc, reason_code, local_contact);
1233         return reason_code;
1234 }
1235
1236 /* setup for RDMA connection of server */
1237 static void smc_listen_work(struct work_struct *work)
1238 {
1239         struct smc_sock *new_smc = container_of(work, struct smc_sock,
1240                                                 smc_listen_work);
1241         struct socket *newclcsock = new_smc->clcsock;
1242         struct smc_clc_msg_accept_confirm cclc;
1243         struct smc_clc_msg_proposal *pclc;
1244         struct smc_init_info ini = {0};
1245         bool ism_supported = false;
1246         u8 buf[SMC_CLC_MAX_LEN];
1247         int rc = 0;
1248
1249         if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
1250                 return smc_listen_out_err(new_smc);
1251
1252         if (new_smc->use_fallback) {
1253                 smc_listen_out_connected(new_smc);
1254                 return;
1255         }
1256
1257         /* check if peer is smc capable */
1258         if (!tcp_sk(newclcsock->sk)->syn_smc) {
1259                 smc_switch_to_fallback(new_smc);
1260                 new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;
1261                 smc_listen_out_connected(new_smc);
1262                 return;
1263         }
1264
1265         /* do inband token exchange -
1266          * wait for and receive SMC Proposal CLC message
1267          */
1268         pclc = (struct smc_clc_msg_proposal *)&buf;
1269         rc = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN,
1270                               SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
1271         if (rc)
1272                 goto out_decl;
1273
1274         /* IPSec connections opt out of SMC-R optimizations */
1275         if (using_ipsec(new_smc)) {
1276                 rc = SMC_CLC_DECL_IPSEC;
1277                 goto out_decl;
1278         }
1279
1280         /* check for matching IP prefix and subnet length */
1281         rc = smc_listen_prfx_check(new_smc, pclc);
1282         if (rc)
1283                 goto out_decl;
1284
1285         /* get vlan id from IP device */
1286         if (smc_vlan_by_tcpsk(new_smc->clcsock, &ini)) {
1287                 rc = SMC_CLC_DECL_GETVLANERR;
1288                 goto out_decl;
1289         }
1290
1291         mutex_lock(&smc_server_lgr_pending);
1292         smc_close_init(new_smc);
1293         smc_rx_init(new_smc);
1294         smc_tx_init(new_smc);
1295
1296         /* check if ISM is available */
1297         if (pclc->hdr.path == SMC_TYPE_D || pclc->hdr.path == SMC_TYPE_B) {
1298                 ini.is_smcd = true; /* prepare ISM check */
1299                 rc = smc_find_ism_device(new_smc, &ini);
1300                 if (!rc)
1301                         rc = smc_listen_ism_init(new_smc, pclc, &ini);
1302                 if (!rc)
1303                         ism_supported = true;
1304                 else if (pclc->hdr.path == SMC_TYPE_D)
1305                         goto out_unlock; /* skip RDMA and decline */
1306         }
1307
1308         /* check if RDMA is available */
1309         if (!ism_supported) { /* SMC_TYPE_R or SMC_TYPE_B */
1310                 /* prepare RDMA check */
1311                 ini.is_smcd = false;
1312                 ini.ism_dev = NULL;
1313                 ini.ib_lcl = &pclc->lcl;
1314                 rc = smc_find_rdma_device(new_smc, &ini);
1315                 if (rc) {
1316                         /* no RDMA device found */
1317                         if (pclc->hdr.path == SMC_TYPE_B)
1318                                 /* neither ISM nor RDMA device found */
1319                                 rc = SMC_CLC_DECL_NOSMCDEV;
1320                         goto out_unlock;
1321                 }
1322                 rc = smc_listen_rdma_init(new_smc, &ini);
1323                 if (rc)
1324                         goto out_unlock;
1325                 rc = smc_listen_rdma_reg(new_smc, ini.cln_first_contact);
1326                 if (rc)
1327                         goto out_unlock;
1328         }
1329
1330         /* send SMC Accept CLC message */
1331         rc = smc_clc_send_accept(new_smc, ini.cln_first_contact);
1332         if (rc)
1333                 goto out_unlock;
1334
1335         /* SMC-D does not need this lock any more */
1336         if (ism_supported)
1337                 mutex_unlock(&smc_server_lgr_pending);
1338
1339         /* receive SMC Confirm CLC message */
1340         rc = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
1341                               SMC_CLC_CONFIRM, CLC_WAIT_TIME);
1342         if (rc) {
1343                 if (!ism_supported)
1344                         goto out_unlock;
1345                 goto out_decl;
1346         }
1347
1348         /* finish worker */
1349         if (!ism_supported) {
1350                 rc = smc_listen_rdma_finish(new_smc, &cclc,
1351                                             ini.cln_first_contact);
1352                 mutex_unlock(&smc_server_lgr_pending);
1353                 if (rc)
1354                         return;
1355         }
1356         smc_conn_save_peer_info(new_smc, &cclc);
1357         smc_listen_out_connected(new_smc);
1358         return;
1359
1360 out_unlock:
1361         mutex_unlock(&smc_server_lgr_pending);
1362 out_decl:
1363         smc_listen_decline(new_smc, rc, ini.cln_first_contact);
1364 }
1365
1366 static void smc_tcp_listen_work(struct work_struct *work)
1367 {
1368         struct smc_sock *lsmc = container_of(work, struct smc_sock,
1369                                              tcp_listen_work);
1370         struct sock *lsk = &lsmc->sk;
1371         struct smc_sock *new_smc;
1372         int rc = 0;
1373
1374         lock_sock(lsk);
1375         while (lsk->sk_state == SMC_LISTEN) {
1376                 rc = smc_clcsock_accept(lsmc, &new_smc);
1377                 if (rc)
1378                         goto out;
1379                 if (!new_smc)
1380                         continue;
1381
1382                 new_smc->listen_smc = lsmc;
1383                 new_smc->use_fallback = lsmc->use_fallback;
1384                 new_smc->fallback_rsn = lsmc->fallback_rsn;
1385                 sock_hold(lsk); /* sock_put in smc_listen_work */
1386                 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1387                 smc_copy_sock_settings_to_smc(new_smc);
1388                 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
1389                 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
1390                 sock_hold(&new_smc->sk); /* sock_put in passive closing */
1391                 if (!schedule_work(&new_smc->smc_listen_work))
1392                         sock_put(&new_smc->sk);
1393         }
1394
1395 out:
1396         release_sock(lsk);
1397         sock_put(&lsmc->sk); /* sock_hold in smc_listen */
1398 }
1399
1400 static int smc_listen(struct socket *sock, int backlog)
1401 {
1402         struct sock *sk = sock->sk;
1403         struct smc_sock *smc;
1404         int rc;
1405
1406         smc = smc_sk(sk);
1407         lock_sock(sk);
1408
1409         rc = -EINVAL;
1410         if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
1411             smc->connect_nonblock)
1412                 goto out;
1413
1414         rc = 0;
1415         if (sk->sk_state == SMC_LISTEN) {
1416                 sk->sk_max_ack_backlog = backlog;
1417                 goto out;
1418         }
1419         /* some socket options are handled in core, so we could not apply
1420          * them to the clc socket -- copy smc socket options to clc socket
1421          */
1422         smc_copy_sock_settings_to_clc(smc);
1423         if (!smc->use_fallback)
1424                 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1425
1426         rc = kernel_listen(smc->clcsock, backlog);
1427         if (rc)
1428                 goto out;
1429         sk->sk_max_ack_backlog = backlog;
1430         sk->sk_ack_backlog = 0;
1431         sk->sk_state = SMC_LISTEN;
1432         sock_hold(sk); /* sock_hold in tcp_listen_worker */
1433         if (!schedule_work(&smc->tcp_listen_work))
1434                 sock_put(sk);
1435
1436 out:
1437         release_sock(sk);
1438         return rc;
1439 }
1440
1441 static int smc_accept(struct socket *sock, struct socket *new_sock,
1442                       int flags, bool kern)
1443 {
1444         struct sock *sk = sock->sk, *nsk;
1445         DECLARE_WAITQUEUE(wait, current);
1446         struct smc_sock *lsmc;
1447         long timeo;
1448         int rc = 0;
1449
1450         lsmc = smc_sk(sk);
1451         sock_hold(sk); /* sock_put below */
1452         lock_sock(sk);
1453
1454         if (lsmc->sk.sk_state != SMC_LISTEN) {
1455                 rc = -EINVAL;
1456                 release_sock(sk);
1457                 goto out;
1458         }
1459
1460         /* Wait for an incoming connection */
1461         timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1462         add_wait_queue_exclusive(sk_sleep(sk), &wait);
1463         while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1464                 set_current_state(TASK_INTERRUPTIBLE);
1465                 if (!timeo) {
1466                         rc = -EAGAIN;
1467                         break;
1468                 }
1469                 release_sock(sk);
1470                 timeo = schedule_timeout(timeo);
1471                 /* wakeup by sk_data_ready in smc_listen_work() */
1472                 sched_annotate_sleep();
1473                 lock_sock(sk);
1474                 if (signal_pending(current)) {
1475                         rc = sock_intr_errno(timeo);
1476                         break;
1477                 }
1478         }
1479         set_current_state(TASK_RUNNING);
1480         remove_wait_queue(sk_sleep(sk), &wait);
1481
1482         if (!rc)
1483                 rc = sock_error(nsk);
1484         release_sock(sk);
1485         if (rc)
1486                 goto out;
1487
1488         if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1489                 /* wait till data arrives on the socket */
1490                 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1491                                                                 MSEC_PER_SEC);
1492                 if (smc_sk(nsk)->use_fallback) {
1493                         struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1494
1495                         lock_sock(clcsk);
1496                         if (skb_queue_empty(&clcsk->sk_receive_queue))
1497                                 sk_wait_data(clcsk, &timeo, NULL);
1498                         release_sock(clcsk);
1499                 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1500                         lock_sock(nsk);
1501                         smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1502                         release_sock(nsk);
1503                 }
1504         }
1505
1506 out:
1507         sock_put(sk); /* sock_hold above */
1508         return rc;
1509 }
1510
1511 static int smc_getname(struct socket *sock, struct sockaddr *addr,
1512                        int peer)
1513 {
1514         struct smc_sock *smc;
1515
1516         if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
1517             (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
1518                 return -ENOTCONN;
1519
1520         smc = smc_sk(sock->sk);
1521
1522         return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
1523 }
1524
1525 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
1526 {
1527         struct sock *sk = sock->sk;
1528         struct smc_sock *smc;
1529         int rc = -EPIPE;
1530
1531         smc = smc_sk(sk);
1532         lock_sock(sk);
1533         if ((sk->sk_state != SMC_ACTIVE) &&
1534             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1535             (sk->sk_state != SMC_INIT))
1536                 goto out;
1537
1538         if (msg->msg_flags & MSG_FASTOPEN) {
1539                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
1540                         smc_switch_to_fallback(smc);
1541                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1542                 } else {
1543                         rc = -EINVAL;
1544                         goto out;
1545                 }
1546         }
1547
1548         if (smc->use_fallback)
1549                 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
1550         else
1551                 rc = smc_tx_sendmsg(smc, msg, len);
1552 out:
1553         release_sock(sk);
1554         return rc;
1555 }
1556
1557 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
1558                        int flags)
1559 {
1560         struct sock *sk = sock->sk;
1561         struct smc_sock *smc;
1562         int rc = -ENOTCONN;
1563
1564         smc = smc_sk(sk);
1565         lock_sock(sk);
1566         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
1567                 /* socket was connected before, no more data to read */
1568                 rc = 0;
1569                 goto out;
1570         }
1571         if ((sk->sk_state == SMC_INIT) ||
1572             (sk->sk_state == SMC_LISTEN) ||
1573             (sk->sk_state == SMC_CLOSED))
1574                 goto out;
1575
1576         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1577                 rc = 0;
1578                 goto out;
1579         }
1580
1581         if (smc->use_fallback) {
1582                 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
1583         } else {
1584                 msg->msg_namelen = 0;
1585                 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
1586         }
1587
1588 out:
1589         release_sock(sk);
1590         return rc;
1591 }
1592
1593 static __poll_t smc_accept_poll(struct sock *parent)
1594 {
1595         struct smc_sock *isk = smc_sk(parent);
1596         __poll_t mask = 0;
1597
1598         spin_lock(&isk->accept_q_lock);
1599         if (!list_empty(&isk->accept_q))
1600                 mask = EPOLLIN | EPOLLRDNORM;
1601         spin_unlock(&isk->accept_q_lock);
1602
1603         return mask;
1604 }
1605
1606 static __poll_t smc_poll(struct file *file, struct socket *sock,
1607                              poll_table *wait)
1608 {
1609         struct sock *sk = sock->sk;
1610         struct smc_sock *smc;
1611         __poll_t mask = 0;
1612
1613         if (!sk)
1614                 return EPOLLNVAL;
1615
1616         smc = smc_sk(sock->sk);
1617         if (smc->use_fallback) {
1618                 /* delegate to CLC child sock */
1619                 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
1620                 sk->sk_err = smc->clcsock->sk->sk_err;
1621         } else {
1622                 if (sk->sk_state != SMC_CLOSED)
1623                         sock_poll_wait(file, sock, wait);
1624                 if (sk->sk_err)
1625                         mask |= EPOLLERR;
1626                 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
1627                     (sk->sk_state == SMC_CLOSED))
1628                         mask |= EPOLLHUP;
1629                 if (sk->sk_state == SMC_LISTEN) {
1630                         /* woken up by sk_data_ready in smc_listen_work() */
1631                         mask |= smc_accept_poll(sk);
1632                 } else if (smc->use_fallback) { /* as result of connect_work()*/
1633                         mask |= smc->clcsock->ops->poll(file, smc->clcsock,
1634                                                            wait);
1635                         sk->sk_err = smc->clcsock->sk->sk_err;
1636                 } else {
1637                         if ((sk->sk_state != SMC_INIT &&
1638                              atomic_read(&smc->conn.sndbuf_space)) ||
1639                             sk->sk_shutdown & SEND_SHUTDOWN) {
1640                                 mask |= EPOLLOUT | EPOLLWRNORM;
1641                         } else {
1642                                 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1643                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1644                         }
1645                         if (atomic_read(&smc->conn.bytes_to_rcv))
1646                                 mask |= EPOLLIN | EPOLLRDNORM;
1647                         if (sk->sk_shutdown & RCV_SHUTDOWN)
1648                                 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
1649                         if (sk->sk_state == SMC_APPCLOSEWAIT1)
1650                                 mask |= EPOLLIN;
1651                         if (smc->conn.urg_state == SMC_URG_VALID)
1652                                 mask |= EPOLLPRI;
1653                 }
1654         }
1655
1656         return mask;
1657 }
1658
1659 static int smc_shutdown(struct socket *sock, int how)
1660 {
1661         struct sock *sk = sock->sk;
1662         struct smc_sock *smc;
1663         int rc = -EINVAL;
1664         int rc1 = 0;
1665
1666         smc = smc_sk(sk);
1667
1668         if ((how < SHUT_RD) || (how > SHUT_RDWR))
1669                 return rc;
1670
1671         lock_sock(sk);
1672
1673         rc = -ENOTCONN;
1674         if ((sk->sk_state != SMC_ACTIVE) &&
1675             (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
1676             (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
1677             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1678             (sk->sk_state != SMC_APPCLOSEWAIT2) &&
1679             (sk->sk_state != SMC_APPFINCLOSEWAIT))
1680                 goto out;
1681         if (smc->use_fallback) {
1682                 rc = kernel_sock_shutdown(smc->clcsock, how);
1683                 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
1684                 if (sk->sk_shutdown == SHUTDOWN_MASK)
1685                         sk->sk_state = SMC_CLOSED;
1686                 goto out;
1687         }
1688         switch (how) {
1689         case SHUT_RDWR:         /* shutdown in both directions */
1690                 rc = smc_close_active(smc);
1691                 break;
1692         case SHUT_WR:
1693                 rc = smc_close_shutdown_write(smc);
1694                 break;
1695         case SHUT_RD:
1696                 rc = 0;
1697                 /* nothing more to do because peer is not involved */
1698                 break;
1699         }
1700         if (smc->clcsock)
1701                 rc1 = kernel_sock_shutdown(smc->clcsock, how);
1702         /* map sock_shutdown_cmd constants to sk_shutdown value range */
1703         sk->sk_shutdown |= how + 1;
1704
1705 out:
1706         release_sock(sk);
1707         return rc ? rc : rc1;
1708 }
1709
1710 static int smc_setsockopt(struct socket *sock, int level, int optname,
1711                           char __user *optval, unsigned int optlen)
1712 {
1713         struct sock *sk = sock->sk;
1714         struct smc_sock *smc;
1715         int val, rc;
1716
1717         smc = smc_sk(sk);
1718
1719         /* generic setsockopts reaching us here always apply to the
1720          * CLC socket
1721          */
1722         rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
1723                                            optval, optlen);
1724         if (smc->clcsock->sk->sk_err) {
1725                 sk->sk_err = smc->clcsock->sk->sk_err;
1726                 sk->sk_error_report(sk);
1727         }
1728
1729         if (optlen < sizeof(int))
1730                 return -EINVAL;
1731         if (get_user(val, (int __user *)optval))
1732                 return -EFAULT;
1733
1734         lock_sock(sk);
1735         if (rc || smc->use_fallback)
1736                 goto out;
1737         switch (optname) {
1738         case TCP_ULP:
1739         case TCP_FASTOPEN:
1740         case TCP_FASTOPEN_CONNECT:
1741         case TCP_FASTOPEN_KEY:
1742         case TCP_FASTOPEN_NO_COOKIE:
1743                 /* option not supported by SMC */
1744                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
1745                         smc_switch_to_fallback(smc);
1746                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1747                 } else {
1748                         rc = -EINVAL;
1749                 }
1750                 break;
1751         case TCP_NODELAY:
1752                 if (sk->sk_state != SMC_INIT &&
1753                     sk->sk_state != SMC_LISTEN &&
1754                     sk->sk_state != SMC_CLOSED) {
1755                         if (val)
1756                                 mod_delayed_work(system_wq, &smc->conn.tx_work,
1757                                                  0);
1758                 }
1759                 break;
1760         case TCP_CORK:
1761                 if (sk->sk_state != SMC_INIT &&
1762                     sk->sk_state != SMC_LISTEN &&
1763                     sk->sk_state != SMC_CLOSED) {
1764                         if (!val)
1765                                 mod_delayed_work(system_wq, &smc->conn.tx_work,
1766                                                  0);
1767                 }
1768                 break;
1769         case TCP_DEFER_ACCEPT:
1770                 smc->sockopt_defer_accept = val;
1771                 break;
1772         default:
1773                 break;
1774         }
1775 out:
1776         release_sock(sk);
1777
1778         return rc;
1779 }
1780
1781 static int smc_getsockopt(struct socket *sock, int level, int optname,
1782                           char __user *optval, int __user *optlen)
1783 {
1784         struct smc_sock *smc;
1785
1786         smc = smc_sk(sock->sk);
1787         /* socket options apply to the CLC socket */
1788         return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
1789                                              optval, optlen);
1790 }
1791
1792 static int smc_ioctl(struct socket *sock, unsigned int cmd,
1793                      unsigned long arg)
1794 {
1795         union smc_host_cursor cons, urg;
1796         struct smc_connection *conn;
1797         struct smc_sock *smc;
1798         int answ;
1799
1800         smc = smc_sk(sock->sk);
1801         conn = &smc->conn;
1802         lock_sock(&smc->sk);
1803         if (smc->use_fallback) {
1804                 if (!smc->clcsock) {
1805                         release_sock(&smc->sk);
1806                         return -EBADF;
1807                 }
1808                 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
1809                 release_sock(&smc->sk);
1810                 return answ;
1811         }
1812         switch (cmd) {
1813         case SIOCINQ: /* same as FIONREAD */
1814                 if (smc->sk.sk_state == SMC_LISTEN) {
1815                         release_sock(&smc->sk);
1816                         return -EINVAL;
1817                 }
1818                 if (smc->sk.sk_state == SMC_INIT ||
1819                     smc->sk.sk_state == SMC_CLOSED)
1820                         answ = 0;
1821                 else
1822                         answ = atomic_read(&smc->conn.bytes_to_rcv);
1823                 break;
1824         case SIOCOUTQ:
1825                 /* output queue size (not send + not acked) */
1826                 if (smc->sk.sk_state == SMC_LISTEN) {
1827                         release_sock(&smc->sk);
1828                         return -EINVAL;
1829                 }
1830                 if (smc->sk.sk_state == SMC_INIT ||
1831                     smc->sk.sk_state == SMC_CLOSED)
1832                         answ = 0;
1833                 else
1834                         answ = smc->conn.sndbuf_desc->len -
1835                                         atomic_read(&smc->conn.sndbuf_space);
1836                 break;
1837         case SIOCOUTQNSD:
1838                 /* output queue size (not send only) */
1839                 if (smc->sk.sk_state == SMC_LISTEN) {
1840                         release_sock(&smc->sk);
1841                         return -EINVAL;
1842                 }
1843                 if (smc->sk.sk_state == SMC_INIT ||
1844                     smc->sk.sk_state == SMC_CLOSED)
1845                         answ = 0;
1846                 else
1847                         answ = smc_tx_prepared_sends(&smc->conn);
1848                 break;
1849         case SIOCATMARK:
1850                 if (smc->sk.sk_state == SMC_LISTEN) {
1851                         release_sock(&smc->sk);
1852                         return -EINVAL;
1853                 }
1854                 if (smc->sk.sk_state == SMC_INIT ||
1855                     smc->sk.sk_state == SMC_CLOSED) {
1856                         answ = 0;
1857                 } else {
1858                         smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
1859                         smc_curs_copy(&urg, &conn->urg_curs, conn);
1860                         answ = smc_curs_diff(conn->rmb_desc->len,
1861                                              &cons, &urg) == 1;
1862                 }
1863                 break;
1864         default:
1865                 release_sock(&smc->sk);
1866                 return -ENOIOCTLCMD;
1867         }
1868         release_sock(&smc->sk);
1869
1870         return put_user(answ, (int __user *)arg);
1871 }
1872
1873 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
1874                             int offset, size_t size, int flags)
1875 {
1876         struct sock *sk = sock->sk;
1877         struct smc_sock *smc;
1878         int rc = -EPIPE;
1879
1880         smc = smc_sk(sk);
1881         lock_sock(sk);
1882         if (sk->sk_state != SMC_ACTIVE) {
1883                 release_sock(sk);
1884                 goto out;
1885         }
1886         release_sock(sk);
1887         if (smc->use_fallback)
1888                 rc = kernel_sendpage(smc->clcsock, page, offset,
1889                                      size, flags);
1890         else
1891                 rc = sock_no_sendpage(sock, page, offset, size, flags);
1892
1893 out:
1894         return rc;
1895 }
1896
1897 /* Map the affected portions of the rmbe into an spd, note the number of bytes
1898  * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
1899  * updates till whenever a respective page has been fully processed.
1900  * Note that subsequent recv() calls have to wait till all splice() processing
1901  * completed.
1902  */
1903 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
1904                                struct pipe_inode_info *pipe, size_t len,
1905                                unsigned int flags)
1906 {
1907         struct sock *sk = sock->sk;
1908         struct smc_sock *smc;
1909         int rc = -ENOTCONN;
1910
1911         smc = smc_sk(sk);
1912         lock_sock(sk);
1913         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
1914                 /* socket was connected before, no more data to read */
1915                 rc = 0;
1916                 goto out;
1917         }
1918         if (sk->sk_state == SMC_INIT ||
1919             sk->sk_state == SMC_LISTEN ||
1920             sk->sk_state == SMC_CLOSED)
1921                 goto out;
1922
1923         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1924                 rc = 0;
1925                 goto out;
1926         }
1927
1928         if (smc->use_fallback) {
1929                 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
1930                                                     pipe, len, flags);
1931         } else {
1932                 if (*ppos) {
1933                         rc = -ESPIPE;
1934                         goto out;
1935                 }
1936                 if (flags & SPLICE_F_NONBLOCK)
1937                         flags = MSG_DONTWAIT;
1938                 else
1939                         flags = 0;
1940                 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
1941         }
1942 out:
1943         release_sock(sk);
1944
1945         return rc;
1946 }
1947
1948 /* must look like tcp */
1949 static const struct proto_ops smc_sock_ops = {
1950         .family         = PF_SMC,
1951         .owner          = THIS_MODULE,
1952         .release        = smc_release,
1953         .bind           = smc_bind,
1954         .connect        = smc_connect,
1955         .socketpair     = sock_no_socketpair,
1956         .accept         = smc_accept,
1957         .getname        = smc_getname,
1958         .poll           = smc_poll,
1959         .ioctl          = smc_ioctl,
1960         .listen         = smc_listen,
1961         .shutdown       = smc_shutdown,
1962         .setsockopt     = smc_setsockopt,
1963         .getsockopt     = smc_getsockopt,
1964         .sendmsg        = smc_sendmsg,
1965         .recvmsg        = smc_recvmsg,
1966         .mmap           = sock_no_mmap,
1967         .sendpage       = smc_sendpage,
1968         .splice_read    = smc_splice_read,
1969 };
1970
1971 static int smc_create(struct net *net, struct socket *sock, int protocol,
1972                       int kern)
1973 {
1974         int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
1975         struct smc_sock *smc;
1976         struct sock *sk;
1977         int rc;
1978
1979         rc = -ESOCKTNOSUPPORT;
1980         if (sock->type != SOCK_STREAM)
1981                 goto out;
1982
1983         rc = -EPROTONOSUPPORT;
1984         if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
1985                 goto out;
1986
1987         rc = -ENOBUFS;
1988         sock->ops = &smc_sock_ops;
1989         sk = smc_sock_alloc(net, sock, protocol);
1990         if (!sk)
1991                 goto out;
1992
1993         /* create internal TCP socket for CLC handshake and fallback */
1994         smc = smc_sk(sk);
1995         smc->use_fallback = false; /* assume rdma capability first */
1996         smc->fallback_rsn = 0;
1997         rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
1998                               &smc->clcsock);
1999         if (rc) {
2000                 sk_common_release(sk);
2001                 goto out;
2002         }
2003         smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
2004         smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
2005
2006 out:
2007         return rc;
2008 }
2009
2010 static const struct net_proto_family smc_sock_family_ops = {
2011         .family = PF_SMC,
2012         .owner  = THIS_MODULE,
2013         .create = smc_create,
2014 };
2015
2016 unsigned int smc_net_id;
2017
2018 static __net_init int smc_net_init(struct net *net)
2019 {
2020         return smc_pnet_net_init(net);
2021 }
2022
2023 static void __net_exit smc_net_exit(struct net *net)
2024 {
2025         smc_pnet_net_exit(net);
2026 }
2027
2028 static struct pernet_operations smc_net_ops = {
2029         .init = smc_net_init,
2030         .exit = smc_net_exit,
2031         .id   = &smc_net_id,
2032         .size = sizeof(struct smc_net),
2033 };
2034
2035 static int __init smc_init(void)
2036 {
2037         int rc;
2038
2039         rc = register_pernet_subsys(&smc_net_ops);
2040         if (rc)
2041                 return rc;
2042
2043         rc = smc_pnet_init();
2044         if (rc)
2045                 goto out_pernet_subsys;
2046
2047         rc = smc_core_init();
2048         if (rc) {
2049                 pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
2050                 goto out_pnet;
2051         }
2052
2053         rc = smc_llc_init();
2054         if (rc) {
2055                 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
2056                 goto out_core;
2057         }
2058
2059         rc = smc_cdc_init();
2060         if (rc) {
2061                 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
2062                 goto out_core;
2063         }
2064
2065         rc = proto_register(&smc_proto, 1);
2066         if (rc) {
2067                 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
2068                 goto out_core;
2069         }
2070
2071         rc = proto_register(&smc_proto6, 1);
2072         if (rc) {
2073                 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
2074                 goto out_proto;
2075         }
2076
2077         rc = sock_register(&smc_sock_family_ops);
2078         if (rc) {
2079                 pr_err("%s: sock_register fails with %d\n", __func__, rc);
2080                 goto out_proto6;
2081         }
2082         INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
2083         INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
2084
2085         rc = smc_ib_register_client();
2086         if (rc) {
2087                 pr_err("%s: ib_register fails with %d\n", __func__, rc);
2088                 goto out_sock;
2089         }
2090
2091         static_branch_enable(&tcp_have_smc);
2092         return 0;
2093
2094 out_sock:
2095         sock_unregister(PF_SMC);
2096 out_proto6:
2097         proto_unregister(&smc_proto6);
2098 out_proto:
2099         proto_unregister(&smc_proto);
2100 out_core:
2101         smc_core_exit();
2102 out_pnet:
2103         smc_pnet_exit();
2104 out_pernet_subsys:
2105         unregister_pernet_subsys(&smc_net_ops);
2106
2107         return rc;
2108 }
2109
2110 static void __exit smc_exit(void)
2111 {
2112         static_branch_disable(&tcp_have_smc);
2113         sock_unregister(PF_SMC);
2114         smc_core_exit();
2115         smc_ib_unregister_client();
2116         proto_unregister(&smc_proto6);
2117         proto_unregister(&smc_proto);
2118         smc_pnet_exit();
2119         unregister_pernet_subsys(&smc_net_ops);
2120         rcu_barrier();
2121 }
2122
2123 module_init(smc_init);
2124 module_exit(smc_exit);
2125
2126 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
2127 MODULE_DESCRIPTION("smc socket address family");
2128 MODULE_LICENSE("GPL");
2129 MODULE_ALIAS_NETPROTO(PF_SMC);